You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2025/12/02 04:17:03 UTC
(fluss) branch main updated (543c703fb -> 07721fe6e)
This is an automated email from the ASF dual-hosted git repository.
jark pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
from 543c703fb [FLUSS-1961][docs] Upgrade Docusaurus to 3.9.2 (#1962)
new ea2d61c76 [server][client] Support Schema Evolution (ADD COLUMN LAST) in Fluss and connector (#2010)
new 07721fe6e [server][client] Improve the design and performance of ADD COLUMN operation
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
.../org/apache/fluss/client/FlussConnection.java | 1 -
.../org/apache/fluss/client/admin/FlussAdmin.java | 16 +-
.../fluss/client/lookup/AbstractLookuper.java | 180 +++++++++++++
.../fluss/client/lookup/PrefixKeyLookuper.java | 49 ++--
.../apache/fluss/client/lookup/PrefixLookup.java | 5 +-
.../fluss/client/lookup/PrimaryKeyLookuper.java | 51 ++--
.../apache/fluss/client/lookup/TableLookup.java | 18 +-
.../fluss/client/metadata/ClientSchemaGetter.java | 117 +++++++++
.../org/apache/fluss/client/table/FlussTable.java | 10 +-
.../fluss/client/table/scanner/TableScan.java | 29 ++-
.../scanner/batch/KvSnapshotBatchScanner.java | 22 +-
.../table/scanner/batch/LimitBatchScanner.java | 45 +++-
.../table/scanner/batch/SnapshotFilesReader.java | 45 +++-
.../fluss/client/table/scanner/log/LogFetcher.java | 15 +-
.../client/table/scanner/log/LogScannerImpl.java | 9 +-
.../fluss/client/utils/ClientRpcMessageUtils.java | 91 +++++++
.../apache/fluss/client/write/KvWriteBatch.java | 9 +
.../fluss/client/write/RecordAccumulator.java | 6 +-
.../org/apache/fluss/client/write/WriteRecord.java | 6 +-
.../apache/fluss/client/write/WriterClient.java | 1 +
.../client/admin/ClientToServerITCaseBase.java | 4 +
.../fluss/client/admin/FlussAdminITCase.java | 119 ++++++++-
.../fluss/client/lookup/LookupSenderTest.java | 3 +-
.../client/metadata/TestingMetadataUpdater.java | 4 +-
.../fluss/client/table/FlussTableITCase.java | 205 +++++++++++++++
.../fluss/client/table/PartitionedTableITCase.java | 2 +-
.../batch/KvSnapshotBatchScannerITCase.java | 83 +++++-
.../scanner/log/DefaultCompletedFetchTest.java | 24 +-
.../table/scanner/log/LogFetchBufferTest.java | 4 +-
.../table/scanner/log/LogFetchCollectorTest.java | 6 +-
.../client/table/scanner/log/LogFetcherTest.java | 131 +++++++++-
.../scanner/log/RemoteCompletedFetchTest.java | 20 +-
.../fluss/client/write/ArrowLogWriteBatchTest.java | 11 +-
.../client/write/IndexedLogWriteBatchTest.java | 11 +-
.../fluss/client/write/KvWriteBatchTest.java | 5 +-
.../fluss/client/write/RecordAccumulatorTest.java | 9 +-
...erConfigOpType.java => ColumnPositionType.java} | 38 ++-
.../org/apache/fluss/exception/ApiException.java | 16 +-
...stException.java => SchemaChangeException.java} | 19 +-
.../java/org/apache/fluss/metadata/Schema.java | 96 +++++--
.../org/apache/fluss/metadata/SchemaGetter.java | 75 ++++++
.../org/apache/fluss/metadata/TableChange.java | 279 ++++++++++++++++++++
.../java/org/apache/fluss/metadata/TableInfo.java | 22 ++
.../{TimestampAndOffset.java => BinaryValue.java} | 57 ++---
.../apache/fluss/record/DefaultLogRecordBatch.java | 43 +++-
.../fluss/record/DefaultValueRecordBatch.java | 1 +
.../org/apache/fluss/record/FileLogProjection.java | 187 +++++++++-----
.../org/apache/fluss/record/IndexedLogRecord.java | 2 +-
.../apache/fluss/record/KvRecordReadContext.java | 29 ++-
.../org/apache/fluss/record/LogRecordBatch.java | 16 ++
.../apache/fluss/record/LogRecordReadContext.java | 137 ++++++----
.../fluss/record/ProjectionPushdownCache.java | 90 +++++++
.../fluss/record/ValueRecordReadContext.java | 29 ++-
.../main/java/org/apache/fluss/row/PaddingRow.java | 133 ++++++++++
.../java/org/apache/fluss/row/ProjectedRow.java | 18 ++
.../fluss/row/decode/FixedSchemaDecoder.java | 95 +++++++
.../org/apache/fluss/row/decode/RowDecoder.java | 5 +-
.../org/apache/fluss/row/encode/ValueDecoder.java | 49 ++--
.../org/apache/fluss/row/encode/ValueEncoder.java | 2 +-
.../main/java/org/apache/fluss/types/RowType.java | 8 +-
.../java/org/apache/fluss/utils/Projection.java | 84 +++++--
.../java/org/apache/fluss/utils/SchemaUtil.java | 81 ++++++
.../java/org/apache/fluss/utils/UnsafeUtils.java | 4 +
.../apache/fluss/utils/json/ColumnJsonSerde.java | 14 +-
.../apache/fluss/utils/json/SchemaJsonSerde.java | 7 +
.../fluss/record/DefaultKvRecordBatchTest.java | 3 +-
.../apache/fluss/record/DefaultKvRecordTest.java | 7 +-
.../fluss/record/DefaultLogRecordBatchTest.java | 21 +-
.../fluss/record/FileLogInputStreamTest.java | 5 +-
.../apache/fluss/record/FileLogProjectionTest.java | 212 +++++++++++-----
.../org/apache/fluss/record/LogRecordsTest.java | 10 +-
.../java/org/apache/fluss/record/LogTestBase.java | 7 +-
.../record/MemoryLogRecordsArrowBuilderTest.java | 6 +-
.../java/org/apache/fluss/record/TestData.java | 3 +
.../apache/fluss/record/TestingSchemaGetter.java | 83 ++++++
.../java/org/apache/fluss/row/PaddingRowTest.java | 79 ++++++
.../org/apache/fluss/row/ProjectedRowTest.java | 69 +++++
.../org/apache/fluss/testutils/DataTestUtils.java | 94 +++++--
.../fluss/testutils/LogRecordBatchAssert.java | 11 +-
.../apache/fluss/testutils/LogRecordsAssert.java | 9 +
.../org/apache/fluss/utils/ProjectionTest.java | 14 +-
.../fluss/utils/json/ColumnJsonSerdeTest.java | 31 ++-
.../fluss/utils/json/SchemaJsonSerdeTest.java | 40 ++-
.../flink/sink/FlinkRowDataChannelComputer.java | 2 +
.../sink/serializer/FlussSerializationSchema.java | 10 +-
.../serializer/RowDataSerializationSchema.java | 74 +++++-
.../sink/serializer/SerializerInitContextImpl.java | 19 +-
.../fluss/flink/sink/writer/FlinkSinkWriter.java | 23 +-
.../deserializer/RowDataDeserializationSchema.java | 5 +-
.../source/lookup/FlinkAsyncLookupFunction.java | 9 +-
.../flink/source/lookup/FlinkLookupFunction.java | 9 +-
.../source/reader/FlinkSourceSplitReader.java | 11 +-
.../apache/fluss/flink/utils/FlinkConversions.java | 55 ++++
.../fluss/flink/catalog/FlinkCatalogITCase.java | 21 +-
.../fluss/flink/sink/FlinkTableSinkITCase.java | 152 +++++++++++
.../flink/sink/writer/FlinkSinkWriterTest.java | 52 +++-
.../FlussDeserializationSchemaTest.java | 13 +-
.../source/lookup/FlinkLookupFunctionTest.java | 127 +++++++++-
.../source/reader/FlinkSourceSplitReaderTest.java | 38 +--
.../committer/TieringCommitOperatorTest.java | 4 +-
fluss-rpc/src/main/proto/FlussApi.proto | 33 ++-
.../coordinator/CoordinatorEventProcessor.java | 44 ++++
.../coordinator/CoordinatorRequestBatch.java | 5 +-
.../server/coordinator/CoordinatorService.java | 43 +++-
.../fluss/server/coordinator/MetadataManager.java | 37 ++-
.../fluss/server/coordinator/SchemaUpdate.java | 123 +++++++++
...bletServerEvent.java => SchemaChangeEvent.java} | 44 ++--
.../event/watcher/TableChangeWatcher.java | 29 +++
.../java/org/apache/fluss/server/kv/KvManager.java | 22 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 94 ++++---
.../java/org/apache/fluss/server/kv/KvTablet.java | 150 +++++++----
.../server/kv/partialupdate/PartialUpdater.java | 45 ++--
.../kv/partialupdate/PartialUpdaterCache.java | 17 +-
.../server/kv/rowmerger/DefaultRowMerger.java | 28 +--
.../server/kv/rowmerger/FirstRowRowMerger.java | 12 +-
.../fluss/server/kv/rowmerger/RowMerger.java | 33 +--
.../server/kv/rowmerger/VersionedRowMerger.java | 31 ++-
.../org/apache/fluss/server/log/FetchParams.java | 13 +-
.../fluss/server/metadata/ServerSchemaCache.java | 206 +++++++++++++++
.../server/metadata/TabletServerMetadataCache.java | 28 +++
.../org/apache/fluss/server/replica/Replica.java | 67 ++---
.../fluss/server/replica/ReplicaManager.java | 22 +-
.../fluss/server/utils/ServerRpcMessageUtils.java | 84 ++++++-
.../apache/fluss/server/zk/ZooKeeperClient.java | 64 +++--
.../org/apache/fluss/server/zk/data/ZkData.java | 28 +++
.../coordinator/CoordinatorEventProcessorTest.java | 108 +++++++-
.../server/coordinator/CoordinatorTestUtils.java | 23 +-
.../event/watcher/TableChangeWatcherTest.java | 92 ++++++-
.../org/apache/fluss/server/kv/KvManagerTest.java | 84 ++++++-
.../org/apache/fluss/server/kv/KvTabletTest.java | 280 +++++++++++++++++----
.../server/kv/rowmerger/DefaultRowMergerTest.java | 67 +++--
.../kv/rowmerger/VersionedRowMergerTest.java | 74 ++++--
.../fluss/server/log/DroppedTableRecoveryTest.java | 11 +-
.../apache/fluss/server/log/FetchParamsTest.java | 36 ++-
.../apache/fluss/server/log/LogManagerTest.java | 4 +-
.../apache/fluss/server/log/LogSegmentTest.java | 4 +-
.../org/apache/fluss/server/log/LogTabletTest.java | 7 +-
.../server/metadata/ServerSchemaCacheTest.java | 173 +++++++++++++
.../fluss/server/replica/ReplicaManagerTest.java | 79 ++++--
.../apache/fluss/server/replica/ReplicaTest.java | 142 ++++++++++-
.../fluss/server/replica/ReplicaTestBase.java | 8 +-
.../replica/fetcher/ReplicaFetcherManagerTest.java | 2 +-
.../replica/fetcher/ReplicaFetcherThreadTest.java | 2 +-
.../fluss/server/tablet/TabletServiceITCase.java | 9 +-
.../server/tablet/TestTabletServerGateway.java | 15 +-
.../server/testutils/FlussClusterExtension.java | 29 +++
.../server/testutils/RpcMessageTestUtils.java | 37 ++-
.../fluss/server/zk/ZooKeeperClientTest.java | 9 +-
.../apache/fluss/server/zk/data/ZkDataTest.java | 18 ++
149 files changed, 5892 insertions(+), 1053 deletions(-)
create mode 100644 fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java
create mode 100644 fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
copy fluss-common/src/main/java/org/apache/fluss/config/cluster/{AlterConfigOpType.java => ColumnPositionType.java} (58%)
copy fluss-common/src/main/java/org/apache/fluss/exception/{DatabaseAlreadyExistException.java => SchemaChangeException.java} (71%)
create mode 100644 fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
copy fluss-common/src/main/java/org/apache/fluss/record/{TimestampAndOffset.java => BinaryValue.java} (53%)
create mode 100644 fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java
create mode 100644 fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java
create mode 100644 fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java
create mode 100644 fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
create mode 100644 fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
create mode 100644 fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java
create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
copy fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/{DeadTabletServerEvent.java => SchemaChangeEvent.java} (54%)
create mode 100644 fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
create mode 100644 fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
(fluss) 01/02: [server][client] Support Schema Evolution (ADD COLUMN LAST) in Fluss and connector (#2010)
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
commit ea2d61c76ca22ebb926618f7d06d4ced12c0f1d6
Author: Hongshun Wang <lo...@gmail.com>
AuthorDate: Wed Nov 26 19:32:11 2025 +0800
[server][client] Support Schema Evolution (ADD COLUMN LAST) in Fluss and connector (#2010)
---
.../org/apache/fluss/client/FlussConnection.java | 1 -
.../org/apache/fluss/client/admin/FlussAdmin.java | 8 +-
.../fluss/client/lookup/PrefixKeyLookuper.java | 55 ++--
.../fluss/client/lookup/PrimaryKeyLookuper.java | 54 ++--
.../apache/fluss/client/lookup/TableLookup.java | 18 +-
.../fluss/client/metadata/ClientSchemaGetter.java | 80 ++++++
.../org/apache/fluss/client/table/FlussTable.java | 14 +-
.../fluss/client/table/scanner/TableScan.java | 29 ++-
.../client/table/scanner/batch/BatchScanUtils.java | 3 +-
.../scanner/batch/KvSnapshotBatchScanner.java | 22 +-
.../table/scanner/batch/LimitBatchScanner.java | 33 ++-
.../table/scanner/batch/SnapshotFilesReader.java | 31 ++-
.../client/table/scanner/log/BucketScanStatus.java | 3 +-
.../fluss/client/table/scanner/log/LogFetcher.java | 14 +-
.../client/table/scanner/log/LogScannerImpl.java | 9 +-
.../client/table/writer/AbstractTableWriter.java | 2 +
.../fluss/client/utils/ClientRpcMessageUtils.java | 80 ++++++
.../apache/fluss/client/write/KvWriteBatch.java | 9 +
.../fluss/client/write/RecordAccumulator.java | 5 +-
.../java/org/apache/fluss/client/write/Sender.java | 6 +-
.../org/apache/fluss/client/write/WriteRecord.java | 6 +-
.../apache/fluss/client/write/WriterClient.java | 1 +
.../client/admin/ClientToServerITCaseBase.java | 4 +
.../fluss/client/admin/FlussAdminITCase.java | 86 ++++++-
.../fluss/client/lookup/LookupSenderTest.java | 3 +-
.../client/metadata/TestingMetadataUpdater.java | 4 +-
.../fluss/client/table/FlussTableITCase.java | 205 +++++++++++++++
.../fluss/client/table/PartitionedTableITCase.java | 2 +-
.../batch/KvSnapshotBatchScannerITCase.java | 83 +++++-
.../scanner/log/DefaultCompletedFetchTest.java | 21 +-
.../table/scanner/log/LogFetchBufferTest.java | 4 +-
.../table/scanner/log/LogFetchCollectorTest.java | 7 +-
.../client/table/scanner/log/LogFetcherTest.java | 130 +++++++++-
.../scanner/log/RemoteCompletedFetchTest.java | 20 +-
.../fluss/client/write/ArrowLogWriteBatchTest.java | 11 +-
.../client/write/IndexedLogWriteBatchTest.java | 11 +-
.../fluss/client/write/KvWriteBatchTest.java | 5 +-
.../fluss/client/write/RecordAccumulatorTest.java | 10 +-
.../cluster/ColumnPositionType.java} | 37 +--
.../fluss/exception/SchemaChangeException.java | 24 +-
.../java/org/apache/fluss/metadata/Schema.java | 97 +++++--
.../org/apache/fluss/metadata/SchemaGetter.java | 25 +-
.../org/apache/fluss/metadata/TableChange.java | 279 +++++++++++++++++++++
.../java/org/apache/fluss/metadata/TableInfo.java | 17 ++
.../apache/fluss/record/DefaultLogRecordBatch.java | 87 ++++++-
.../fluss/record/DefaultValueRecordBatch.java | 1 +
.../org/apache/fluss/record/FileLogProjection.java | 199 ++++++++++-----
.../org/apache/fluss/record/IndexedLogRecord.java | 2 +-
.../apache/fluss/record/KvRecordReadContext.java | 29 ++-
.../org/apache/fluss/record/LogRecordBatch.java | 16 ++
.../apache/fluss/record/LogRecordReadContext.java | 138 ++++++----
.../fluss/record/ValueRecordReadContext.java | 29 ++-
.../java/org/apache/fluss/row/ProjectedRow.java | 24 +-
.../org/apache/fluss/row/encode/ValueDecoder.java | 33 ++-
.../org/apache/fluss/row/encode/ValueEncoder.java | 23 ++
.../main/java/org/apache/fluss/types/RowType.java | 8 +-
.../java/org/apache/fluss/utils/Projection.java | 88 +++++--
.../java/org/apache/fluss/utils/SchemaUtil.java | 101 ++++++++
.../java/org/apache/fluss/utils/UnsafeUtils.java | 4 +
.../apache/fluss/utils/json/ColumnJsonSerde.java | 14 +-
.../apache/fluss/utils/json/SchemaJsonSerde.java | 7 +
.../fluss/record/DefaultKvRecordBatchTest.java | 3 +-
.../apache/fluss/record/DefaultKvRecordTest.java | 7 +-
.../fluss/record/DefaultLogRecordBatchTest.java | 21 +-
.../fluss/record/FileLogInputStreamTest.java | 5 +-
.../apache/fluss/record/FileLogProjectionTest.java | 209 +++++++++++----
.../org/apache/fluss/record/LogRecordsTest.java | 10 +-
.../java/org/apache/fluss/record/LogTestBase.java | 8 +-
.../record/MemoryLogRecordsArrowBuilderTest.java | 6 +-
.../java/org/apache/fluss/record/TestData.java | 3 +
.../apache/fluss/record/TestingSchemaGetter.java | 73 ++++++
.../org/apache/fluss/row/ProjectedRowTest.java | 68 +++++
.../org/apache/fluss/testutils/DataTestUtils.java | 94 +++++--
.../fluss/testutils/LogRecordBatchAssert.java | 11 +-
.../apache/fluss/testutils/LogRecordsAssert.java | 9 +
.../org/apache/fluss/utils/ProjectionTest.java | 14 +-
.../fluss/utils/json/ColumnJsonSerdeTest.java | 31 ++-
.../fluss/utils/json/SchemaJsonSerdeTest.java | 40 ++-
.../src/test/resources/log4j2-test.properties | 2 +-
.../apache/fluss/flink/row/FlinkAsFlussRow.java | 100 ++++++--
.../flink/sink/FlinkRowDataChannelComputer.java | 2 +
.../sink/serializer/FlussSerializationSchema.java | 9 +-
.../serializer/RowDataSerializationSchema.java | 2 +-
.../sink/serializer/SerializerInitContextImpl.java | 19 +-
.../fluss/flink/sink/writer/FlinkSinkWriter.java | 8 +-
.../org/apache/fluss/flink/source/FlinkSource.java | 7 +-
.../fluss/flink/source/FlinkTableSource.java | 1 -
.../org/apache/fluss/flink/source/FlussSource.java | 4 -
.../fluss/flink/source/FlussSourceBuilder.java | 1 -
.../deserializer/RowDataDeserializationSchema.java | 5 +-
.../source/lookup/FlinkAsyncLookupFunction.java | 20 +-
.../flink/source/lookup/FlinkLookupFunction.java | 18 +-
.../flink/source/reader/FlinkSourceReader.java | 4 -
.../source/reader/FlinkSourceSplitReader.java | 68 ++---
.../apache/fluss/flink/utils/FlinkConversions.java | 67 +++++
.../flink/utils/FlussRowToFlinkRowConverter.java | 39 ++-
.../fluss/flink/catalog/FlinkCatalogITCase.java | 21 +-
.../fluss/flink/row/FlinkAsFlussRowTest.java | 3 +-
.../fluss/flink/sink/FlinkTableSinkITCase.java | 152 +++++++++++
.../flink/sink/writer/FlinkSinkWriterTest.java | 52 +++-
.../FlussDeserializationSchemaTest.java | 13 +-
.../source/lookup/FlinkLookupFunctionTest.java | 123 ++++++++-
.../flink/source/reader/FlinkSourceReaderTest.java | 1 -
.../source/reader/FlinkSourceSplitReaderTest.java | 42 ++--
.../committer/TieringCommitOperatorTest.java | 4 +-
.../fluss/lake/iceberg/conf/HadoopUtils.java | 5 +-
.../iceberg/maintenance/IcebergRewriteITCase.java | 2 +-
.../fluss/rpc/netty/client/ServerConnection.java | 3 +-
fluss-rpc/src/main/proto/FlussApi.proto | 31 ++-
.../org/apache/fluss/server/RpcServiceBase.java | 2 +-
.../coordinator/CompletedSnapshotStoreManager.java | 2 +-
.../coordinator/CoordinatorEventProcessor.java | 45 ++++
.../coordinator/CoordinatorRequestBatch.java | 5 +-
.../server/coordinator/CoordinatorService.java | 43 +++-
.../fluss/server/coordinator/MetadataManager.java | 37 ++-
.../fluss/server/coordinator/SchemaUpdate.java | 119 +++++++++
.../fluss/server/coordinator/UpdateSchema.java | 26 +-
.../coordinator/event/SchemaChangeEvent.java | 62 +++++
.../event/watcher/TableChangeWatcher.java | 38 +++
.../server/entity/BatchRegisterLeadAndIsr.java | 2 +-
.../java/org/apache/fluss/server/kv/KvManager.java | 22 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 47 ++--
.../java/org/apache/fluss/server/kv/KvTablet.java | 140 ++++++++---
.../kv/partialupdate/PartialUpdaterCache.java | 8 +-
.../server/kv/rowmerger/DefaultRowMerger.java | 24 +-
.../server/kv/rowmerger/FirstRowRowMerger.java | 10 +-
.../fluss/server/kv/rowmerger/RowMerger.java | 15 +-
.../server/kv/rowmerger/VersionedRowMerger.java | 28 ++-
.../server/kv/snapshot/CompletedSnapshotStore.java | 4 +-
.../fluss/server/kv/snapshot/SnapshotLocation.java | 2 +-
.../org/apache/fluss/server/log/FetchParams.java | 9 +-
.../java/org/apache/fluss/server/log/LocalLog.java | 2 +
.../server/log/remote/FsRemoteLogOutputStream.java | 2 +-
.../fluss/server/log/remote/LogSegmentFiles.java | 2 +-
.../fluss/server/log/remote/LogTieringTask.java | 4 +-
.../server/metadata/SchemaMetadataManager.java | 193 ++++++++++++++
.../server/metadata/TabletServerMetadataCache.java | 30 +++
.../fluss/server/replica/AdjustIsrManager.java | 2 +-
.../org/apache/fluss/server/replica/Replica.java | 70 ++----
.../fluss/server/replica/ReplicaManager.java | 17 +-
.../replica/fetcher/ReplicaFetcherManager.java | 2 +-
.../replica/fetcher/ReplicaFetcherThread.java | 2 +-
.../fluss/server/utils/RpcGatewayManager.java | 2 +-
.../fluss/server/utils/ServerRpcMessageUtils.java | 84 ++++++-
.../org/apache/fluss/server/utils/timer/Timer.java | 2 +-
.../fluss/server/utils/timer/TimingWheel.java | 4 +-
.../apache/fluss/server/zk/ZooKeeperClient.java | 14 +-
.../org/apache/fluss/server/zk/data/ZkData.java | 28 +++
.../server/authorizer/DefaultAuthorizerTest.java | 4 +-
.../CompletedSnapshotStoreManagerTest.java | 8 +-
.../coordinator/CoordinatorChannelManagerTest.java | 6 +-
.../coordinator/CoordinatorEventProcessorTest.java | 108 +++++++-
.../server/coordinator/CoordinatorTestUtils.java | 23 +-
.../fluss/server/coordinator/TableManagerTest.java | 2 +
.../event/watcher/TableChangeWatcherTest.java | 92 ++++++-
.../org/apache/fluss/server/kv/KvManagerTest.java | 84 ++++++-
.../org/apache/fluss/server/kv/KvTabletTest.java | 273 ++++++++++++++++----
.../server/kv/prewrite/KvPreWriteBufferTest.java | 4 +-
.../server/kv/rowmerger/DefaultRowMergerTest.java | 56 ++++-
.../kv/rowmerger/VersionedRowMergerTest.java | 63 +++--
.../kv/snapshot/CompletedSnapshotStoreTest.java | 2 +-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 2 +-
.../ZooKeeperCompletedSnapshotStoreTest.java | 4 +-
.../fluss/server/log/DroppedTableRecoveryTest.java | 9 +-
.../apache/fluss/server/log/FetchParamsTest.java | 30 ++-
.../apache/fluss/server/log/LogManagerTest.java | 2 +-
.../apache/fluss/server/log/LogSegmentTest.java | 4 +-
.../org/apache/fluss/server/log/LogTabletTest.java | 11 +-
.../server/log/remote/RemoteLogTabletTest.java | 4 +-
.../server/metadata/MetadataUpdateITCase.java | 2 +-
.../server/metadata/SchemaMetadataManagerTest.java | 179 +++++++++++++
.../fluss/server/replica/AdjustIsrITCase.java | 2 +-
.../fluss/server/replica/AdjustIsrManagerTest.java | 2 +-
.../replica/HighWatermarkPersistenceTest.java | 2 +-
.../replica/RemoveOfflineReplicaFromIsrITCase.java | 2 +-
.../fluss/server/replica/ReplicaManagerTest.java | 72 ++++--
.../apache/fluss/server/replica/ReplicaTest.java | 134 +++++++++-
.../fluss/server/replica/ReplicaTestBase.java | 4 +-
.../replica/fetcher/ReplicaFetcherManagerTest.java | 2 +-
.../fluss/server/tablet/TabletServiceITCase.java | 13 +-
.../server/tablet/TestTabletServerGateway.java | 15 +-
.../server/testutils/FlussClusterExtension.java | 31 ++-
.../server/testutils/RpcMessageTestUtils.java | 37 ++-
.../fluss/server/zk/ZooKeeperClientTest.java | 5 +
.../apache/fluss/server/zk/data/ZkDataTest.java | 18 ++
.../src/test/resources/log4j2-test.properties | 2 +-
fluss-test-coverage/pom.xml | 4 +-
187 files changed, 5282 insertions(+), 996 deletions(-)
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
index f29d90ba9..7af5d6b89 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
@@ -100,7 +100,6 @@ public final class FlussConnection implements Connection {
public Table getTable(TablePath tablePath) {
// force to update the table info from server to avoid stale data in cache.
metadataUpdater.updateTableOrPartitionMetadata(tablePath, null);
-
Admin admin = getOrCreateAdmin();
return new FlussTable(this, tablePath, admin.getTableInfo(tablePath).join());
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
index 57dc52008..25bfcfbd8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
@@ -90,6 +90,7 @@ import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
+import static org.apache.fluss.client.utils.ClientRpcMessageUtils.addPbAlterSchemas;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeCreatePartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeDropPartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeListOffsetsRequest;
@@ -249,12 +250,17 @@ public class FlussAdmin implements Admin {
TablePath tablePath, List<TableChange> tableChanges, boolean ignoreIfNotExists) {
tablePath.validate();
AlterTableRequest request = new AlterTableRequest();
-
List<PbAlterConfig> pbFlussTableChanges =
tableChanges.stream()
+ .filter(tableChange -> !(tableChange instanceof TableChange.SchemaChange))
.map(ClientRpcMessageUtils::toPbAlterConfigs)
.collect(Collectors.toList());
+ List<TableChange> schemaChanges =
+ tableChanges.stream()
+ .filter(tableChange -> tableChange instanceof TableChange.SchemaChange)
+ .collect(Collectors.toList());
+ addPbAlterSchemas(request, schemaChanges);
request.addAllConfigChanges(pbFlussTableChanges)
.setIgnoreIfNotExists(ignoreIfNotExists)
.setTablePath()
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
index 61645374f..c9f6308f8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
@@ -22,13 +22,14 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.decode.RowDecoder;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -70,8 +71,11 @@ class PrefixKeyLookuper implements Lookuper {
/** Decode the lookup bytes to result row. */
private final ValueDecoder kvValueDecoder;
+ private final SchemaGetter schemaGetter;
+
public PrefixKeyLookuper(
TableInfo tableInfo,
+ SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient,
List<String> lookupColumnNames) {
@@ -93,10 +97,8 @@ class PrefixKeyLookuper implements Lookuper {
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
this.kvValueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- tableInfo.getTableConfig().getKvFormat(),
- tableInfo.getRowType().getChildren().toArray(new DataType[0])));
+ new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
+ this.schemaGetter = schemaGetter;
}
private void validatePrefixLookup(TableInfo tableInfo, List<String> lookupColumns) {
@@ -167,18 +169,35 @@ class PrefixKeyLookuper implements Lookuper {
}
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- return lookupClient
- .prefixLookup(tableBucket, bucketKeyBytes)
- .thenApply(
- result -> {
- List<InternalRow> rowList = new ArrayList<>(result.size());
- for (byte[] valueBytes : result) {
- if (valueBytes == null) {
- continue;
- }
- rowList.add(kvValueDecoder.decodeValue(valueBytes).row);
+ CompletableFuture<LookupResult> future = new CompletableFuture<>();
+
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ List<byte[]> result =
+ lookupClient.prefixLookup(tableBucket, bucketKeyBytes).get();
+ List<InternalRow> rowList = new ArrayList<>(result.size());
+ for (byte[] valueBytes : result) {
+ if (valueBytes == null) {
+ continue;
+ }
+ ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
+ InternalRow row;
+ if (value.schemaId == tableInfo.getSchemaId()) {
+ row = value.row;
+ } else {
+ Schema schema = schemaGetter.getSchema(value.schemaId);
+ row =
+ ProjectedRow.from(schema, tableInfo.getSchema())
+ .replaceRow(value.row);
}
- return new LookupResult(rowList);
- });
+ rowList.add(row);
+ }
+ future.complete(new LookupResult(rowList));
+ } catch (Exception e) {
+ future.complete(new LookupResult(Collections.emptyList()));
+ }
+ });
+ return future;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
index 2945f1ab4..091a8a38e 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
@@ -22,13 +22,14 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.decode.RowDecoder;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -39,6 +40,7 @@ import java.util.concurrent.CompletableFuture;
import static org.apache.fluss.client.utils.ClientUtils.getPartitionId;
import static org.apache.fluss.utils.Preconditions.checkArgument;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** An implementation of {@link Lookuper} that lookups by primary key. */
@NotThreadSafe
@@ -67,8 +69,13 @@ class PrimaryKeyLookuper implements Lookuper {
/** Decode the lookup bytes to result row. */
private final ValueDecoder kvValueDecoder;
+ private final SchemaGetter schemaGetter;
+
public PrimaryKeyLookuper(
- TableInfo tableInfo, MetadataUpdater metadataUpdater, LookupClient lookupClient) {
+ TableInfo tableInfo,
+ SchemaGetter schemaGetter,
+ MetadataUpdater metadataUpdater,
+ LookupClient lookupClient) {
checkArgument(
tableInfo.hasPrimaryKey(),
"Log table %s doesn't support lookup",
@@ -96,10 +103,8 @@ class PrimaryKeyLookuper implements Lookuper {
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
this.kvValueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- tableInfo.getTableConfig().getKvFormat(),
- tableInfo.getRowType().getChildren().toArray(new DataType[0])));
+ new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
+ this.schemaGetter = schemaGetter;
}
@Override
@@ -127,15 +132,30 @@ class PrimaryKeyLookuper implements Lookuper {
int bucketId = bucketingFunction.bucketing(bkBytes, numBuckets);
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- return lookupClient
- .lookup(tableBucket, pkBytes)
- .thenApply(
- valueBytes -> {
- InternalRow row =
- valueBytes == null
- ? null
- : kvValueDecoder.decodeValue(valueBytes).row;
- return new LookupResult(row);
- });
+ CompletableFuture<LookupResult> future = new CompletableFuture<>();
+
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ byte[] valueBytes = lookupClient.lookup(tableBucket, pkBytes).get();
+ InternalRow row = null;
+ if (valueBytes != null) {
+ ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
+ if (value.schemaId == tableInfo.getSchemaId()) {
+ row = value.row;
+ } else {
+ Schema schema = schemaGetter.getSchema(value.schemaId);
+ checkNotNull(schema, "schema is null");
+ row =
+ ProjectedRow.from(schema, tableInfo.getSchema())
+ .replaceRow(value.row);
+ }
+ }
+ future.complete(new LookupResult(row));
+ } catch (Exception e) {
+ future.completeExceptionally(e);
+ }
+ });
+ return future;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java
index 2212f123c..84c9bdb21 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java
@@ -18,6 +18,7 @@
package org.apache.fluss.client.lookup;
import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableInfo;
import javax.annotation.Nullable;
@@ -28,22 +29,28 @@ import java.util.List;
public class TableLookup implements Lookup {
private final TableInfo tableInfo;
+ private final SchemaGetter schemaGetter;
private final MetadataUpdater metadataUpdater;
private final LookupClient lookupClient;
@Nullable private final List<String> lookupColumnNames;
public TableLookup(
- TableInfo tableInfo, MetadataUpdater metadataUpdater, LookupClient lookupClient) {
- this(tableInfo, metadataUpdater, lookupClient, null);
+ TableInfo tableInfo,
+ SchemaGetter schemaGetter,
+ MetadataUpdater metadataUpdater,
+ LookupClient lookupClient) {
+ this(tableInfo, schemaGetter, metadataUpdater, lookupClient, null);
}
private TableLookup(
TableInfo tableInfo,
+ SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient,
@Nullable List<String> lookupColumnNames) {
this.tableInfo = tableInfo;
+ this.schemaGetter = schemaGetter;
this.metadataUpdater = metadataUpdater;
this.lookupClient = lookupClient;
this.lookupColumnNames = lookupColumnNames;
@@ -51,16 +58,17 @@ public class TableLookup implements Lookup {
@Override
public Lookup lookupBy(List<String> lookupColumnNames) {
- return new TableLookup(tableInfo, metadataUpdater, lookupClient, lookupColumnNames);
+ return new TableLookup(
+ tableInfo, schemaGetter, metadataUpdater, lookupClient, lookupColumnNames);
}
@Override
public Lookuper createLookuper() {
if (lookupColumnNames == null) {
- return new PrimaryKeyLookuper(tableInfo, metadataUpdater, lookupClient);
+ return new PrimaryKeyLookuper(tableInfo, schemaGetter, metadataUpdater, lookupClient);
} else {
return new PrefixKeyLookuper(
- tableInfo, metadataUpdater, lookupClient, lookupColumnNames);
+ tableInfo, schemaGetter, metadataUpdater, lookupClient, lookupColumnNames);
}
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
new file mode 100644
index 000000000..181767781
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.metadata;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TablePath;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.fluss.utils.MapUtils.newConcurrentHashMap;
+
+/** Schema getter for client. */
+@Internal
+public class ClientSchemaGetter implements SchemaGetter {
+ private static final Logger LOG = LoggerFactory.getLogger(ClientSchemaGetter.class);
+
+ private final TablePath tablePath;
+ private final Map<Integer, Schema> schemasById;
+ private final Admin admin;
+ private SchemaInfo latestSchemaInfo;
+
+ public ClientSchemaGetter(TablePath tablePath, SchemaInfo latestSchemaInfo, Admin admin) {
+ this.tablePath = tablePath;
+ this.latestSchemaInfo = latestSchemaInfo;
+ this.admin = admin;
+ this.schemasById = newConcurrentHashMap();
+ schemasById.put(latestSchemaInfo.getSchemaId(), latestSchemaInfo.getSchema());
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ return schemasById.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ try {
+ SchemaInfo schemaInfo =
+ admin.getTableSchema(tablePath, schemaId).get(1, TimeUnit.MINUTES);
+ if (id > latestSchemaInfo.getSchemaId()) {
+ latestSchemaInfo = schemaInfo;
+ }
+ return schemaInfo.getSchema();
+
+ } catch (Exception e) {
+ LOG.warn("Failed to get schema for table: " + tablePath);
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ return latestSchemaInfo;
+ }
+
+ @Override
+ public void release() {}
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
index 6a982a699..629abe6ab 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
@@ -21,12 +21,15 @@ import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.client.FlussConnection;
import org.apache.fluss.client.lookup.Lookup;
import org.apache.fluss.client.lookup.TableLookup;
+import org.apache.fluss.client.metadata.ClientSchemaGetter;
import org.apache.fluss.client.table.scanner.Scan;
import org.apache.fluss.client.table.scanner.TableScan;
import org.apache.fluss.client.table.writer.Append;
import org.apache.fluss.client.table.writer.TableAppend;
import org.apache.fluss.client.table.writer.TableUpsert;
import org.apache.fluss.client.table.writer.Upsert;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -44,12 +47,18 @@ public class FlussTable implements Table {
private final TablePath tablePath;
private final TableInfo tableInfo;
private final boolean hasPrimaryKey;
+ private final SchemaGetter schemaGetter;
public FlussTable(FlussConnection conn, TablePath tablePath, TableInfo tableInfo) {
this.conn = conn;
this.tablePath = tablePath;
this.tableInfo = tableInfo;
this.hasPrimaryKey = tableInfo.hasPrimaryKey();
+ this.schemaGetter =
+ new ClientSchemaGetter(
+ tableInfo.getTablePath(),
+ new SchemaInfo(tableInfo.getSchema(), tableInfo.getSchemaId()),
+ conn.getAdmin());
}
@Override
@@ -59,13 +68,13 @@ public class FlussTable implements Table {
@Override
public Scan newScan() {
- return new TableScan(conn, tableInfo);
+ return new TableScan(conn, tableInfo, schemaGetter);
}
@Override
public Lookup newLookup() {
return new TableLookup(
- tableInfo, conn.getMetadataUpdater(), conn.getOrCreateLookupClient());
+ tableInfo, schemaGetter, conn.getMetadataUpdater(), conn.getOrCreateLookupClient());
}
@Override
@@ -89,5 +98,6 @@ public class FlussTable implements Table {
@Override
public void close() throws Exception {
// do nothing
+ schemaGetter.release();
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java
index 7fade0547..0ca1952cf 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java
@@ -27,6 +27,7 @@ import org.apache.fluss.client.table.scanner.log.LogScanner;
import org.apache.fluss.client.table.scanner.log.LogScannerImpl;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.types.RowType;
@@ -40,30 +41,33 @@ public class TableScan implements Scan {
private final FlussConnection conn;
private final TableInfo tableInfo;
+ private final SchemaGetter schemaGetter;
/** The projected fields to do projection. No projection if is null. */
@Nullable private final int[] projectedColumns;
/** The limited row number to read. No limit if is null. */
@Nullable private final Integer limit;
- public TableScan(FlussConnection conn, TableInfo tableInfo) {
- this(conn, tableInfo, null, null);
+ public TableScan(FlussConnection conn, TableInfo tableInfo, SchemaGetter schemaGetter) {
+ this(conn, tableInfo, schemaGetter, null, null);
}
private TableScan(
FlussConnection conn,
TableInfo tableInfo,
+ SchemaGetter schemaGetter,
@Nullable int[] projectedColumns,
@Nullable Integer limit) {
this.conn = conn;
this.tableInfo = tableInfo;
this.projectedColumns = projectedColumns;
this.limit = limit;
+ this.schemaGetter = schemaGetter;
}
@Override
public Scan project(@Nullable int[] projectedColumns) {
- return new TableScan(conn, tableInfo, projectedColumns, limit);
+ return new TableScan(conn, tableInfo, schemaGetter, projectedColumns, limit);
}
@Override
@@ -82,12 +86,12 @@ public class TableScan implements Scan {
}
columnIndexes[i] = index;
}
- return new TableScan(conn, tableInfo, columnIndexes, limit);
+ return new TableScan(conn, tableInfo, schemaGetter, columnIndexes, limit);
}
@Override
public Scan limit(int rowNumber) {
- return new TableScan(conn, tableInfo, projectedColumns, rowNumber);
+ return new TableScan(conn, tableInfo, schemaGetter, projectedColumns, rowNumber);
}
@Override
@@ -98,13 +102,15 @@ public class TableScan implements Scan {
"LogScanner doesn't support limit pushdown. Table: %s, requested limit: %d",
tableInfo.getTablePath(), limit));
}
+
return new LogScannerImpl(
conn.getConfiguration(),
tableInfo,
conn.getMetadataUpdater(),
conn.getClientMetricGroup(),
conn.getOrCreateRemoteFileDownloader(),
- projectedColumns);
+ projectedColumns,
+ schemaGetter);
}
@Override
@@ -116,7 +122,12 @@ public class TableScan implements Scan {
tableInfo.getTablePath(), tableBucket));
}
return new LimitBatchScanner(
- tableInfo, tableBucket, conn.getMetadataUpdater(), projectedColumns, limit);
+ tableInfo,
+ tableBucket,
+ schemaGetter,
+ conn.getMetadataUpdater(),
+ projectedColumns,
+ limit);
}
@Override
@@ -142,7 +153,9 @@ public class TableScan implements Scan {
}
return new KvSnapshotBatchScanner(
- tableInfo.getRowType(),
+ tableInfo.getSchemaId(),
+ tableInfo.getSchema(),
+ schemaGetter,
tableBucket,
snapshotMeta.getSnapshotFiles(),
projectedColumns,
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
index d36bf9a7d..f4e150f30 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
@@ -71,7 +71,8 @@ public class BatchScanUtils {
// If the scanner has more data, add it back to the queue
scannerQueue.add(scanner);
} else {
- // Close the scanner if it has no more data, and not add it back to the queue
+ // Close the scanner if it has no more data, and not add it back to the
+ // queue
scanner.close();
}
} catch (Exception e) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java
index 2c749c662..52a2c36fb 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java
@@ -22,9 +22,10 @@ import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.fs.FsPathAndFileName;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.FileUtils;
@@ -72,7 +73,10 @@ public class KvSnapshotBatchScanner implements BatchScanner {
public static final CloseableIterator<InternalRow> NO_DATA_AVAILABLE =
CloseableIterator.emptyIterator();
- private final RowType tableRowType;
+ private final Schema targetSchema;
+ private final int targetSchemaId;
+ private final SchemaGetter schemaGetter;
+
private final TableBucket tableBucket;
private final List<FsPathAndFileName> fsPathAndFileNames;
@Nullable private final int[] projectedFields;
@@ -93,14 +97,18 @@ public class KvSnapshotBatchScanner implements BatchScanner {
@Nullable private volatile Throwable initSnapshotFilesReaderException = null;
public KvSnapshotBatchScanner(
- RowType tableRowType,
+ int targetSchemaId,
+ Schema targetSchema,
+ SchemaGetter schemaGetter,
TableBucket tableBucket,
List<FsPathAndFileName> fsPathAndFileNames,
@Nullable int[] projectedFields,
String scannerTmpDir,
KvFormat kvFormat,
RemoteFileDownloader remoteFileDownloader) {
- this.tableRowType = tableRowType;
+ this.targetSchema = targetSchema;
+ this.targetSchemaId = targetSchemaId;
+ this.schemaGetter = schemaGetter;
this.tableBucket = tableBucket;
this.fsPathAndFileNames = fsPathAndFileNames;
this.projectedFields = projectedFields;
@@ -209,8 +217,10 @@ public class KvSnapshotBatchScanner implements BatchScanner {
new SnapshotFilesReader(
kvFormat,
snapshotLocalDirectory,
- tableRowType,
- projectedFields);
+ projectedFields,
+ targetSchemaId,
+ targetSchema,
+ schemaGetter);
readerIsReady.signalAll();
} catch (Throwable e) {
IOUtils.closeQuietly(closeableRegistry);
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
index 4915ab918..e6208f368 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
@@ -19,6 +19,8 @@ package org.apache.fluss.client.table.scanner.batch;
import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.exception.LeaderNotAvailableException;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.DefaultValueRecordBatch;
@@ -32,12 +34,9 @@ import org.apache.fluss.record.ValueRecordReadContext;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.ProjectedRow;
-import org.apache.fluss.row.decode.RowDecoder;
-import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
import org.apache.fluss.rpc.messages.LimitScanRequest;
import org.apache.fluss.rpc.messages.LimitScanResponse;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
@@ -61,20 +60,24 @@ public class LimitBatchScanner implements BatchScanner {
private final int limit;
private final InternalRow.FieldGetter[] fieldGetters;
private final CompletableFuture<LimitScanResponse> scanFuture;
- private final ValueDecoder kvValueDecoder;
+ private final SchemaGetter schemaGetter;
+ private final KvFormat kvFormat;
+ private final int targetSchemaId;
private boolean endOfInput;
public LimitBatchScanner(
TableInfo tableInfo,
TableBucket tableBucket,
+ SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
@Nullable int[] projectedFields,
int limit) {
this.tableInfo = tableInfo;
this.projectedFields = projectedFields;
this.limit = limit;
-
+ this.targetSchemaId = tableInfo.getSchemaId();
+ this.schemaGetter = schemaGetter;
RowType rowType = tableInfo.getRowType();
this.fieldGetters = new InternalRow.FieldGetter[rowType.getFieldCount()];
for (int i = 0; i < rowType.getFieldCount(); i++) {
@@ -102,11 +105,7 @@ public class LimitBatchScanner implements BatchScanner {
}
this.scanFuture = gateway.limitScan(limitScanRequest);
- this.kvValueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- tableInfo.getTableConfig().getKvFormat(),
- rowType.getChildren().toArray(new DataType[0])));
+ this.kvFormat = tableInfo.getTableConfig().getKvFormat();
this.endOfInput = false;
}
@@ -139,13 +138,21 @@ public class LimitBatchScanner implements BatchScanner {
DefaultValueRecordBatch valueRecords =
DefaultValueRecordBatch.pointToByteBuffer(recordsBuffer);
ValueRecordReadContext readContext =
- new ValueRecordReadContext(kvValueDecoder.getRowDecoder());
+ ValueRecordReadContext.createReadContext(schemaGetter, kvFormat);
for (ValueRecord record : valueRecords.records(readContext)) {
- scanRows.add(maybeProject(record.getRow()));
+ InternalRow row = record.getRow();
+ if (targetSchemaId != record.schemaId()) {
+ row =
+ ProjectedRow.from(
+ schemaGetter.getSchema(record.schemaId()),
+ schemaGetter.getSchema(targetSchemaId))
+ .replaceRow(row);
+ }
+ scanRows.add(maybeProject(row));
}
} else {
LogRecordReadContext readContext =
- LogRecordReadContext.createReadContext(tableInfo, false, null);
+ LogRecordReadContext.createReadContext(tableInfo, false, null, schemaGetter);
LogRecords records = MemoryLogRecords.pointToByteBuffer(recordsBuffer);
for (LogRecordBatch logRecordBatch : records.batches()) {
// A batch of log record maybe little more than limit, thus we need slice the
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
index 3c95ab39f..10e773609 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
@@ -20,14 +20,13 @@ package org.apache.fluss.client.table.scanner.batch;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.rocksdb.RocksDBHandle;
import org.apache.fluss.rocksdb.RocksIteratorWrapper;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.ProjectedRow;
-import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.types.DataType;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.IOUtils;
@@ -52,6 +51,9 @@ import java.nio.file.Path;
@NotThreadSafe
class SnapshotFilesReader implements CloseableIterator<InternalRow> {
+ private final int targetSchemaId;
+ private final Schema targetSchema;
+ private final SchemaGetter schemaGetter;
private final ValueDecoder valueDecoder;
@Nullable private final int[] projectedFields;
private RocksIteratorWrapper rocksIteratorWrapper;
@@ -65,13 +67,15 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
SnapshotFilesReader(
KvFormat kvFormat,
Path rocksDbPath,
- RowType tableRowType,
- @Nullable int[] projectedFields)
+ @Nullable int[] projectedFields,
+ int targetSchemaId,
+ Schema targetSchema,
+ SchemaGetter schemaGetter)
throws IOException {
- this.valueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- kvFormat, tableRowType.getChildren().toArray(new DataType[0])));
+ this.targetSchemaId = targetSchemaId;
+ this.targetSchema = targetSchema;
+ this.schemaGetter = schemaGetter;
+ this.valueDecoder = new ValueDecoder(schemaGetter, kvFormat);
this.projectedFields = projectedFields;
closeableRegistry = new CloseableRegistry();
try {
@@ -151,7 +155,14 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
byte[] value = rocksIteratorWrapper.value();
rocksIteratorWrapper.next();
- InternalRow originRow = valueDecoder.decodeValue(value).row;
+ ValueDecoder.Value originValue = valueDecoder.decodeValue(value);
+ InternalRow originRow = originValue.row;
+ if (targetSchemaId != originValue.schemaId) {
+ originRow =
+ ProjectedRow.from(schemaGetter.getSchema(originValue.schemaId), targetSchema)
+ .replaceRow(originRow);
+ }
+
if (projectedFields != null) {
ProjectedRow projectedRow = ProjectedRow.from(projectedFields);
projectedRow.replaceRow(originRow);
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
index 738e04a49..bee0910e1 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
@@ -24,7 +24,8 @@ import org.apache.fluss.annotation.Internal;
class BucketScanStatus {
private long offset; // last consumed position
private long highWatermark; // the high watermark from last fetch
- // TODO add resetStrategy and nextAllowedRetryTimeMs.
+ // TODO add
+ // resetStrategy and nextAllowedRetryTimeMs.
public BucketScanStatus() {
this.offset = 0L;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
index f2df66aaa..e8067dd4c 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
@@ -30,6 +30,7 @@ import org.apache.fluss.exception.InvalidMetadataException;
import org.apache.fluss.exception.LeaderNotAvailableException;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePartition;
@@ -115,12 +116,14 @@ public class LogFetcher implements Closeable {
Configuration conf,
MetadataUpdater metadataUpdater,
ScannerMetricGroup scannerMetricGroup,
- RemoteFileDownloader remoteFileDownloader) {
+ RemoteFileDownloader remoteFileDownloader,
+ SchemaGetter schemaGetter) {
this.tablePath = tableInfo.getTablePath();
this.isPartitioned = tableInfo.isPartitioned();
- this.readContext = LogRecordReadContext.createReadContext(tableInfo, false, projection);
+ this.readContext =
+ LogRecordReadContext.createReadContext(tableInfo, false, projection, schemaGetter);
this.remoteReadContext =
- LogRecordReadContext.createReadContext(tableInfo, true, projection);
+ LogRecordReadContext.createReadContext(tableInfo, true, projection, schemaGetter);
this.projection = projection;
this.logScannerStatus = logScannerStatus;
this.maxFetchBytes =
@@ -360,7 +363,8 @@ public class LogFetcher implements Closeable {
LogRecords logRecords = fetchResultForBucket.recordsOrEmpty();
if (!MemoryLogRecords.EMPTY.equals(logRecords)
|| fetchResultForBucket.getErrorCode() != Errors.NONE.code()) {
- // In oder to not signal notEmptyCondition, add completed fetch to
+ // In oder to not signal notEmptyCondition, add completed
+ // fetch to
// buffer until log records is not empty.
DefaultCompletedFetch completedFetch =
new DefaultCompletedFetch(
@@ -479,7 +483,7 @@ public class LogFetcher implements Closeable {
assert projection != null;
reqForTable
.setProjectionPushdownEnabled(true)
- .setProjectedFields(projection.getProjectionInOrder());
+ .setProjectedFields(projection.getProjectionIdInOrder());
} else {
reqForTable.setProjectionPushdownEnabled(false);
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
index cef05808f..b50bc3622 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
@@ -24,6 +24,7 @@ import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.WakeupException;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -82,7 +83,8 @@ public class LogScannerImpl implements LogScanner {
MetadataUpdater metadataUpdater,
ClientMetricGroup clientMetricGroup,
RemoteFileDownloader remoteFileDownloader,
- @Nullable int[] projectedFields) {
+ @Nullable int[] projectedFields,
+ SchemaGetter schemaGetter) {
this.tablePath = tableInfo.getTablePath();
this.tableId = tableInfo.getTableId();
this.isPartitionedTable = tableInfo.isPartitioned();
@@ -100,7 +102,8 @@ public class LogScannerImpl implements LogScanner {
conf,
metadataUpdater,
scannerMetricGroup,
- remoteFileDownloader);
+ remoteFileDownloader,
+ schemaGetter);
}
/**
@@ -120,7 +123,7 @@ public class LogScannerImpl implements LogScanner {
+ tableRowType);
}
}
- return Projection.of(projectedFields);
+ return Projection.of(projectedFields, tableInfo.getSchema());
} else {
return null;
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
index 98a93fdd2..f1811dfa2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
@@ -37,11 +37,13 @@ public abstract class AbstractTableWriter implements TableWriter {
protected final TablePath tablePath;
protected final WriterClient writerClient;
protected final int fieldCount;
+ protected TableInfo tableInfo;
private final @Nullable PartitionGetter partitionFieldGetter;
protected AbstractTableWriter(
TablePath tablePath, TableInfo tableInfo, WriterClient writerClient) {
this.tablePath = tablePath;
+ this.tableInfo = tableInfo;
this.writerClient = writerClient;
this.fieldCount = tableInfo.getRowType().getFieldCount();
this.partitionFieldGetter =
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
index b98b6fb6c..8b36b5cf8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
@@ -26,6 +26,7 @@ import org.apache.fluss.client.metadata.LakeSnapshot;
import org.apache.fluss.client.write.KvWriteBatch;
import org.apache.fluss.client.write.ReadyWriteBatch;
import org.apache.fluss.config.cluster.AlterConfigOpType;
+import org.apache.fluss.config.cluster.ColumnPositionType;
import org.apache.fluss.config.cluster.ConfigEntry;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.FsPathAndFileName;
@@ -36,6 +37,7 @@ import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.rpc.messages.AlterTableRequest;
import org.apache.fluss.rpc.messages.CreatePartitionRequest;
import org.apache.fluss.rpc.messages.DropPartitionRequest;
import org.apache.fluss.rpc.messages.GetFileSystemSecurityTokenResponse;
@@ -46,20 +48,26 @@ import org.apache.fluss.rpc.messages.ListOffsetsRequest;
import org.apache.fluss.rpc.messages.ListPartitionInfosResponse;
import org.apache.fluss.rpc.messages.LookupRequest;
import org.apache.fluss.rpc.messages.MetadataRequest;
+import org.apache.fluss.rpc.messages.PbAddColumn;
import org.apache.fluss.rpc.messages.PbAlterConfig;
import org.apache.fluss.rpc.messages.PbDescribeConfig;
+import org.apache.fluss.rpc.messages.PbDropColumn;
import org.apache.fluss.rpc.messages.PbKeyValue;
import org.apache.fluss.rpc.messages.PbKvSnapshot;
import org.apache.fluss.rpc.messages.PbLakeSnapshotForBucket;
import org.apache.fluss.rpc.messages.PbLookupReqForBucket;
+import org.apache.fluss.rpc.messages.PbModifyColumn;
import org.apache.fluss.rpc.messages.PbPartitionSpec;
import org.apache.fluss.rpc.messages.PbPrefixLookupReqForBucket;
import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket;
import org.apache.fluss.rpc.messages.PbPutKvReqForBucket;
import org.apache.fluss.rpc.messages.PbRemotePathAndLocalFile;
+import org.apache.fluss.rpc.messages.PbRenameColumn;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.PutKvRequest;
+import org.apache.fluss.utils.json.DataTypeJsonSerde;
+import org.apache.fluss.utils.json.JsonSerdeUtils;
import javax.annotation.Nullable;
@@ -372,6 +380,78 @@ public class ClientRpcMessageUtils {
return info;
}
+ public static void addPbAlterSchemas(
+ AlterTableRequest request, List<TableChange> tableChanges) {
+ List<PbAddColumn> addColumns = new ArrayList<>();
+ List<PbDropColumn> dropColumns = new ArrayList<>();
+ List<PbRenameColumn> renameColumns = new ArrayList<>();
+ List<PbModifyColumn> modifyColumns = new ArrayList<>();
+ for (TableChange tableChange : tableChanges) {
+ if (tableChange instanceof TableChange.AddColumn) {
+ addColumns.add(toPbAddColumn((TableChange.AddColumn) tableChange));
+ } else if (tableChange instanceof TableChange.DropColumn) {
+ dropColumns.add(toPbDropColumn((TableChange.DropColumn) tableChange));
+ } else if (tableChange instanceof TableChange.RenameColumn) {
+ renameColumns.add(toPbRenameColumn((TableChange.RenameColumn) tableChange));
+ } else if (tableChange instanceof TableChange.ModifyColumn) {
+ modifyColumns.add(toPbModifyColumn((TableChange.ModifyColumn) tableChange));
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported table change: " + tableChange.getClass());
+ }
+ }
+
+ request.addAllAddColumns(addColumns)
+ .addAllDropColumns(dropColumns)
+ .addAllRenameColumns(renameColumns)
+ .addAllModifyColumns(modifyColumns);
+ }
+
+ public static PbAddColumn toPbAddColumn(TableChange.AddColumn addColumn) {
+ ColumnPositionType columnPositionType = ColumnPositionType.from(addColumn.getPosition());
+
+ PbAddColumn pbAddColumn =
+ new PbAddColumn()
+ .setColumnName(addColumn.getName())
+ .setDataTypeJson(
+ JsonSerdeUtils.writeValueAsBytes(
+ addColumn.getDataType(), DataTypeJsonSerde.INSTANCE))
+ .setColumnPositionType(columnPositionType.value());
+ if (addColumn.getComment() != null) {
+ pbAddColumn.setComment(addColumn.getComment());
+ }
+
+ return pbAddColumn;
+ }
+
+ public static PbDropColumn toPbDropColumn(TableChange.DropColumn dropColumn) {
+ return new PbDropColumn().setColumnName(dropColumn.getName());
+ }
+
+ public static PbRenameColumn toPbRenameColumn(TableChange.RenameColumn dropColumn) {
+ return new PbRenameColumn()
+ .setOldColumnName(dropColumn.getOldColumnName())
+ .setNewColumnName(dropColumn.getNewColumnName());
+ }
+
+ public static PbModifyColumn toPbModifyColumn(TableChange.ModifyColumn modifyColumn) {
+ PbModifyColumn pbModifyColumn =
+ new PbModifyColumn()
+ .setColumnName(modifyColumn.getName())
+ .setDataTypeJson(
+ JsonSerdeUtils.writeValueAsBytes(
+ modifyColumn.getDataType(), DataTypeJsonSerde.INSTANCE));
+ if (modifyColumn.getNewPosition() != null) {
+ ColumnPositionType columnPositionType =
+ ColumnPositionType.from(modifyColumn.getNewPosition());
+ pbModifyColumn.setColumnPositionType(columnPositionType.value());
+ }
+ if (modifyColumn.getComment() != null) {
+ pbModifyColumn.setComment(modifyColumn.getComment());
+ }
+ return pbModifyColumn;
+ }
+
public static List<ConfigEntry> toConfigEntries(List<PbDescribeConfig> pbDescribeConfigs) {
return pbDescribeConfigs.stream()
.map(
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
index 963e9866b..72f596be9 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
@@ -50,6 +50,7 @@ public class KvWriteBatch extends WriteBatch {
private final AbstractPagedOutputView outputView;
private final KvRecordBatchBuilder recordsBuilder;
private final @Nullable int[] targetColumns;
+ private final int schemaId;
public KvWriteBatch(
int bucketId,
@@ -65,6 +66,7 @@ public class KvWriteBatch extends WriteBatch {
this.recordsBuilder =
KvRecordBatchBuilder.builder(schemaId, writeLimit, outputView, kvFormat);
this.targetColumns = targetColumns;
+ this.schemaId = schemaId;
}
@Override
@@ -74,6 +76,13 @@ public class KvWriteBatch extends WriteBatch {
@Override
public boolean tryAppend(WriteRecord writeRecord, WriteCallback callback) throws Exception {
+ if (schemaId != writeRecord.getSchemaId()) {
+ throw new IllegalStateException(
+ String.format(
+ "schema id %d of the write record to append is not the same as the current schema id %d in the batch.",
+ writeRecord.getSchemaId(), schemaId));
+ }
+
// currently, we throw exception directly when the target columns of the write record is
// not the same as the current target columns in the batch.
// this should be quite fast as they should be the same objects.
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
index 9dd8bade6..f7d7d3ef0 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
@@ -867,7 +867,8 @@ public final class RecordAccumulator {
orderedBatches.size());
// Either we have reached a point where there are batches without a sequence (i.e. never
// been drained and are hence in order by default), or the batch at the front of the
- // queue has a sequence greater than the incoming batch. This is the right place to add
+ // queue has a sequence greater than the incoming batch. This is the right place to
+ // add
// the incoming batch.
deque.addFirst(batch);
@@ -920,6 +921,8 @@ public final class RecordAccumulator {
writerBufferPool.close();
arrowWriterPool.close();
+ // Release all the memory segments.
+ bufferAllocator.releaseBytes(bufferAllocator.getAllocatedMemory());
bufferAllocator.close();
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
index e9c285306..1b9c79397 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
@@ -130,7 +130,7 @@ public class Sender implements Runnable {
this.idempotenceManager = idempotenceManager;
this.writerMetricGroup = writerMetricGroup;
- // TODO add retry logic while send failed. See FLUSS-56364375
+ // TODOadd retry logic while send failed. See FLUSS-56364375
}
@VisibleForTesting
@@ -167,7 +167,7 @@ public class Sender implements Runnable {
}
}
- // TODO if force close failed, add logic to abort incomplete batches.
+ // TODO if force close failed,add logic to abort incomplete batches.
LOG.debug("Shutdown of Fluss write sender I/O thread has completed.");
}
@@ -245,7 +245,7 @@ public class Sender implements Runnable {
if (!batches.isEmpty()) {
addToInflightBatches(batches);
- // TODO add logic for batch expire.
+ // TODOadd logic for batch expire.
sendWriteRequests(batches);
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java
index e1ab2ec24..68a3fe8a2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java
@@ -50,7 +50,7 @@ public final class WriteRecord {
@Nullable int[] targetColumns) {
checkNotNull(row, "row must not be null");
checkNotNull(key, "key must not be null");
- checkNotNull(bucketKey, "key must not be null");
+ checkNotNull(bucketKey, "bucketKey must not be null");
int estimatedSizeInBytes =
DefaultKvRecord.sizeOf(key, row) + DefaultKvRecordBatch.RECORD_BATCH_HEADER_SIZE;
return new WriteRecord(
@@ -205,4 +205,8 @@ public final class WriteRecord {
}
return estimatedSizeInBytes;
}
+
+ public int getSchemaId() {
+ return tableInfo.getSchemaId();
+ }
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
index f61e6f544..788d385a1 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
@@ -120,6 +120,7 @@ public class WriterClient {
conf.get(ConfigOptions.CLIENT_WRITER_DYNAMIC_CREATE_PARTITION_ENABLED),
this::maybeAbortBatches);
} catch (Throwable t) {
+ LOG.error("Failed to construct writer.", t);
close(Duration.ofMillis(0));
throw new FlussRuntimeException(
String.format(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java
index 57cdd1c8b..a62c1b70d 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java
@@ -240,6 +240,10 @@ public abstract class ClientToServerITCaseBase {
}
}
+ public static void waitAllSchemaSync(TablePath tablePath, int schemaId) {
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(tablePath, schemaId);
+ }
+
protected static void verifyRows(
RowType rowType,
Map<Long, List<InternalRow>> actualRows,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
index 10890f947..05f196fdd 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
@@ -228,7 +228,7 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
}
@Test
- void testAlterTable() throws Exception {
+ void testAlterTableConfig() throws Exception {
// create table
TablePath tablePath = TablePath.of("test_db", "alter_table_1");
admin.createTable(tablePath, DEFAULT_TABLE_DESCRIPTOR, false).get();
@@ -302,6 +302,90 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
.get();
}
+ @Test
+ void testAlterTableColumn() throws Exception {
+ // create table
+ TablePath tablePath = TablePath.of("test_db", "alter_table_1");
+ admin.createTable(tablePath, DEFAULT_TABLE_DESCRIPTOR, false).get();
+
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.dropColumn("id")),
+ false)
+ .get())
+ .hasMessageContaining("Not support drop column now.");
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.renameColumn("id", "id2")),
+ false)
+ .get())
+ .hasMessageContaining("Not support rename column now.");
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.modifyColumn(
+ "id",
+ DataTypes.INT(),
+ "person id",
+ null)),
+ false)
+ .get())
+ .hasMessageContaining("Not support modify column now.");
+
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING().copy(false),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get())
+ .hasMessageContaining("Column c1 must be nullable");
+
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+
+ Schema expectedSchema =
+ Schema.newBuilder()
+ .primaryKey("id")
+ .fromColumns(
+ Arrays.asList(
+ new Schema.Column(
+ "id", DataTypes.INT(), "person id", (short) 0),
+ new Schema.Column(
+ "name",
+ DataTypes.STRING(),
+ "person name",
+ (short) 1),
+ new Schema.Column(
+ "age", DataTypes.INT(), "person age", (short) 2),
+ new Schema.Column(
+ "c1", DataTypes.STRING(), null, (short) 3)))
+ .build();
+ SchemaInfo schemaInfo = admin.getTableSchema(tablePath).get();
+ assertThat(schemaInfo).isEqualTo(new SchemaInfo(expectedSchema, 2));
+ }
+
@Test
void testCreateInvalidDatabaseAndTable() throws Exception {
assertThatThrownBy(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java
index 61dc5cbc8..f56535b01 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Duration;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -478,7 +479,7 @@ public class LookupSenderTest {
prefixLookupHandler;
public ConfigurableTestTabletServerGateway() {
- super(false);
+ super(false, Collections.emptySet());
}
public void setLookupHandler(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java b/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java
index 6ca025f33..b89025a78 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java
@@ -74,7 +74,9 @@ public class TestingMetadataUpdater extends MetadataUpdater {
} else {
tabletServerGatewayMap = new HashMap<>();
for (ServerNode tabletServer : tabletServers) {
- tabletServerGatewayMap.put(tabletServer.id(), new TestTabletServerGateway(false));
+ tabletServerGatewayMap.put(
+ tabletServer.id(),
+ new TestTabletServerGateway(false, Collections.emptySet()));
}
}
}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java
index 3e7484f19..cf2084675 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java
@@ -40,6 +40,7 @@ import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -241,6 +242,7 @@ class FlussTableITCase extends ClientToServerITCaseBase {
Table table = conn.getTable(tablePath);
verifyPutAndLookup(table, new Object[] {1, "a"});
+ TableInfo tableInfo = table.getTableInfo();
// test put/lookup data for primary table with pk index is not 0
Schema schema =
@@ -261,6 +263,30 @@ class FlussTableITCase extends ClientToServerITCaseBase {
// now, check put/lookup data
Table table2 = conn.getTable(data1PkTablePath2);
verifyPutAndLookup(table2, new Object[] {"a", 1});
+
+ // Test schema change: add new column which equals to DATA2_ROW_TYPE
+ admin.alterTable(
+ tableInfo.getTablePath(),
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ waitAllSchemaSync(tablePath, 2);
+ Table newSchemaTable = conn.getTable(tableInfo.getTablePath());
+ // schema change case1: read new data with new schema.
+ verifyPutAndLookup(newSchemaTable, new Object[] {2, "b", "bb"});
+ // schema change case2: read new data with old schema.
+ assertThatRow(lookupRow(table.newLookup().createLookuper(), row(2)))
+ .withSchema(tableInfo.getSchema().getRowType())
+ .isEqualTo(row(2, "b"));
+ // schema change case3: read old data with new schema.
+ assertThatRow(lookupRow(newSchemaTable.newLookup().createLookuper(), row(1)))
+ .withSchema(newSchemaTable.getTableInfo().getSchema().getRowType())
+ .isEqualTo(row(1, "a", null));
}
@Test
@@ -278,6 +304,7 @@ class FlussTableITCase extends ClientToServerITCaseBase {
TableDescriptor.builder().schema(schema).distributedBy(3, "a", "b").build();
createTable(tablePath, descriptor, false);
Table table = conn.getTable(tablePath);
+ TableInfo tableInfo = table.getTableInfo();
verifyPutAndLookup(table, new Object[] {1, "a", 1L, "value1"});
verifyPutAndLookup(table, new Object[] {1, "a", 2L, "value2"});
verifyPutAndLookup(table, new Object[] {1, "a", 3L, "value3"});
@@ -315,6 +342,64 @@ class FlussTableITCase extends ClientToServerITCaseBase {
assertThat(prefixLookupResult).isNotNull();
rowList = prefixLookupResult.getRowList();
assertThat(rowList.size()).isEqualTo(0);
+
+ // Test schema change: add new column.
+ Schema newSchema =
+ Schema.newBuilder()
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.STRING())
+ .column("c", DataTypes.BIGINT())
+ .column("d", DataTypes.STRING())
+ .column("e", DataTypes.STRING())
+ .primaryKey("a", "b", "c")
+ .build();
+ admin.alterTable(
+ tableInfo.getTablePath(),
+ Collections.singletonList(
+ TableChange.addColumn(
+ "e",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ waitAllSchemaSync(tablePath, 2);
+ try (Connection connection = ConnectionFactory.createConnection(clientConf);
+ Table newSchemaTable = connection.getTable(tableInfo.getTablePath())) {
+ // schema change case1: read new data with new schema.
+ verifyPutAndLookup(
+ newSchemaTable, new Object[] {1, "a", 4L, "value4", "add_column_value"});
+ // schema change case2: read new data with old schema.
+ result = prefixLookuper.lookup(row(1, "a"));
+ prefixLookupResult = result.get();
+ assertThat(prefixLookupResult).isNotNull();
+ rowList = prefixLookupResult.getRowList();
+ assertThat(rowList.size()).isEqualTo(4);
+ for (int i = 0; i < rowList.size(); i++) {
+ assertRowValueEquals(
+ rowType, rowList.get(i), new Object[] {1, "a", i + 1L, "value" + (i + 1)});
+ }
+ admin.getTableSchema(tablePath, 1).get();
+ // schema change case3: read old data with new schema.
+ Lookuper newPrefixLookuper =
+ newSchemaTable
+ .newLookup()
+ .lookupBy(prefixKeyRowType.getFieldNames())
+ .createLookuper();
+ result = newPrefixLookuper.lookup(row(1, "a"));
+ prefixLookupResult = result.get();
+ assertThat(prefixLookupResult).isNotNull();
+ rowList = prefixLookupResult.getRowList();
+ assertThat(rowList.size()).isEqualTo(4);
+ for (int i = 0; i < rowList.size(); i++) {
+ assertRowValueEquals(
+ rowType,
+ rowList.get(i),
+ new Object[] {
+ 1, "a", i + 1L, "value" + (i + 1), i == 3 ? "add_column_value" : null
+ });
+ }
+ }
}
@Test
@@ -908,6 +993,126 @@ class FlussTableITCase extends ClientToServerITCaseBase {
}
}
+ @Test
+ void testPutAndProjectDuringAddColumn() throws Exception {
+ Schema schema =
+ Schema.newBuilder()
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.INT())
+ .column("c", DataTypes.STRING())
+ .primaryKey("a")
+ .build();
+ TableDescriptor tableDescriptor = TableDescriptor.builder().schema(schema).build();
+ TablePath tablePath = TablePath.of("test_db_1", "test_pk_table_1");
+ createTable(tablePath, tableDescriptor, false);
+
+ int batches = 3;
+ int keyId = 0;
+ int expectedSize = 0;
+
+ try (Table table = conn.getTable(tablePath)) {
+ // produce data with new schema.
+ // Test schema change: add new column which equals to DATA2_ROW_TYPE
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "d",
+ DataTypes.BIGINT(),
+ "add new column",
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ waitAllSchemaSync(tablePath, 2);
+ try (Connection connection = ConnectionFactory.createConnection(clientConf);
+ Table newSchemaTable = connection.getTable(tablePath)) {
+ UpsertWriter oldSchemaUpsertWriter = table.newUpsert().createWriter();
+ UpsertWriter newSchemaUpsertWriter = newSchemaTable.newUpsert().createWriter();
+ for (int b = 0; b < batches; b++) {
+ // insert 10 rows with old schema.
+ for (int i = keyId; i < keyId + 10; i++) {
+ InternalRow row = row(i, 100, "hello, friend" + i);
+ oldSchemaUpsertWriter.upsert(row);
+ expectedSize += 1;
+ oldSchemaUpsertWriter.flush();
+ }
+ // update 5 rows with new schema: [keyId, keyId+4]
+ for (int i = keyId; i < keyId + 5; i++) {
+ InternalRow row = row(i, 100, "HELLO, FRIEND" + i, i * 10L);
+ newSchemaUpsertWriter.upsert(row);
+ expectedSize += 2;
+ newSchemaUpsertWriter.flush();
+ }
+ // delete 1 row with old schema: [keyId+5]
+ int deleteKey = keyId + 5;
+ InternalRow row = row(deleteKey, 100, "hello, friend" + deleteKey);
+ oldSchemaUpsertWriter.delete(row);
+ expectedSize += 1;
+ // flush the mutation batch
+ oldSchemaUpsertWriter.flush();
+ keyId += 10;
+ }
+
+ // read with new schema
+ try (LogScanner logScanner = createLogScanner(newSchemaTable, new int[] {2, 0})) {
+ subscribeFromBeginning(logScanner, table);
+ int count = 0;
+ int id = 0;
+ while (count < expectedSize) {
+ ScanRecords scanRecords = logScanner.poll(Duration.ofSeconds(1));
+ Iterator<ScanRecord> iterator = scanRecords.iterator();
+ while (iterator.hasNext()) {
+ // 10 inserts
+ for (int i = 0; i < 10; i++) {
+ ScanRecord scanRecord = iterator.next();
+ assertThat(scanRecord.getChangeType()).isEqualTo(ChangeType.INSERT);
+ assertThat(scanRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(scanRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(scanRecord.getRow().getString(0).toString())
+ .isEqualTo("hello, friend" + id);
+ count++;
+ id++;
+ }
+ id -= 10;
+ // 10 updates
+ for (int i = 0; i < 5; i++) {
+ ScanRecord beforeRecord = iterator.next();
+ assertThat(beforeRecord.getChangeType())
+ .isEqualTo(ChangeType.UPDATE_BEFORE);
+ assertThat(beforeRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(beforeRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(beforeRecord.getRow().getString(0).toString())
+ .isEqualTo("hello, friend" + id);
+
+ ScanRecord afterRecord = iterator.next();
+ assertThat(afterRecord.getChangeType())
+ .isEqualTo(ChangeType.UPDATE_AFTER);
+ assertThat(afterRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(afterRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(afterRecord.getRow().getString(0).toString())
+ .isEqualTo("HELLO, FRIEND" + id);
+
+ id++;
+ count += 2;
+ }
+
+ // 1 delete
+ ScanRecord beforeRecord = iterator.next();
+ assertThat(beforeRecord.getChangeType()).isEqualTo(ChangeType.DELETE);
+ assertThat(beforeRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(beforeRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(beforeRecord.getRow().getString(0).toString())
+ .isEqualTo("hello, friend" + id);
+ count++;
+ id += 5;
+ }
+ }
+ assertThat(count).isEqualTo(expectedSize);
+ }
+ }
+ }
+ }
+
@Test
void testInvalidColumnProjection() throws Exception {
TableDescriptor tableDescriptor =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java
index 7797437e0..d204d087a 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java
@@ -170,7 +170,7 @@ class PartitionedTableITCase extends ClientToServerITCaseBase {
AppendWriter appendWriter = table.newAppend().createWriter();
int partitionSize = 5;
- // first try to add records add wait partition created.
+ // first try to add records and wait partition created.
Map<String, List<InternalRow>> expectPartitionNameAndAppendRows = new HashMap<>();
for (int i = 0; i < partitionSize; i++) {
String partitionName = String.valueOf(i);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
index b3f1eee8f..527994777 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
@@ -27,9 +27,11 @@ import org.apache.fluss.client.write.HashBucketAssigner;
import org.apache.fluss.metadata.DataLakeFormat;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.CompactedKeyEncoder;
import org.apache.fluss.row.encode.KeyEncoder;
import org.apache.fluss.types.DataTypes;
@@ -39,6 +41,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -116,13 +119,85 @@ class KvSnapshotBatchScannerITCase extends ClientToServerITCaseBase {
testSnapshotRead(tablePath, expectedRowByBuckets);
}
- private Map<TableBucket, List<InternalRow>> putRows(long tableId, TablePath tablePath, int rows)
- throws Exception {
+ @Test
+ void testScanSnapshotDuringSchemaChange() throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "test-table-snapshot-schema-change");
+ long tableId = createTable(tablePath, DEFAULT_TABLE_DESCRIPTOR, true);
+
+ // put into values with old schema.
+ Map<TableBucket, List<InternalRow>> oldSchemaRowByBuckets = putRows(tableId, tablePath, 10);
+ waitUntilAllSnapshotFinished(oldSchemaRowByBuckets.keySet(), 0);
+
+ // add a new column and rename an existing column
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "new_column",
+ DataTypes.BIGINT().copy(false).copy(true),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(tablePath, 2);
+
+ Schema newSchema =
+ Schema.newBuilder()
+ .primaryKey("id")
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("new_column", DataTypes.BIGINT())
+ .build();
+ // put into values with old schema.
+ List<InternalRow> rows = new ArrayList<>();
+ for (int i = 10; i < 20; i++) {
+ InternalRow row =
+ compactedRow(newSchema.getRowType(), new Object[] {i, "v" + i, (long) i});
+ rows.add(row);
+ }
+ Map<TableBucket, List<InternalRow>> newSchemaByBuckets = putRows(tableId, tablePath, rows);
+
+ Map<TableBucket, List<InternalRow>> expectedRowByBuckets = new HashMap<>();
+ for (TableBucket tableBucket : oldSchemaRowByBuckets.keySet()) {
+ List<InternalRow> expectedRows =
+ expectedRowByBuckets.computeIfAbsent(tableBucket, k -> new ArrayList<>());
+ oldSchemaRowByBuckets
+ .get(tableBucket)
+ .forEach(
+ row ->
+ expectedRows.add(
+ ProjectedRow.from(DEFAULT_SCHEMA, newSchema)
+ .replaceRow(row)));
+ }
+ for (TableBucket tableBucket : newSchemaByBuckets.keySet()) {
+ expectedRowByBuckets
+ .computeIfAbsent(tableBucket, k -> new ArrayList<>())
+ .addAll(newSchemaByBuckets.get(tableBucket));
+ }
+
+ // wait snapshot finish
+ waitUntilAllSnapshotFinished(expectedRowByBuckets.keySet(), 1);
+
+ // test read snapshot with new Schema
+ testSnapshotRead(tablePath, expectedRowByBuckets);
+ }
+
+ private Map<TableBucket, List<InternalRow>> putRows(
+ long tableId, TablePath tablePath, int rowNumber) throws Exception {
+ List<InternalRow> rows = new ArrayList<>();
+ for (int i = 0; i < rowNumber; i++) {
+ InternalRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {i, "v" + i});
+ rows.add(row);
+ }
+ return putRows(tableId, tablePath, rows);
+ }
+
+ private Map<TableBucket, List<InternalRow>> putRows(
+ long tableId, TablePath tablePath, List<InternalRow> rows) throws Exception {
Map<TableBucket, List<InternalRow>> rowsByBuckets = new HashMap<>();
try (Table table = conn.getTable(tablePath)) {
UpsertWriter upsertWriter = table.newUpsert().createWriter();
- for (int i = 0; i < rows; i++) {
- InternalRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {i, "v" + i});
+ for (InternalRow row : rows) {
upsertWriter.upsert(row);
TableBucket tableBucket = new TableBucket(tableId, getBucketId(row));
rowsByBuckets.computeIfAbsent(tableBucket, k -> new ArrayList<>()).add(row);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
index 3502dcc3c..bfd686dc1 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
@@ -20,6 +20,7 @@ package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
@@ -28,6 +29,7 @@ import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
import org.apache.fluss.types.DataTypes;
@@ -71,9 +73,13 @@ public class DefaultCompletedFetchTest {
private @TempDir File tempDir;
private TableInfo tableInfo;
private RowType rowType;
+ private TestingSchemaGetter testingSchemaGetter;
@BeforeEach
void beforeEach() {
+ testingSchemaGetter =
+ new TestingSchemaGetter(
+ new SchemaInfo(DATA2_TABLE_INFO.getSchema(), DEFAULT_SCHEMA_ID));
tableInfo = DATA2_TABLE_INFO;
rowType = DATA2_ROW_TYPE;
Map<TableBucket, Long> scanBuckets = new HashMap<>();
@@ -162,7 +168,7 @@ public class DefaultCompletedFetchTest {
long fetchOffset = 0L;
int bucketId = 0; // records for 0-10.
TableBucket tb = new TableBucket(DATA2_TABLE_ID, bucketId);
- Projection projection = Projection.of(new int[] {0, 2});
+ Projection projection = Projection.of(new int[] {0, 2}, schema);
MemoryLogRecords memoryLogRecords;
if (logFormat == LogFormat.ARROW) {
memoryLogRecords = genRecordsWithProjection(DATA2, projection, magic);
@@ -198,7 +204,7 @@ public class DefaultCompletedFetchTest {
// test projection reorder.
defaultCompletedFetch =
makeCompletedFetch(
- tb, resultForBucket0, fetchOffset, Projection.of(new int[] {2, 0}));
+ tb, resultForBucket0, fetchOffset, Projection.of(new int[] {2, 0}, schema));
scanRecords = defaultCompletedFetch.fetchRecords(8);
assertThat(scanRecords.size()).isEqualTo(8);
for (int i = 0; i < scanRecords.size(); i++) {
@@ -225,7 +231,11 @@ public class DefaultCompletedFetchTest {
return new DefaultCompletedFetch(
tableBucket,
resultForBucket,
- LogRecordReadContext.createReadContext(tableInfo, false, projection),
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ false,
+ projection,
+ new TestingSchemaGetter(tableInfo.getSchemaId(), tableInfo.getSchema())),
logScannerStatus,
true,
offset);
@@ -257,7 +267,10 @@ public class DefaultCompletedFetchTest {
FileLogProjection fileLogProjection = new FileLogProjection();
fileLogProjection.setCurrentProjection(
- DATA2_TABLE_ID, rowType, DEFAULT_COMPRESSION, projection.getProjectionInOrder());
+ DATA2_TABLE_ID,
+ testingSchemaGetter,
+ DEFAULT_COMPRESSION,
+ projection.getProjectionIdInOrder());
ByteBuffer buffer =
toByteBuffer(
fileLogProjection
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
index f7a1a07c4..095131b1e 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
@@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.assertj.core.api.Assertions.assertThat;
@@ -63,7 +64,8 @@ public class LogFetchBufferTest {
logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(scanBuckets);
readContext =
- LogRecordReadContext.createArrowReadContext(DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createArrowReadContext(
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
}
@AfterEach
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
index 9888fe03a..83878a816 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
@@ -40,6 +40,7 @@ import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.assertj.core.api.Assertions.assertThat;
@@ -66,7 +67,8 @@ public class LogFetchCollectorTest {
new LogFetchCollector(
DATA1_TABLE_PATH, logScannerStatus, new Configuration(), metadataUpdater);
readContext =
- LogRecordReadContext.createArrowReadContext(DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createArrowReadContext(
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
}
@AfterEach
@@ -91,7 +93,8 @@ public class LogFetchCollectorTest {
logFetchBuffer.add(completedFetch);
assertThat(logFetchBuffer.isEmpty()).isFalse();
- // Validate that the completed fetch isn't initialized just because we add it to the buffer.
+ // Validate that the completed fetch isn't initialized just because we add it to the
+ // buffer.
assertThat(completedFetch.isInitialized()).isFalse();
// Fetch the data and validate that we get all the records we want back.
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
index b131986e7..03f908733 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
@@ -18,18 +18,26 @@
package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.client.admin.ClientToServerITCaseBase;
+import org.apache.fluss.client.admin.FlussAdmin;
+import org.apache.fluss.client.metadata.ClientSchemaGetter;
import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.metrics.TestingScannerMetricGroup;
import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.cluster.Cluster;
import org.apache.fluss.cluster.ServerNode;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.row.GenericRow;
import org.apache.fluss.rpc.RpcClient;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
+import org.apache.fluss.testutils.DataTestUtils;
+import org.apache.fluss.types.DataTypes;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -43,11 +51,17 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE;
import static org.apache.fluss.record.TestData.DATA1;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
+import static org.apache.fluss.record.TestData.DATA2;
+import static org.apache.fluss.record.TestData.DATA2_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA2_SCHEMA;
import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newProduceLogRequest;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
@@ -60,6 +74,9 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
private final int bucketId0 = 0;
private final int bucketId1 = 1;
private LogScannerStatus logScannerStatus;
+ private FlussAdmin admin;
+ private MetadataUpdater metadataUpdater;
+ private ClientSchemaGetter clientSchemaGetter;
// TODO covert this test to UT as kafka.
@@ -72,7 +89,7 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId);
RpcClient rpcClient = FLUSS_CLUSTER_EXTENSION.getRpcClient();
- MetadataUpdater metadataUpdater = new MetadataUpdater(clientConf, rpcClient);
+ metadataUpdater = new MetadataUpdater(clientConf, rpcClient);
metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(DATA1_TABLE_PATH));
Map<TableBucket, Long> scanBuckets = new HashMap<>();
@@ -81,6 +98,9 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
scanBuckets.put(new TableBucket(tableId, bucketId1), 0L);
logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(scanBuckets);
+ admin = new FlussAdmin(rpcClient, metadataUpdater);
+ clientSchemaGetter =
+ new ClientSchemaGetter(DATA1_TABLE_PATH, new SchemaInfo(DATA1_SCHEMA, 1), admin);
logFetcher =
new LogFetcher(
DATA1_TABLE_INFO,
@@ -89,7 +109,96 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
clientConf,
metadataUpdater,
TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1));
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
+ }
+
+ @Test
+ void testFetchWithSchemaChange() throws Exception {
+ // add one batch records to tb0.
+ TableBucket tb0 = new TableBucket(tableId, bucketId0);
+ addRecordsToBucket(tb0, genMemoryLogRecordsByObject(DATA1), 0L);
+
+ // add new column(which equals to DATA2_ROW_TYPE)
+ admin.alterTable(
+ DATA1_TABLE_PATH,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ // add one batch records with new schema to tb0.
+ addRecordsToBucket(
+ tb0,
+ genMemoryLogRecordsByObject(DATA2_ROW_TYPE, 2, CURRENT_LOG_MAGIC_VALUE, DATA2),
+ 10L);
+
+ // Read data with old schema, thus DATA2 will be truncated as DATA1
+ List<GenericRow> expectedRows =
+ DATA1.stream().map(DataTestUtils::row).collect(Collectors.toList());
+ expectedRows.addAll(DATA1.stream().map(DataTestUtils::row).collect(Collectors.toList()));
+ logFetcher.sendFetches();
+ // The fetcher is async to fetch data, so we need to wait the result write to the
+ // logFetchBuffer.
+ retry(
+ Duration.ofMinutes(1),
+ () -> {
+ assertThat(logFetcher.hasAvailableFetches()).isTrue();
+ assertThat(logFetcher.getCompletedFetchesSize()).isEqualTo(2);
+ });
+ Map<TableBucket, List<ScanRecord>> records = logFetcher.collectFetch();
+ assertThat(records.size()).isEqualTo(1);
+ List<ScanRecord> scanRecords = records.get(tb0);
+ assertThat(scanRecords.stream().map(ScanRecord::getRow).collect(Collectors.toList()))
+ .isEqualTo(expectedRows);
+
+ // read data with new schema, thus DATA2 will be appended with null value.
+ expectedRows =
+ DATA1.stream()
+ .map(row -> DataTestUtils.row(row[0], row[1], null))
+ .collect(Collectors.toList());
+ expectedRows.addAll(DATA2.stream().map(DataTestUtils::row).collect(Collectors.toList()));
+ logScannerStatus.assignScanBuckets(Collections.singletonMap(tb0, 0L));
+ LogFetcher newSchemaLogFetcher =
+ new LogFetcher(
+ new TableInfo(
+ DATA1_TABLE_INFO.getTablePath(),
+ tableId,
+ 2,
+ DATA2_SCHEMA,
+ DATA1_TABLE_INFO.getBucketKeys(),
+ DATA1_TABLE_INFO.getPartitionKeys(),
+ DATA1_TABLE_INFO.getNumBuckets(),
+ DATA1_TABLE_INFO.getProperties(),
+ DATA1_TABLE_INFO.getCustomProperties(),
+ DATA1_TABLE_INFO.getComment().orElse(null),
+ DATA1_TABLE_INFO.getCreatedTime(),
+ DATA1_TABLE_INFO.getModifiedTime()),
+ null,
+ logScannerStatus,
+ clientConf,
+ metadataUpdater,
+ TestingScannerMetricGroup.newInstance(),
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
+ newSchemaLogFetcher.sendFetches();
+ // The fetcher is async to fetch data, so we need to wait the result write to the
+ // logFetchBuffer.
+ retry(
+ Duration.ofMinutes(1),
+ () -> {
+ assertThat(newSchemaLogFetcher.hasAvailableFetches()).isTrue();
+ assertThat(newSchemaLogFetcher.getCompletedFetchesSize()).isEqualTo(2);
+ });
+ records = newSchemaLogFetcher.collectFetch();
+ assertThat(records.size()).isEqualTo(1);
+ assertThat(records.get(tb0)).hasSize(20);
+ scanRecords = records.get(tb0);
+ assertThat(scanRecords.stream().map(ScanRecord::getRow).collect(Collectors.toList()))
+ .isEqualTo(expectedRows);
}
@Test
@@ -157,6 +266,12 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
LogScannerStatus logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(Collections.singletonMap(tb0, 0L));
+ ClientSchemaGetter clientSchemaGetter =
+ new ClientSchemaGetter(
+ DATA1_TABLE_PATH,
+ new SchemaInfo(DATA1_SCHEMA, 1),
+ new FlussAdmin(rpcClient, metadataUpdater));
+
LogFetcher logFetcher =
new LogFetcher(
DATA1_TABLE_INFO,
@@ -165,7 +280,8 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
clientConf,
metadataUpdater,
TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1));
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
// send fetches to fetch data, should have no available fetch.
logFetcher.sendFetches();
@@ -191,6 +307,11 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
void testFetchWithInvalidTableOrPartitions() throws Exception {
MetadataUpdater metadataUpdater1 =
new MetadataUpdater(clientConf, FLUSS_CLUSTER_EXTENSION.getRpcClient());
+ ClientSchemaGetter clientSchemaGetter =
+ new ClientSchemaGetter(
+ DATA1_TABLE_PATH,
+ new SchemaInfo(DATA1_SCHEMA, 1),
+ new FlussAdmin(FLUSS_CLUSTER_EXTENSION.getRpcClient(), metadataUpdater1));
logFetcher =
new LogFetcher(
DATA1_TABLE_INFO,
@@ -199,7 +320,8 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
clientConf,
metadataUpdater1,
TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1));
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<?> future =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
index f52a86945..edc47a7ec 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
@@ -22,12 +22,14 @@ import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.DataTypes;
@@ -55,6 +57,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.fluss.record.TestData.DATA2;
import static org.apache.fluss.record.TestData.DATA2_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA2_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA2_SCHEMA;
import static org.apache.fluss.record.TestData.DATA2_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA2_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA2_TABLE_PATH;
@@ -69,6 +72,7 @@ class RemoteCompletedFetchTest {
private LogRecordReadContext remoteReadContext;
private @TempDir File tempDir;
private TableInfo tableInfo;
+ private SchemaGetter schemaGetter;
@BeforeEach
void beforeEach() {
@@ -79,8 +83,10 @@ class RemoteCompletedFetchTest {
scanBuckets.put(new TableBucket(DATA2_TABLE_ID, 2), 0L);
logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(scanBuckets);
+ schemaGetter = new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA2_SCHEMA);
remoteReadContext =
- LogRecordReadContext.createArrowReadContext(DATA2_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createArrowReadContext(
+ DATA2_ROW_TYPE, DEFAULT_SCHEMA_ID, schemaGetter);
}
@AfterEach
@@ -220,7 +226,10 @@ class RemoteCompletedFetchTest {
createFileLogRecords(tableBucket, DATA2_PHYSICAL_TABLE_PATH, DATA2, logFormat);
RemoteCompletedFetch completedFetch =
makeCompletedFetch(
- tableBucket, fileLogRecords, fetchOffset, Projection.of(new int[] {0, 2}));
+ tableBucket,
+ fileLogRecords,
+ fetchOffset,
+ Projection.of(new int[] {0, 2}, schema));
List<ScanRecord> scanRecords = completedFetch.fetchRecords(8);
List<Object[]> expectedObjects =
@@ -246,7 +255,10 @@ class RemoteCompletedFetchTest {
completedFetch =
makeCompletedFetch(
- tableBucket, fileLogRecords, fetchOffset, Projection.of(new int[] {2, 0}));
+ tableBucket,
+ fileLogRecords,
+ fetchOffset,
+ Projection.of(new int[] {2, 0}, schema));
scanRecords = completedFetch.fetchRecords(8);
assertThat(scanRecords.size()).isEqualTo(8);
for (int i = 0; i < scanRecords.size(); i++) {
@@ -292,7 +304,7 @@ class RemoteCompletedFetchTest {
tableBucket,
fileLogRecords,
10L,
- LogRecordReadContext.createReadContext(tableInfo, true, projection),
+ LogRecordReadContext.createReadContext(tableInfo, true, projection, schemaGetter),
logScannerStatus,
true,
fetchOffset,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java
index 6b7eb6fe7..cb76d7688 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java
@@ -49,6 +49,7 @@ import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.row;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -95,7 +96,10 @@ public class ArrowLogWriteBatchTest {
LogRecordBatch batch = records.batches().iterator().next();
assertThat(batch.getRecordCount()).isEqualTo(count);
try (LogRecordReadContext readContext =
- createArrowReadContext(DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ createArrowReadContext(
+ DATA1_ROW_TYPE,
+ DATA1_TABLE_INFO.getSchemaId(),
+ TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> recordsIter = batch.records(readContext)) {
int readCount = 0;
while (recordsIter.hasNext()) {
@@ -153,7 +157,10 @@ public class ArrowLogWriteBatchTest {
LogRecordBatch batch = records.batches().iterator().next();
assertThat(batch.getRecordCount()).isEqualTo(count);
try (LogRecordReadContext readContext =
- createArrowReadContext(DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ createArrowReadContext(
+ DATA1_ROW_TYPE,
+ DATA1_TABLE_INFO.getSchemaId(),
+ TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> recordsIter = batch.records(readContext)) {
int readCount = 0;
while (recordsIter.hasNext()) {
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java
index 3081814a5..db39493bd 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java
@@ -19,6 +19,7 @@ package org.apache.fluss.client.write;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.memory.PreAllocatedPagedOutputView;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.IndexedLogRecord;
@@ -26,6 +27,7 @@ import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.record.bytesview.BytesView;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.utils.CloseableIterator;
@@ -43,6 +45,7 @@ import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE;
import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize;
import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.testutils.DataTestUtils.indexedRow;
@@ -98,7 +101,8 @@ public class IndexedLogWriteBatchTest {
BytesView bytesView = logProducerBatch.build();
MemoryLogRecords logRecords = MemoryLogRecords.pointToBytesView(bytesView);
Iterator<LogRecordBatch> iterator = logRecords.batches().iterator();
- assertDefaultLogRecordBatchEquals(iterator.next());
+ assertDefaultLogRecordBatchEquals(
+ iterator.next(), new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA, (short) 1)));
assertThat(iterator.hasNext()).isFalse();
}
@@ -219,13 +223,14 @@ public class IndexedLogWriteBatchTest {
System.currentTimeMillis());
}
- private void assertDefaultLogRecordBatchEquals(LogRecordBatch recordBatch) {
+ private void assertDefaultLogRecordBatchEquals(
+ LogRecordBatch recordBatch, TestingSchemaGetter schemaGetter) {
assertThat(recordBatch.getRecordCount()).isEqualTo(1);
assertThat(recordBatch.baseLogOffset()).isEqualTo(0L);
assertThat(recordBatch.schemaId()).isEqualTo((short) DATA1_TABLE_INFO.getSchemaId());
try (LogRecordReadContext readContext =
LogRecordReadContext.createIndexedReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId(), schemaGetter);
CloseableIterator<LogRecord> iterator = recordBatch.records(readContext)) {
assertThat(iterator.hasNext()).isTrue();
LogRecord record = iterator.next();
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java
index 5321da0f2..82f6ee590 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java
@@ -30,6 +30,7 @@ import org.apache.fluss.record.DefaultKvRecord;
import org.apache.fluss.record.DefaultKvRecordBatch;
import org.apache.fluss.record.KvRecord;
import org.apache.fluss.record.KvRecordReadContext;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.encode.CompactedKeyEncoder;
import org.apache.fluss.types.DataType;
@@ -44,6 +45,7 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK;
@@ -243,7 +245,8 @@ class KvWriteBatchTest {
recordBatch
.records(
KvRecordReadContext.createReadContext(
- KvFormat.COMPACTED, dataTypes))
+ KvFormat.COMPACTED,
+ new TestingSchemaGetter(1, DATA1_SCHEMA)))
.iterator();
assertThat(iterator.hasNext()).isTrue();
KvRecord kvRecord = iterator.next();
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
index 5aeee2d10..bc4e53b50 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
@@ -27,6 +27,8 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.config.MemorySize;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
@@ -38,6 +40,7 @@ import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.arrow.ArrowWriter;
@@ -127,12 +130,15 @@ class RecordAccumulatorTest {
private final ManualClock clock = new ManualClock(System.currentTimeMillis());
private Configuration conf;
private Cluster cluster;
+ private SchemaGetter schemaGetter;
@BeforeEach
public void start() {
conf = new Configuration();
// init cluster.
cluster = updateCluster(Arrays.asList(bucket1, bucket2, bucket3));
+ // todo: 从cluster中获取schema getter
+ schemaGetter = new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA, (short) 1));
}
// TODO Add more tests to test lingMs, retryBackoffMs, deliveryTimeoutMs and
@@ -292,7 +298,7 @@ class RecordAccumulatorTest {
Iterator<LogRecordBatch> iterator = memoryLogRecords.batches().iterator();
try (LogRecordReadContext readContext =
LogRecordReadContext.createIndexedReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId(), schemaGetter);
CloseableIterator<LogRecord> iter = iterator.next().records(readContext)) {
for (int i = 0; i < appends; i++) {
LogRecord record = iter.next();
@@ -329,7 +335,7 @@ class RecordAccumulatorTest {
LogRecordBatch logRecordBatch = iterator.next();
try (LogRecordReadContext readContext =
LogRecordReadContext.createIndexedReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId())) {
+ DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId(), schemaGetter)) {
LogRecord record = logRecordBatch.records(readContext).next();
assertThat(record.getChangeType()).isEqualTo(ChangeType.APPEND_ONLY);
assertThat(record.logOffset()).isEqualTo(0L);
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/config/cluster/ColumnPositionType.java
similarity index 51%
copy from fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
copy to fluss-common/src/main/java/org/apache/fluss/config/cluster/ColumnPositionType.java
index a0cc396b1..0f73c50b0 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/config/cluster/ColumnPositionType.java
@@ -15,28 +15,35 @@
* limitations under the License.
*/
-package org.apache.fluss.record;
+package org.apache.fluss.config.cluster;
-import org.apache.fluss.metadata.KvFormat;
-import org.apache.fluss.row.decode.RowDecoder;
-import org.apache.fluss.types.DataType;
+import org.apache.fluss.metadata.TableChange;
-/** A default implementation of {@link KvRecordBatch.ReadContext} . */
-public class KvRecordReadContext implements KvRecordBatch.ReadContext {
+/** ColumnPositionType. */
+public enum ColumnPositionType {
+ LAST(0);
- private final RowDecoder rowDecoder;
+ public final int value;
- private KvRecordReadContext(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
+ ColumnPositionType(int value) {
+ this.value = value;
}
- public static KvRecordReadContext createReadContext(KvFormat kvFormat, DataType[] fieldTypes) {
- return new KvRecordReadContext(RowDecoder.create(kvFormat, fieldTypes));
+ public static ColumnPositionType from(int opType) {
+ if (opType == 0) {
+ return LAST;
+ }
+ throw new IllegalArgumentException("Unsupported ColumnPositionType: " + opType);
}
- @Override
- public RowDecoder getRowDecoder(int schemaId) {
- // assume schema id are always not changed for now.
- return rowDecoder;
+ public static ColumnPositionType from(TableChange.ColumnPosition opType) {
+ if (opType == TableChange.ColumnPosition.last()) {
+ return LAST;
+ }
+ throw new IllegalArgumentException("Unsupported ColumnPositionType: " + opType);
+ }
+
+ public int value() {
+ return this.value;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
similarity index 57%
copy from fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
copy to fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
index b2cea1382..8c9446682 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
@@ -15,26 +15,16 @@
* limitations under the License.
*/
-package org.apache.fluss.flink.sink.serializer;
+package org.apache.fluss.exception;
-import org.apache.fluss.types.RowType;
+/** Exception thrown when schema change failed. */
+public class SchemaChangeException extends ApiException {
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
-
-/**
- * Contextual information that can be used during initialization of {@link
- * FlussSerializationSchema}.
- */
-public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
-
- private final RowType rowSchema;
-
- public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
+ public SchemaChangeException(String message, Throwable cause) {
+ super(message, cause);
}
- @Override
- public RowType getRowSchema() {
- return rowSchema;
+ public SchemaChangeException(String message) {
+ this(message, null);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
index 2de841936..367013363 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -55,14 +56,19 @@ import static org.apache.fluss.utils.Preconditions.checkState;
public final class Schema implements Serializable {
private static final long serialVersionUID = 1L;
-
private static final Schema EMPTY = Schema.newBuilder().build();
private final List<Column> columns;
private final @Nullable PrimaryKey primaryKey;
private final RowType rowType;
- private Schema(List<Column> columns, @Nullable PrimaryKey primaryKey) {
+ /**
+ * The highest field ID in this schema, this can only increase during the life cycle of the
+ * schema. Otherwise, the removed columns will influence it.
+ */
+ private final int highestFieldId;
+
+ private Schema(List<Column> columns, @Nullable PrimaryKey primaryKey, int highestFieldId) {
this.columns = normalizeColumns(columns, primaryKey);
this.primaryKey = primaryKey;
// pre-create the row type as it is the most frequently used part of the schema
@@ -74,6 +80,7 @@ public final class Schema implements Serializable {
new DataField(
column.getName(), column.getDataType()))
.collect(Collectors.toList()));
+ this.highestFieldId = highestFieldId;
}
public List<Column> getColumns() {
@@ -125,6 +132,10 @@ public final class Schema implements Serializable {
return columns.stream().map(Column::getName).collect(Collectors.toList());
}
+ public List<Integer> getColumnIds() {
+ return columns.stream().map(Column::getColumnId).collect(Collectors.toList());
+ }
+
/** Returns the column names in given column indexes. */
public List<String> getColumnNames(int[] columnIndexes) {
List<String> columnNames = new ArrayList<>();
@@ -143,6 +154,10 @@ public final class Schema implements Serializable {
return keyIndexes;
}
+ public int getHighestFieldId() {
+ return highestFieldId;
+ }
+
@Override
public String toString() {
final List<Object> components = new ArrayList<>(columns);
@@ -188,9 +203,11 @@ public final class Schema implements Serializable {
public static final class Builder {
private final List<Column> columns;
private @Nullable PrimaryKey primaryKey;
+ private AtomicInteger highestFieldId;
private Builder() {
columns = new ArrayList<>();
+ highestFieldId = new AtomicInteger(-1);
}
/** Adopts all members from the given schema. */
@@ -199,10 +216,15 @@ public final class Schema implements Serializable {
if (schema.primaryKey != null) {
primaryKeyNamed(schema.primaryKey.constraintName, schema.primaryKey.columnNames);
}
+ this.highestFieldId = new AtomicInteger(schema.highestFieldId);
+ return this;
+ }
+
+ public Builder highestFieldId(int highestFieldId) {
+ this.highestFieldId = new AtomicInteger(highestFieldId);
return this;
}
- /** Adopts all fields of the given row as columns of the schema. */
public Builder fromRowType(RowType rowType) {
checkNotNull(rowType, "rowType must not be null.");
final List<DataType> fieldDataTypes = rowType.getChildren();
@@ -227,7 +249,33 @@ public final class Schema implements Serializable {
/** Adopts all columns from the given list. */
public Builder fromColumns(List<Column> inputColumns) {
- columns.addAll(inputColumns);
+ boolean nonMatchColumnId =
+ inputColumns.stream()
+ .noneMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
+ boolean allMatchColumnId =
+ inputColumns.stream()
+ .allMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
+ checkState(
+ nonMatchColumnId || allMatchColumnId,
+ "All columns must have columnId or none of them must have columnId.");
+
+ if (allMatchColumnId) {
+ columns.addAll(inputColumns);
+ highestFieldId =
+ new AtomicInteger(
+ inputColumns.stream()
+ .mapToInt(Column::getColumnId)
+ .max()
+ .orElse(-1));
+ } else {
+ // if all columnId is not set, this maybe from old version schema. Just use its
+ // position as columnId.
+ inputColumns.forEach(
+ column ->
+ this.column(column.columnName, column.dataType)
+ .withComment(column.comment));
+ }
+
return this;
}
@@ -239,12 +287,11 @@ public final class Schema implements Serializable {
* from and written to an external system.
*
* @param columnName column name
- * @param dataType data type of the column
*/
public Builder column(String columnName, DataType dataType) {
checkNotNull(columnName, "Column name must not be null.");
checkNotNull(dataType, "Data type must not be null.");
- columns.add(new Column(columnName, dataType));
+ columns.add(new Column(columnName, dataType, null, highestFieldId.incrementAndGet()));
return this;
}
@@ -316,7 +363,17 @@ public final class Schema implements Serializable {
/** Returns an instance of an {@link Schema}. */
public Schema build() {
- return new Schema(columns, primaryKey);
+ Integer maximumColumnId =
+ columns.stream().map(Column::getColumnId).max(Integer::compareTo).orElse(0);
+
+ checkState(
+ columns.isEmpty() || highestFieldId.get() >= maximumColumnId,
+ "Highest field id must be greater than or equal to the maximum column id.");
+
+ checkState(
+ columns.stream().map(Column::getColumnId).distinct().count() == columns.size(),
+ "Column ids must be unique.");
+ return new Schema(columns, primaryKey, highestFieldId.get());
}
}
@@ -331,19 +388,27 @@ public final class Schema implements Serializable {
*/
@PublicStable
public static final class Column implements Serializable {
+ public static final int UNKNOWN_COLUMN_ID = -1;
private static final long serialVersionUID = 1L;
+ private final int columnId;
private final String columnName;
private final DataType dataType;
private final @Nullable String comment;
public Column(String columnName, DataType dataType) {
- this(columnName, dataType, null);
+ this(columnName, dataType, null, UNKNOWN_COLUMN_ID);
}
public Column(String columnName, DataType dataType, @Nullable String comment) {
+ this(columnName, dataType, comment, UNKNOWN_COLUMN_ID);
+ }
+
+ public Column(
+ String columnName, DataType dataType, @Nullable String comment, int columnId) {
this.columnName = columnName;
this.dataType = dataType;
this.comment = comment;
+ this.columnId = columnId;
}
public String getName() {
@@ -354,12 +419,16 @@ public final class Schema implements Serializable {
return Optional.ofNullable(comment);
}
+ public int getColumnId() {
+ return columnId;
+ }
+
public DataType getDataType() {
return dataType;
}
public Column withComment(String comment) {
- return new Column(columnName, dataType, comment);
+ return new Column(columnName, dataType, comment, columnId);
}
@Override
@@ -387,12 +456,13 @@ public final class Schema implements Serializable {
Column that = (Column) o;
return Objects.equals(columnName, that.columnName)
&& Objects.equals(dataType, that.dataType)
- && Objects.equals(comment, that.comment);
+ && Objects.equals(comment, that.comment)
+ && Objects.equals(columnId, that.columnId);
}
@Override
public int hashCode() {
- return Objects.hash(columnName, dataType, comment);
+ return Objects.hash(columnName, dataType, comment, columnId);
}
}
@@ -492,9 +562,8 @@ public final class Schema implements Serializable {
new Column(
column.getName(),
column.getDataType().copy(false),
- column.getComment().isPresent()
- ? column.getComment().get()
- : null));
+ column.getComment().isPresent() ? column.getComment().get() : null,
+ column.getColumnId()));
} else {
newColumns.add(column);
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
similarity index 56%
copy from fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
copy to fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
index b2cea1382..8cdc14670 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
@@ -15,26 +15,13 @@
* limitations under the License.
*/
-package org.apache.fluss.flink.sink.serializer;
+package org.apache.fluss.metadata;
-import org.apache.fluss.types.RowType;
+/** A schema getter interface. */
+public interface SchemaGetter {
+ Schema getSchema(int schemaId);
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
+ SchemaInfo getLatestSchemaInfo();
-/**
- * Contextual information that can be used during initialization of {@link
- * FlussSerializationSchema}.
- */
-public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
-
- private final RowType rowSchema;
-
- public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
- }
-
- @Override
- public RowType getRowSchema() {
- return rowSchema;
- }
+ void release();
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
index 98e907e5f..36162f30c 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
@@ -17,11 +17,98 @@
package org.apache.fluss.metadata;
+import org.apache.fluss.types.DataType;
+
+import javax.annotation.Nullable;
+
import java.util.Objects;
/** {@link TableChange} represents the modification of the Fluss Table. */
public interface TableChange {
+ /**
+ * A table change toadd the column with specified position.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> ADD <column_definition> <column_position>
+ * </pre>
+ *
+ * @return a TableChange represents the modification.
+ */
+ static AddColumn addColumn(
+ String columnName,
+ DataType dataType,
+ @Nullable String comment,
+ ColumnPosition position) {
+ return new AddColumn(columnName, dataType, comment, position);
+ }
+
+ /**
+ * A table change to modify a column. The modification includes:
+ *
+ * <ul>
+ * <li>change column data type
+ * <li>reorder column position
+ * <li>modify column comment
+ * <li>rename column name
+ * <li>change the computed expression
+ * <li>change the metadata column expression
+ * </ul>
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> MODIFY <column_definition> COMMENT '<column_comment>' <column_position>
+ * </pre>
+ *
+ * @param oldColumn the definition of the old column.
+ * @param newColumn the definition of the new column.
+ * @param columnPosition the new position of the column.
+ * @return a TableChange represents the modification.
+ */
+ static ModifyColumn modifyColumn(
+ String columnName,
+ DataType dataType,
+ @Nullable String comment,
+ @Nullable ColumnPosition columnPosition) {
+ return new ModifyColumn(columnName, dataType, comment, columnPosition);
+ }
+
+ /**
+ * A table change to modify the column name.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> RENAME <old_column_name> TO <new_column_name>
+ * </pre>
+ *
+ * @param oldColumn the definition of the old column.
+ * @param newName the name of the new column.
+ * @return a TableChange represents the modification.
+ */
+ static RenameColumn renameColumn(String oldColumnName, String newColumnName) {
+ return new RenameColumn(oldColumnName, newColumnName);
+ }
+
+ /**
+ * A table change to drop column.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> DROP COLUMN <column_name>
+ * </pre>
+ *
+ * @param columnName the column to drop.
+ * @return a TableChange represents the modification.
+ */
+ static DropColumn dropColumn(String columnName) {
+ return new DropColumn(columnName);
+ }
+
static SetOption set(String key, String value) {
return new SetOption(key, value);
}
@@ -126,4 +213,196 @@ public interface TableChange {
return "ResetOption{" + "key='" + key + '\'' + '}';
}
}
+
+ /** A table change to modify the table schema. */
+ interface SchemaChange extends TableChange {}
+
+ /**
+ * A table change to add a column.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> ADD <column_definition> <column_position>
+ * </pre>
+ */
+ class AddColumn implements SchemaChange {
+ private final String name;
+ private final DataType dataType;
+ private final @Nullable String comment;
+
+ private final ColumnPosition position;
+
+ private AddColumn(
+ String name, DataType dataType, @Nullable String comment, ColumnPosition position) {
+ this.name = name;
+ this.dataType = dataType;
+ this.comment = comment;
+ this.position = position;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ @Nullable
+ public String getComment() {
+ return comment;
+ }
+
+ public ColumnPosition getPosition() {
+ return position;
+ }
+ }
+
+ /** A table change to drop a column. */
+ class DropColumn implements SchemaChange {
+ private final String name;
+
+ private DropColumn(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ /** A table change to modify a column. */
+ class ModifyColumn implements SchemaChange {
+ private final String name;
+ private final DataType dataType;
+ private final @Nullable String comment;
+
+ private final @Nullable ColumnPosition newPosition;
+
+ private ModifyColumn(
+ String name,
+ DataType dataType,
+ @Nullable String comment,
+ @Nullable ColumnPosition newPosition) {
+ this.name = name;
+ this.dataType = dataType;
+ this.comment = comment;
+ this.newPosition = newPosition;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ @Nullable
+ public String getComment() {
+ return comment;
+ }
+
+ @Nullable
+ public ColumnPosition getNewPosition() {
+ return newPosition;
+ }
+ }
+
+ /** A table change to modify a column's name. */
+ class RenameColumn implements SchemaChange {
+ private final String oldColumnName;
+ private final String newColumnName;
+
+ private RenameColumn(String oldColumnName, String newColumnName) {
+ this.oldColumnName = oldColumnName;
+ this.newColumnName = newColumnName;
+ }
+
+ public String getOldColumnName() {
+ return oldColumnName;
+ }
+
+ public String getNewColumnName() {
+ return newColumnName;
+ }
+ }
+
+ /** The position of the modified or added column. */
+ interface ColumnPosition {
+ /** Get the position to place the column at the first. */
+ static ColumnPosition last() {
+ return Last.INSTANCE;
+ }
+
+ /** Get the position to place the column at the first. */
+ static ColumnPosition first() {
+ return First.INSTANCE;
+ }
+
+ /** Get the position to place the column after the specified column. */
+ static ColumnPosition after(String column) {
+ return new After(column);
+ }
+ }
+
+ /** Column position FIRST means the specified column should be the first column. */
+ final class First implements ColumnPosition {
+ private static final First INSTANCE = new First();
+
+ private First() {}
+
+ @Override
+ public String toString() {
+ return "FIRST";
+ }
+ }
+
+ /** Column position Last means the specified column should be the last column. */
+ final class Last implements ColumnPosition {
+ private static final Last INSTANCE = new Last();
+
+ private Last() {}
+
+ @Override
+ public String toString() {
+ return "LAST";
+ }
+ }
+
+ /** Column position AFTER means the specified column should be put after the given `column`. */
+ final class After implements ColumnPosition {
+ private final String columnName;
+
+ private After(String columnName) {
+ this.columnName = columnName;
+ }
+
+ public String columnName() {
+ return columnName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof After)) {
+ return false;
+ }
+ After after = (After) o;
+ return Objects.equals(columnName, after.columnName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(columnName);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("AFTER %s", columnName);
+ }
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
index 329d113c2..af293f120 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
@@ -335,6 +335,23 @@ public final class TableInfo {
modifiedTime);
}
+ /** Replace a TableInfo with a new SchemaInfo. */
+ public TableInfo withNewSchema(SchemaInfo schemaInfo) {
+ return new TableInfo(
+ tablePath,
+ tableId,
+ schemaInfo.getSchemaId(),
+ schemaInfo.getSchema(),
+ bucketKeys,
+ partitionKeys,
+ numBuckets,
+ properties,
+ customProperties,
+ comment,
+ createdTime,
+ modifiedTime);
+ }
+
@Override
public boolean equals(Object o) {
if (o == null || getClass() != o.getClass()) {
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
index eaacc48a6..b6ed40eeb 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
@@ -21,7 +21,10 @@ import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.exception.CorruptMessageException;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.arrow.ArrowReader;
+import org.apache.fluss.row.columnar.ColumnarRow;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.fluss.types.DataType;
@@ -33,9 +36,11 @@ import org.apache.fluss.utils.crc.Crc32C;
import java.nio.ByteBuffer;
import java.util.NoSuchElementException;
+import java.util.function.Function;
import static org.apache.fluss.record.LogRecordBatchFormat.BASE_OFFSET_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.COMMIT_TIMESTAMP_OFFSET;
+import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_LENGTH;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_OVERHEAD;
@@ -208,6 +213,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
@Override
+ @SuppressWarnings("unchecked")
public CloseableIterator<LogRecord> records(ReadContext context) {
if (getRecordCount() == 0) {
return CloseableIterator.emptyIterator();
@@ -217,15 +223,26 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
long timestamp = commitTimestamp();
LogFormat logFormat = context.getLogFormat();
RowType rowType = context.getRowType(schemaId);
+ Function<InternalRow, InternalRow> outputRowProcessor =
+ (row) -> {
+ ProjectedRow projectedRow = context.getOutputProjectedRow(schemaId);
+ if (projectedRow != null) {
+ return projectedRow.replaceRow(row);
+ } else {
+ return row;
+ }
+ };
+
switch (logFormat) {
case ARROW:
return columnRecordIterator(
rowType,
+ outputRowProcessor,
context.getVectorSchemaRoot(schemaId),
context.getBufferAllocator(),
timestamp);
case INDEXED:
- return rowRecordIterator(rowType, timestamp);
+ return rowRecordIterator(rowType, outputRowProcessor, timestamp);
default:
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
@@ -252,7 +269,10 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
return MurmurHashUtils.hashBytes(segment, position, sizeInBytes());
}
- private CloseableIterator<LogRecord> rowRecordIterator(RowType rowType, long timestamp) {
+ private CloseableIterator<LogRecord> rowRecordIterator(
+ RowType rowType,
+ Function<InternalRow, InternalRow> outputRowProcessor,
+ long timestamp) {
DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
return new LogRecordIterator() {
int position = DefaultLogRecordBatch.this.position + recordBatchHeaderSize(magic);
@@ -261,8 +281,13 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
@Override
protected LogRecord readNext(long baseOffset) {
IndexedLogRecord logRecord =
- IndexedLogRecord.readFrom(
- segment, position, baseOffset + rowId, timestamp, fieldTypes);
+ ProjectedIndexedLogRecord.readFrom(
+ segment,
+ position,
+ baseOffset + rowId,
+ timestamp,
+ fieldTypes,
+ outputRowProcessor);
rowId++;
position += logRecord.getSizeInBytes();
return logRecord;
@@ -279,7 +304,11 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
private CloseableIterator<LogRecord> columnRecordIterator(
- RowType rowType, VectorSchemaRoot root, BufferAllocator allocator, long timestamp) {
+ RowType rowType,
+ Function<InternalRow, InternalRow> outputRowProcessor,
+ VectorSchemaRoot root,
+ BufferAllocator allocator,
+ long timestamp) {
boolean isAppendOnly = (attributes() & APPEND_ONLY_FLAG_MASK) > 0;
if (isAppendOnly) {
// append only batch, no change type vector,
@@ -290,7 +319,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
@Override
protected ChangeType getChangeType(int rowId) {
return ChangeType.APPEND_ONLY;
@@ -308,7 +337,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
@Override
protected ChangeType getChangeType(int rowId) {
return changeTypeVector.getChangeType(rowId);
@@ -322,10 +351,15 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
private final ArrowReader reader;
private final long timestamp;
private int rowId = 0;
+ private final Function<InternalRow, InternalRow> outputRowProcessor;
- private ArrowLogRecordIterator(ArrowReader reader, long timestamp) {
+ private ArrowLogRecordIterator(
+ ArrowReader reader,
+ long timestamp,
+ Function<InternalRow, InternalRow> outputRowProcessor) {
this.reader = reader;
this.timestamp = timestamp;
+ this.outputRowProcessor = outputRowProcessor;
}
protected abstract ChangeType getChangeType(int rowId);
@@ -337,12 +371,13 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
@Override
protected LogRecord readNext(long baseOffset) {
+ ColumnarRow originalRow = reader.read(rowId);
LogRecord record =
new GenericRecord(
baseOffset + rowId,
timestamp,
getChangeType(rowId),
- reader.read(rowId));
+ outputRowProcessor.apply(originalRow));
rowId++;
return record;
}
@@ -412,4 +447,38 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
throw new UnsupportedOperationException();
}
}
+
+ private static class ProjectedIndexedLogRecord extends IndexedLogRecord {
+ private final Function<InternalRow, InternalRow> outputRowProcessor;
+
+ ProjectedIndexedLogRecord(
+ long logOffset,
+ long timestamp,
+ DataType[] fieldTypes,
+ Function<InternalRow, InternalRow> outputRowProcessor) {
+ super(logOffset, timestamp, fieldTypes);
+ this.outputRowProcessor = outputRowProcessor;
+ }
+
+ public static IndexedLogRecord readFrom(
+ MemorySegment segment,
+ int position,
+ long logOffset,
+ long logTimestamp,
+ DataType[] colTypes,
+ Function<InternalRow, InternalRow> outputRowProcessor) {
+ int sizeInBytes = segment.getInt(position);
+ ProjectedIndexedLogRecord logRecord =
+ new ProjectedIndexedLogRecord(
+ logOffset, logTimestamp, colTypes, outputRowProcessor);
+ logRecord.pointTo(segment, position, sizeInBytes + LENGTH_LENGTH);
+ return logRecord;
+ }
+
+ @Override
+ public InternalRow getRow() {
+ InternalRow row = super.getRow();
+ return outputRowProcessor.apply(row);
+ }
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java
index b8aefc75b..8d41ec4c8 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java
@@ -163,6 +163,7 @@ public class DefaultValueRecordBatch implements ValueRecordBatch {
}
readRecords++;
ValueRecord rec = readNext();
+
if (readRecords == numRecords) {
// Validate that the actual size of the batch is equal to declared size
// by checking that after reading declared number of items, there no items left
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
index 1f889fff1..11c50bb3d 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
@@ -20,6 +20,7 @@ package org.apache.fluss.record;
import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.compression.ArrowCompressionInfo;
import org.apache.fluss.exception.InvalidColumnProjectionException;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.bytesview.MultiBytesView;
import org.apache.fluss.shaded.arrow.com.google.flatbuffers.FlatBufferBuilder;
import org.apache.fluss.shaded.arrow.org.apache.arrow.flatbuf.Buffer;
@@ -53,6 +54,7 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import static org.apache.fluss.record.DefaultLogRecordBatch.APPEND_ONLY_FLAG_MASK;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
@@ -66,6 +68,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.arrowChangeTypeOffset
import static org.apache.fluss.record.LogRecordBatchFormat.attributeOffset;
import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize;
import static org.apache.fluss.record.LogRecordBatchFormat.recordsCountOffset;
+import static org.apache.fluss.record.LogRecordBatchFormat.schemaIdOffset;
import static org.apache.fluss.utils.FileUtils.readFully;
import static org.apache.fluss.utils.FileUtils.readFullyOrFail;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
@@ -82,7 +85,7 @@ public class FileLogProjection {
private static final int ARROW_HEADER_SIZE =
ARROW_IPC_CONTINUATION_LENGTH + ARROW_IPC_METADATA_SIZE_LENGTH;
- final Map<Long, ProjectionInfo> projectionsCache = new HashMap<>();
+ final Map<ProjectionKey, ProjectionInfo> projectionsCache = new HashMap<>();
ProjectionInfo currentProjection;
// shared resources for multiple projections
@@ -97,6 +100,10 @@ public class FileLogProjection {
private final ByteBuffer arrowHeaderBuffer = ByteBuffer.allocate(ARROW_HEADER_SIZE);
private ByteBuffer arrowMetadataBuffer;
+ private SchemaGetter schemaGetter;
+ private long tableId;
+ private ArrowCompressionInfo compressionInfo;
+ private int[] selectedFieldPositions;
public FileLogProjection() {
this.outputStream = new ByteArrayOutputStream();
@@ -109,60 +116,20 @@ public class FileLogProjection {
public void setCurrentProjection(
long tableId,
- RowType schema,
+ SchemaGetter schemaGetter,
ArrowCompressionInfo compressionInfo,
- int[] selectedFields) {
- if (projectionsCache.containsKey(tableId)) {
- // the schema and projection should identical for the same table id.
- currentProjection = projectionsCache.get(tableId);
- if (!Arrays.equals(currentProjection.selectedFields, selectedFields)
- || !currentProjection.schema.equals(schema)) {
- throw new InvalidColumnProjectionException(
- "The schema and projection should be identical for the same table id.");
- }
- return;
- }
-
- // initialize the projection util information
- Schema arrowSchema = ArrowUtils.toArrowSchema(schema);
- BitSet selection = toBitSet(arrowSchema.getFields().size(), selectedFields);
- List<Tuple2<Field, Boolean>> flattenedFields = new ArrayList<>();
- flattenFields(arrowSchema.getFields(), selection, flattenedFields);
- int totalFieldNodes = flattenedFields.size();
- int[] bufferLayoutCount = new int[totalFieldNodes];
- BitSet nodesProjection = new BitSet(totalFieldNodes);
- int totalBuffers = 0;
- for (int i = 0; i < totalFieldNodes; i++) {
- Field fieldNode = flattenedFields.get(i).f0;
- boolean selected = flattenedFields.get(i).f1;
- nodesProjection.set(i, selected);
- bufferLayoutCount[i] = TypeLayout.getTypeBufferCount(fieldNode.getType());
- totalBuffers += bufferLayoutCount[i];
- }
- BitSet buffersProjection = new BitSet(totalBuffers);
- int bufferIndex = 0;
- for (int i = 0; i < totalFieldNodes; i++) {
- if (nodesProjection.get(i)) {
- buffersProjection.set(bufferIndex, bufferIndex + bufferLayoutCount[i]);
- }
- bufferIndex += bufferLayoutCount[i];
- }
-
- Schema projectedArrowSchema = ArrowUtils.toArrowSchema(schema.project(selectedFields));
- ArrowBodyCompression bodyCompression =
- CompressionUtil.createBodyCompression(compressionInfo.createCompressionCodec());
- int metadataLength =
- ArrowUtils.estimateArrowMetadataLength(projectedArrowSchema, bodyCompression);
- currentProjection =
- new ProjectionInfo(
- nodesProjection,
- buffersProjection,
- bufferIndex,
- schema,
- metadataLength,
- bodyCompression,
- selectedFields);
- projectionsCache.put(tableId, currentProjection);
+ int[] selectedFieldIds) {
+ this.tableId = tableId;
+ this.schemaGetter = schemaGetter;
+ this.compressionInfo = compressionInfo;
+
+ // Currently, only add last column is supported.Thus selectedFieldPositions is always same
+ // from same selectedFieldIds.
+ // TODO: if support drop column or add column in middle, this selectedFieldPositions should
+ // be re-calculated for each schema.
+ this.selectedFieldPositions =
+ selectedFieldPositions(
+ schemaGetter.getLatestSchemaInfo().getSchema(), selectedFieldIds);
}
/**
@@ -173,7 +140,7 @@ public class FileLogProjection {
*/
public BytesViewLogRecords project(FileChannel channel, int start, int end, int maxBytes)
throws IOException {
- checkNotNull(currentProjection, "There is no projection registered yet.");
+
MultiBytesView.Builder builder = MultiBytesView.builder();
int position = start;
@@ -194,6 +161,10 @@ public class FileLogProjection {
byte magic = logHeaderBuffer.get(MAGIC_OFFSET);
int recordBatchHeaderSize = recordBatchHeaderSize(magic);
int batchSizeInBytes = LOG_OVERHEAD + logHeaderBuffer.getInt(LENGTH_OFFSET);
+ short schemaId = logHeaderBuffer.getShort(schemaIdOffset(magic));
+ setCurrentSchema(schemaId);
+ checkNotNull(currentProjection, "There is no projection registered yet.");
+
if (position > end - batchSizeInBytes) {
// the remaining bytes in the file are not enough to read a full batch
return new BytesViewLogRecords(builder.build());
@@ -434,6 +405,118 @@ public class FileLogProjection {
return logHeaderBuffer;
}
+ private void setCurrentSchema(short schemaId) {
+ org.apache.fluss.metadata.Schema schema = schemaGetter.getSchema(schemaId);
+ RowType rowType = schema.getRowType();
+ ProjectionKey projectionKey = new ProjectionKey(tableId, schemaId);
+ if (projectionsCache.containsKey(projectionKey)) {
+ // the schema and projection should identical for the same table id.
+ currentProjection = projectionsCache.get(projectionKey);
+ if (!Arrays.equals(currentProjection.selectedFieldPositions, selectedFieldPositions)
+ || !currentProjection.schema.equals(rowType)) {
+ throw new InvalidColumnProjectionException(
+ "The schema and projection should be identical for the same table id.");
+ }
+ return;
+ }
+
+ // initialize the projection util information
+ Schema arrowSchema = ArrowUtils.toArrowSchema(rowType);
+ BitSet selection = toBitSet(arrowSchema.getFields().size(), selectedFieldPositions);
+ List<Tuple2<Field, Boolean>> flattenedFields = new ArrayList<>();
+ flattenFields(arrowSchema.getFields(), selection, flattenedFields);
+ int totalFieldNodes = flattenedFields.size();
+ int[] bufferLayoutCount = new int[totalFieldNodes];
+ BitSet nodesProjection = new BitSet(totalFieldNodes);
+ int totalBuffers = 0;
+ for (int i = 0; i < totalFieldNodes; i++) {
+ Field fieldNode = flattenedFields.get(i).f0;
+ boolean selected = flattenedFields.get(i).f1;
+ nodesProjection.set(i, selected);
+ bufferLayoutCount[i] = TypeLayout.getTypeBufferCount(fieldNode.getType());
+ totalBuffers += bufferLayoutCount[i];
+ }
+ BitSet buffersProjection = new BitSet(totalBuffers);
+ int bufferIndex = 0;
+ for (int i = 0; i < totalFieldNodes; i++) {
+ if (nodesProjection.get(i)) {
+ buffersProjection.set(bufferIndex, bufferIndex + bufferLayoutCount[i]);
+ }
+ bufferIndex += bufferLayoutCount[i];
+ }
+
+ Schema projectedArrowSchema =
+ ArrowUtils.toArrowSchema(rowType.project(selectedFieldPositions));
+ ArrowBodyCompression bodyCompression =
+ CompressionUtil.createBodyCompression(compressionInfo.createCompressionCodec());
+ int metadataLength =
+ ArrowUtils.estimateArrowMetadataLength(projectedArrowSchema, bodyCompression);
+ currentProjection =
+ new ProjectionInfo(
+ nodesProjection,
+ buffersProjection,
+ bufferIndex,
+ rowType,
+ metadataLength,
+ bodyCompression,
+ selectedFieldPositions);
+ projectionsCache.put(projectionKey, currentProjection);
+ }
+
+ int[] selectedFieldPositions(org.apache.fluss.metadata.Schema schema, int[] projectedFields) {
+ Map<Integer, Integer> columnIdPositions = new HashMap<>();
+ List<Integer> columnIds = schema.getColumnIds();
+ for (int i = 0; i < columnIds.size(); i++) {
+ columnIdPositions.put(columnIds.get(i), i);
+ }
+
+ int prev = -1;
+ int[] selectedFieldPositions = new int[projectedFields.length];
+ for (int i = 0; i < projectedFields.length; i++) {
+ int fieldId = projectedFields[i];
+ Integer position = columnIdPositions.get(fieldId);
+ if (position == null) {
+ throw new InvalidColumnProjectionException(
+ String.format(
+ "Projected field id %s is not contains in %s", fieldId, columnIds));
+ }
+
+ selectedFieldPositions[i] = position;
+ if (position < prev) {
+ throw new InvalidColumnProjectionException(
+ "The projection indexes should be in field order, but is "
+ + Arrays.toString(projectedFields));
+ }
+
+ prev = position;
+ }
+ return selectedFieldPositions;
+ }
+
+ static final class ProjectionKey {
+ private final long tableId;
+ private final short schemaId;
+
+ ProjectionKey(long tableId, short schemaId) {
+ this.tableId = tableId;
+ this.schemaId = schemaId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ProjectionKey)) {
+ return false;
+ }
+ ProjectionKey that = (ProjectionKey) o;
+ return tableId == that.tableId && schemaId == that.schemaId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableId, schemaId);
+ }
+ }
+
static final class ProjectionInfo {
final BitSet nodesProjection;
final BitSet buffersProjection;
@@ -441,7 +524,7 @@ public class FileLogProjection {
final RowType schema;
final int arrowMetadataLength;
final ArrowBodyCompression bodyCompression;
- final int[] selectedFields;
+ final int[] selectedFieldPositions;
private ProjectionInfo(
BitSet nodesProjection,
@@ -450,14 +533,14 @@ public class FileLogProjection {
RowType schema,
int arrowMetadataLength,
ArrowBodyCompression bodyCompression,
- int[] selectedFields) {
+ int[] selectedFieldPositions) {
this.nodesProjection = nodesProjection;
this.buffersProjection = buffersProjection;
this.bufferCount = bufferCount;
this.schema = schema;
this.arrowMetadataLength = arrowMetadataLength;
this.bodyCompression = bodyCompression;
- this.selectedFields = selectedFields;
+ this.selectedFieldPositions = selectedFieldPositions;
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java b/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java
index 385e457b8..88bc6a3c7 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java
@@ -73,7 +73,7 @@ public class IndexedLogRecord implements LogRecord {
this.timestamp = timestamp;
}
- private void pointTo(MemorySegment segment, int offset, int sizeInBytes) {
+ void pointTo(MemorySegment segment, int offset, int sizeInBytes) {
this.segment = segment;
this.offset = offset;
this.sizeInBytes = sizeInBytes;
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
index a0cc396b1..5fffc3f4f 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
@@ -18,25 +18,40 @@
package org.apache.fluss.record;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.types.DataType;
+import java.util.HashMap;
+import java.util.Map;
+
/** A default implementation of {@link KvRecordBatch.ReadContext} . */
public class KvRecordReadContext implements KvRecordBatch.ReadContext {
- private final RowDecoder rowDecoder;
+ private final KvFormat kvFormat;
+ private final SchemaGetter schemaGetter;
+ private final Map<Integer, RowDecoder> rowDecoderCache;
- private KvRecordReadContext(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
+ private KvRecordReadContext(KvFormat kvFormat, SchemaGetter schemaGetter) {
+ this.kvFormat = kvFormat;
+ this.schemaGetter = schemaGetter;
+ this.rowDecoderCache = new HashMap<>();
}
- public static KvRecordReadContext createReadContext(KvFormat kvFormat, DataType[] fieldTypes) {
- return new KvRecordReadContext(RowDecoder.create(kvFormat, fieldTypes));
+ public static KvRecordReadContext createReadContext(
+ KvFormat kvFormat, SchemaGetter schemaGetter) {
+ return new KvRecordReadContext(kvFormat, schemaGetter);
}
@Override
public RowDecoder getRowDecoder(int schemaId) {
- // assume schema id are always not changed for now.
- return rowDecoder;
+ return rowDecoderCache.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema schema = schemaGetter.getSchema((short) schemaId);
+ return RowDecoder.create(
+ kvFormat, schema.getRowType().getChildren().toArray(new DataType[0]));
+ });
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java
index cadc1ef9d..06536e582 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java
@@ -19,11 +19,14 @@ package org.apache.fluss.record;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
+import javax.annotation.Nullable;
+
import java.util.Iterator;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V0;
@@ -195,5 +198,18 @@ public interface LogRecordBatch {
/** Gets the buffer allocator. */
BufferAllocator getBufferAllocator();
+
+ /**
+ * If the read context defines an output projection (for example, log records may add new
+ * columns or reorder columns, but reader need a static schema for the output rows), return
+ * a {@link ProjectedRow} that describes the projected output row for the given schemaId.
+ * The returned object is used by readers to transform or materialize rows according to the
+ * output projection. Returns {@code null} if no output projection is configured.
+ *
+ * @param schemaId the current row schema id
+ * @return a {@link ProjectedRow} describing the output projection, or {@code null} if none
+ */
+ @Nullable
+ ProjectedRow getOutputProjectedRow(int schemaId);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
index 1158dbc35..50ec7ee66 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
@@ -19,9 +19,12 @@ package org.apache.fluss.record;
import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.InternalRow.FieldGetter;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
@@ -32,10 +35,11 @@ import org.apache.fluss.utils.Projection;
import javax.annotation.Nullable;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.stream.IntStream;
-import static org.apache.fluss.utils.Preconditions.checkArgument;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** A simple implementation for {@link LogRecordBatch.ReadContext}. */
@@ -46,7 +50,7 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
// the schema of the date read form server or remote. (which is projected in the server side)
private final RowType dataRowType;
// the static schemaId of the table, should support dynamic schema evolution in the future
- private final int schemaId;
+ private final int expectedSchemaId;
// the Arrow vector schema root of the table, should be null if not ARROW log format
@Nullable private final VectorSchemaRoot vectorSchemaRoot;
// the Arrow memory buffer allocator for the table, should be null if not ARROW log format
@@ -55,12 +59,14 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
private final FieldGetter[] selectedFieldGetters;
// whether the projection is push downed to the server side and the returned data is pruned.
private final boolean projectionPushDowned;
+ private final SchemaGetter schemaGetter;
+ private final Map<Integer, VectorSchemaRoot> vectorSchemaRootMap;
- /**
- * Creates a LogRecordReadContext for the given table information and projection information.
- */
public static LogRecordReadContext createReadContext(
- TableInfo tableInfo, boolean readFromRemote, @Nullable Projection projection) {
+ TableInfo tableInfo,
+ boolean readFromRemote,
+ @Nullable Projection projection,
+ SchemaGetter schemaGetter) {
RowType rowType = tableInfo.getRowType();
LogFormat logFormat = tableInfo.getTableConfig().getLogFormat();
// only for arrow log format, the projection can be push downed to the server side
@@ -68,33 +74,45 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
int schemaId = tableInfo.getSchemaId();
if (projection == null) {
// set a default dummy projection to simplify code
- projection = Projection.of(IntStream.range(0, rowType.getFieldCount()).toArray());
+ projection =
+ Projection.of(
+ IntStream.range(0, rowType.getFieldCount()).toArray(),
+ tableInfo.getSchema());
}
if (logFormat == LogFormat.ARROW) {
if (readFromRemote) {
// currently, for remote read, arrow log doesn't support projection pushdown,
// so set the rowType as is.
- int[] selectedFields = projection.getProjection();
- return createArrowReadContext(rowType, schemaId, selectedFields, false);
+ int[] selectedFields = projection.getProjectionPositions();
+ return createArrowReadContext(
+ rowType, schemaId, selectedFields, false, schemaGetter);
} else {
// arrow data that returned from server has been projected (in order)
RowType projectedRowType = projection.projectInOrder(rowType);
// need to reorder the fields for final output
int[] selectedFields = projection.getReorderingIndexes();
return createArrowReadContext(
- projectedRowType, schemaId, selectedFields, projectionPushDowned);
+ projectedRowType,
+ schemaId,
+ selectedFields,
+ projectionPushDowned,
+ schemaGetter);
}
} else if (logFormat == LogFormat.INDEXED) {
- int[] selectedFields = projection.getProjection();
- return createIndexedReadContext(rowType, schemaId, selectedFields);
+ int[] selectedFields = projection.getProjectionPositions();
+ return createIndexedReadContext(rowType, schemaId, selectedFields, schemaGetter);
} else {
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
}
private static LogRecordReadContext createArrowReadContext(
- RowType dataRowType, int schemaId, int[] selectedFields, boolean projectionPushDowned) {
+ RowType dataRowType,
+ int schemaId,
+ int[] selectedFields,
+ boolean projectionPushDowned,
+ SchemaGetter schemaGetter) {
// TODO: use a more reasonable memory limit
BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
VectorSchemaRoot vectorRoot =
@@ -107,31 +125,28 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
vectorRoot,
allocator,
fieldGetters,
- projectionPushDowned);
+ projectionPushDowned,
+ schemaGetter);
}
/**
- * Creates a testing purpose LogRecordReadContext for ARROW log format, that underlying Arrow
- * resources are not reused.
+ * Creates a LogRecordReadContext for INDEXED log format.
*
* @param rowType the schema of the table
* @param schemaId the schemaId of the table
*/
@VisibleForTesting
- public static LogRecordReadContext createArrowReadContext(RowType rowType, int schemaId) {
+ public static LogRecordReadContext createArrowReadContext(
+ RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createArrowReadContext(rowType, schemaId, selectedFields, false);
+ return createArrowReadContext(rowType, schemaId, selectedFields, false, schemaGetter);
}
- /**
- * Creates a LogRecordReadContext for INDEXED log format.
- *
- * @param rowType the schema of the table
- * @param schemaId the schemaId of the table
- */
- public static LogRecordReadContext createIndexedReadContext(RowType rowType, int schemaId) {
+ @VisibleForTesting
+ public static LogRecordReadContext createIndexedReadContext(
+ RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createIndexedReadContext(rowType, schemaId, selectedFields);
+ return createIndexedReadContext(rowType, schemaId, selectedFields, schemaGetter);
}
/**
@@ -142,28 +157,38 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
* @param selectedFields the final selected fields of the read data
*/
public static LogRecordReadContext createIndexedReadContext(
- RowType rowType, int schemaId, int[] selectedFields) {
+ RowType rowType, int schemaId, int[] selectedFields, SchemaGetter schemaGetter) {
FieldGetter[] fieldGetters = buildProjectedFieldGetters(rowType, selectedFields);
// for INDEXED log format, the projection is NEVER push downed to the server side
return new LogRecordReadContext(
- LogFormat.INDEXED, rowType, schemaId, null, null, fieldGetters, false);
+ LogFormat.INDEXED,
+ rowType,
+ schemaId,
+ null,
+ null,
+ fieldGetters,
+ false,
+ schemaGetter);
}
private LogRecordReadContext(
LogFormat logFormat,
- RowType dataRowType,
- int schemaId,
+ RowType targetDataRowType,
+ int expectedSchemaId,
VectorSchemaRoot vectorSchemaRoot,
BufferAllocator bufferAllocator,
FieldGetter[] selectedFieldGetters,
- boolean projectionPushDowned) {
+ boolean projectionPushDowned,
+ SchemaGetter schemaGetter) {
this.logFormat = logFormat;
- this.dataRowType = dataRowType;
- this.schemaId = schemaId;
+ this.dataRowType = targetDataRowType;
+ this.expectedSchemaId = expectedSchemaId;
this.vectorSchemaRoot = vectorSchemaRoot;
this.bufferAllocator = bufferAllocator;
this.selectedFieldGetters = selectedFieldGetters;
this.projectionPushDowned = projectionPushDowned;
+ this.schemaGetter = schemaGetter;
+ this.vectorSchemaRootMap = new HashMap<>();
}
@Override
@@ -173,12 +198,12 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
@Override
public RowType getRowType(int schemaId) {
- checkArgument(
- schemaId == this.schemaId,
- "The schemaId (%s) in the record batch is not the same as the context (%s).",
- schemaId,
- this.schemaId);
- return dataRowType;
+ if (isSameRowType(schemaId)) {
+ return dataRowType;
+ }
+
+ Schema schema = schemaGetter.getSchema(schemaId);
+ return schema.getRowType();
}
/** Get the selected field getters for the read data. */
@@ -193,17 +218,22 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
@Override
public VectorSchemaRoot getVectorSchemaRoot(int schemaId) {
- checkArgument(
- schemaId == this.schemaId,
- "The schemaId (%s) in the record batch is not the same as the context (%s).",
- schemaId,
- this.schemaId);
if (logFormat != LogFormat.ARROW) {
throw new IllegalArgumentException(
"Only Arrow log format provides vector schema root.");
}
- checkNotNull(vectorSchemaRoot, "The vector schema root is not available.");
- return vectorSchemaRoot;
+
+ if (schemaId == this.expectedSchemaId) {
+ checkNotNull(vectorSchemaRoot, "The vector schema root is not available.");
+ return vectorSchemaRoot;
+ }
+
+ RowType rowType = getRowType(schemaId);
+ return vectorSchemaRootMap.computeIfAbsent(
+ schemaId,
+ (id) ->
+ VectorSchemaRoot.create(
+ ArrowUtils.toArrowSchema(rowType), bufferAllocator));
}
@Override
@@ -215,15 +245,31 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
return bufferAllocator;
}
+ @Nullable
+ @Override
+ public ProjectedRow getOutputProjectedRow(int schemaId) {
+ if (isSameRowType(schemaId)) {
+ return null;
+ }
+ Schema originSchema = schemaGetter.getSchema(schemaId);
+ Schema expectedSchema = schemaGetter.getSchema(expectedSchemaId);
+ return ProjectedRow.from(originSchema, expectedSchema);
+ }
+
public void close() {
if (vectorSchemaRoot != null) {
vectorSchemaRoot.close();
}
+ vectorSchemaRootMap.values().forEach(VectorSchemaRoot::close);
if (bufferAllocator != null) {
bufferAllocator.close();
}
}
+ private boolean isSameRowType(int schemaId) {
+ return expectedSchemaId == schemaId || isProjectionPushDowned();
+ }
+
private static FieldGetter[] buildProjectedFieldGetters(RowType rowType, int[] selectedFields) {
List<DataType> dataTypeList = rowType.getChildren();
FieldGetter[] fieldGetters = new FieldGetter[selectedFields.length];
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java
index c1a1a0af9..e00559e00 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java
@@ -18,26 +18,39 @@
package org.apache.fluss.record;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.types.DataType;
+import java.util.HashMap;
+import java.util.Map;
+
/** A default implementation of {@link ValueRecordBatch.ReadContext} . */
public class ValueRecordReadContext implements ValueRecordBatch.ReadContext {
+ private final Map<Integer, RowDecoder> rowDecoderCache;
+ private final SchemaGetter schemaGetter;
+ private final KvFormat kvFormat;
- private final RowDecoder rowDecoder;
-
- public ValueRecordReadContext(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
+ private ValueRecordReadContext(SchemaGetter schemaGetter, KvFormat kvFormat) {
+ this.rowDecoderCache = new HashMap<>();
+ this.schemaGetter = schemaGetter;
+ this.kvFormat = kvFormat;
}
public static ValueRecordReadContext createReadContext(
- KvFormat kvFormat, DataType[] fieldTypes) {
- return new ValueRecordReadContext(RowDecoder.create(kvFormat, fieldTypes));
+ SchemaGetter schemaGetter, KvFormat kvFormat) {
+ return new ValueRecordReadContext(schemaGetter, kvFormat);
}
@Override
public RowDecoder getRowDecoder(int schemaId) {
- // assume schema id are always not changed for now.
- return rowDecoder;
+ return rowDecoderCache.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema schema = schemaGetter.getSchema(schemaId);
+ return RowDecoder.create(
+ kvFormat, schema.getRowType().getChildren().toArray(new DataType[0]));
+ });
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
index 0de55354f..6b4c516d1 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
@@ -18,9 +18,12 @@
package org.apache.fluss.row;
import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.metadata.Schema;
import java.util.Arrays;
+import static org.apache.fluss.utils.SchemaUtil.getIndexMapping;
+
/**
* An implementation of {@link InternalRow} which provides a projected view of the underlying {@link
* InternalRow}.
@@ -31,12 +34,13 @@ import java.util.Arrays;
*/
@PublicEvolving
public class ProjectedRow implements InternalRow {
+ public static final int UNEXIST_MAPPING = -1;
- private final int[] indexMapping;
+ protected final int[] indexMapping;
- private InternalRow row;
+ protected InternalRow row;
- private ProjectedRow(int[] indexMapping) {
+ protected ProjectedRow(int[] indexMapping) {
this.indexMapping = indexMapping;
}
@@ -60,6 +64,9 @@ public class ProjectedRow implements InternalRow {
@Override
public boolean isNullAt(int pos) {
+ if (indexMapping[pos] == UNEXIST_MAPPING) {
+ return true;
+ }
return row.isNullAt(indexMapping[pos]);
}
@@ -143,6 +150,12 @@ public class ProjectedRow implements InternalRow {
@Override
public boolean equals(Object o) {
+ // only can compare when the index mapping and row are equal.
+ if (o instanceof ProjectedRow
+ && Arrays.equals(indexMapping, ((ProjectedRow) o).indexMapping)
+ && row.equals(((ProjectedRow) o).row)) {
+ return true;
+ }
throw new UnsupportedOperationException("Projected row data cannot be compared");
}
@@ -168,4 +181,9 @@ public class ProjectedRow implements InternalRow {
public static ProjectedRow from(int[] projection) {
return new ProjectedRow(projection);
}
+
+ public static ProjectedRow from(Schema originSchema, Schema expectedSchema) {
+ int[] indexMapping = getIndexMapping(originSchema, expectedSchema);
+ return new ProjectedRow(indexMapping);
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
index 2fdc32f8e..e0252ae11 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
@@ -18,10 +18,17 @@
package org.apache.fluss.row.encode;
import org.apache.fluss.memory.MemorySegment;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.decode.RowDecoder;
+import org.apache.fluss.types.DataType;
+
+import java.util.Map;
import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH;
+import static org.apache.fluss.utils.MapUtils.newConcurrentHashMap;
/**
* A decoder to decode a schema id and {@link BinaryRow} from a byte array value which is encoded by
@@ -29,21 +36,31 @@ import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH;
*/
public class ValueDecoder {
- // todo: the row decoder should be inferred from the schema id encoded in the value
- private final RowDecoder rowDecoder;
-
- public ValueDecoder(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
- }
+ private final Map<Short, RowDecoder> rowDecoders;
+ private final SchemaGetter schemaGetter;
+ private final KvFormat kvFormat;
- public RowDecoder getRowDecoder() {
- return rowDecoder;
+ public ValueDecoder(SchemaGetter schemaGetter, KvFormat kvFormat) {
+ this.rowDecoders = newConcurrentHashMap();
+ this.schemaGetter = schemaGetter;
+ this.kvFormat = kvFormat;
}
/** Decode the value bytes and return the schema id and the row encoded in the value bytes. */
public Value decodeValue(byte[] valueBytes) {
MemorySegment memorySegment = MemorySegment.wrap(valueBytes);
short schemaId = memorySegment.getShort(0);
+
+ RowDecoder rowDecoder =
+ rowDecoders.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema schema = schemaGetter.getSchema(schemaId);
+ return RowDecoder.create(
+ kvFormat,
+ schema.getRowType().getChildren().toArray(new DataType[0]));
+ });
+
BinaryRow row =
rowDecoder.decode(
memorySegment, SCHEMA_ID_LENGTH, valueBytes.length - SCHEMA_ID_LENGTH);
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
index 80fea90fa..bbfc6a7f5 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
@@ -17,7 +17,10 @@
package org.apache.fluss.row.encode;
+import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.UnsafeUtils;
/** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */
@@ -38,4 +41,24 @@ public class ValueEncoder {
row.copyTo(values, SCHEMA_ID_LENGTH);
return values;
}
+
+ public static byte[] encodeRow(
+ short schemaId, KvFormat kvFormat, RowType currentRowType, InternalRow row)
+ throws Exception {
+ if (row instanceof BinaryRow) {
+ return encodeValue(schemaId, (BinaryRow) row);
+ }
+ // todo: reuse the encoder here
+ try (RowEncoder rowEncoder = RowEncoder.create(kvFormat, currentRowType)) {
+ rowEncoder.startNewRow();
+ for (int i = 0; i < currentRowType.getFieldCount(); i++) {
+ rowEncoder.encodeField(
+ i,
+ InternalRow.createFieldGetter(currentRowType.getTypeAt(i), i)
+ .getFieldOrNull(row));
+ }
+ BinaryRow binaryRow = rowEncoder.finishRow();
+ return encodeValue(schemaId, binaryRow);
+ }
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/types/RowType.java b/fluss-common/src/main/java/org/apache/fluss/types/RowType.java
index a282c3286..090fe9655 100644
--- a/fluss-common/src/main/java/org/apache/fluss/types/RowType.java
+++ b/fluss-common/src/main/java/org/apache/fluss/types/RowType.java
@@ -98,17 +98,17 @@ public final class RowType extends DataType {
return -1;
}
- public RowType project(int[] projectFields) {
+ public RowType project(int[] projectFieldPositions) {
List<DataField> projectedFields = new ArrayList<>();
- for (int projectField : projectFields) {
+ for (int projectField : projectFieldPositions) {
projectedFields.add(this.fields.get(projectField));
}
return new RowType(this.isNullable(), projectedFields);
}
- public RowType project(List<String> projectFields) {
+ public RowType project(List<String> projectFieldNames) {
List<DataField> projectedFields = new ArrayList<>();
- for (String projectField : projectFields) {
+ for (String projectField : projectFieldNames) {
int index = getFieldIndex(projectField);
if (index == -1) {
throw new IllegalArgumentException(
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
index c16c6b10b..c597caf81 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
@@ -17,9 +17,11 @@
package org.apache.fluss.utils;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.types.RowType;
import java.util.Arrays;
+import java.util.List;
import static org.apache.fluss.utils.Preconditions.checkState;
@@ -28,12 +30,14 @@ import static org.apache.fluss.utils.Preconditions.checkState;
* projection includes both reducing the accessible fields and reordering them. Currently, this only
* supports top-level projection. Nested projection will be supported in the future.
*
- * <p>For example, given a row with fields [a, b, c, d, e], a projection [2, 0, 3] will project the
- * row to [c, a, d]. The projection indexes are 0-based.
+ * <p>For example, given a row with fields [a, b, c, d, e] with id [0, 1, 3, 4, 5], a projection [2,
+ * 0, 3] will project the row to [c, a, d] with id [3, 0, 4]. The projection indexes are 0-based.
*
* <ul>
- * <li>The {@link #projection} indexes will be [2, 0, 3]
- * <li>The {@link #projectionInOrder} indexes will be [0, 2, 3]
+ * <li>The {@link #projectionPositions} indexes will be [2, 0, 3]
+ * <li>The {@link #projectionPositions} id will be [3, 0, 4]
+ * <li>The {@link #projectionIdsInOrder} indexes will be [0, 3, 4]
+ * <li>The {@link #projectionPositionsOrderById} indexes will be [0, 2, 3]
* <li>The {@link #reorderingIndexes} indexes will be [1, 0, 2]
* </ul>
*
@@ -41,42 +45,80 @@ import static org.apache.fluss.utils.Preconditions.checkState;
*/
public class Projection {
/** the projection indexes including both selected fields and reordering them. */
- final int[] projection;
+ final int[] projectionPositions;
/** the projection indexes that only select fields but not reordering them. */
- final int[] projectionInOrder;
- /** the indexes to reorder the fields of {@link #projectionInOrder} to {@link #projection}. */
+ final int[] projectionPositionsOrderById;
+ /** the projection indexes that only select fields but not reordering them. */
+ final int[] projectionIdsInOrder;
+ /**
+ * the indexes to reorder the fields of {@link #projectionPositionsOrderById} to {@link
+ * #projectionPositions}.
+ */
final int[] reorderingIndexes;
/** the flag to indicate whether reordering is needed. */
final boolean reorderingNeeded;
- /** Create a {@link Projection} of the provided {@code indexes}. */
- public static Projection of(int[] indexes) {
- return new Projection(indexes);
+ public static Projection of(int[] indexes, Schema schema) {
+ int[] ids = new int[indexes.length];
+ List<Integer> columnIds = schema.getColumnIds();
+ for (int i = 0; i < indexes.length; i++) {
+ ids[i] = columnIds.get(indexes[i]);
+ }
+
+ return new Projection(indexes, ids);
}
- private Projection(int[] projection) {
- this.projection = projection;
- this.projectionInOrder = Arrays.copyOf(projection, projection.length);
- Arrays.sort(projectionInOrder);
- this.reorderingNeeded = !Arrays.equals(projection, projectionInOrder);
- this.reorderingIndexes = new int[projection.length];
- for (int i = 0; i < projection.length; i++) {
- int index = Arrays.binarySearch(projectionInOrder, projection[i]);
+ /**
+ * Create a {@link Projection} of the provided {@code indexes} and {@code projectionIds}.
+ *
+ * <p>Typically, {@code projectionIndexes} and {@code projectionIds} are the same. But when
+ * removing a middle column or reordering columns, they won't be the same. In this case, when
+ * the schema on the fluss server side differs from the schema on the client side during user
+ * queries, using {@code projectionIds} ensures correctness by mapping to the actual server-side
+ * column positions.
+ *
+ * @param projectionIndexes the indexes of the projection, which is used for user to read data
+ * from client side. These indexes are based on the schema visible to the client side and
+ * are used to parse the data returned by the fluss server.
+ * @param projectionIds the ids of the projection, which is used for fluss server scan. These
+ * ids are based on the actual schema on the fluss server side and are pushed down to the
+ * server for querying.
+ */
+ private Projection(int[] projectionIndexes, int[] projectionIds) {
+ checkState(
+ projectionIds.length == projectionIndexes.length,
+ "The number of ids must be equal to the number of indexes.");
+ this.projectionPositions = projectionIndexes;
+
+ // reorder the projection id for lookup.
+ this.projectionIdsInOrder = Arrays.copyOf(projectionIds, projectionIds.length);
+ Arrays.sort(projectionIdsInOrder);
+ this.reorderingNeeded = !Arrays.equals(projectionIds, projectionIdsInOrder);
+ this.reorderingIndexes = new int[projectionPositions.length];
+ this.projectionPositionsOrderById = new int[projectionPositions.length];
+ for (int i = 0; i < projectionIds.length; i++) {
+ int index = Arrays.binarySearch(projectionIdsInOrder, projectionIds[i]);
checkState(index >= 0, "The projection index is invalid.");
reorderingIndexes[i] = index;
+ projectionPositionsOrderById[i] = projectionIndexes[index];
}
}
public RowType projectInOrder(RowType rowType) {
- return rowType.project(projectionInOrder);
+ return rowType.project(projectionPositionsOrderById);
}
- public int[] getProjection() {
- return projection;
+ public int[] getProjectionPositions() {
+ return projectionPositions;
}
- public int[] getProjectionInOrder() {
- return projectionInOrder;
+ /**
+ * The id of the projection, which is used for fluss server scan.
+ *
+ * @return
+ */
+ public int[] getProjectionIdInOrder() {
+ return projectionIdsInOrder;
}
public boolean isReorderingNeeded() {
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
new file mode 100644
index 000000000..6b66c0cd6
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.utils;
+
+import org.apache.fluss.exception.SchemaChangeException;
+import org.apache.fluss.metadata.Schema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.fluss.row.ProjectedRow.UNEXIST_MAPPING;
+
+/** Schema util. */
+public class SchemaUtil {
+
+ /**
+ * get the target columns in expected schema.
+ *
+ * @param targetColumns
+ * @param originSchema
+ * @param expectedSchema
+ * @return
+ */
+ public static int[] getTargetColumns(
+ int[] targetColumns, Schema originSchema, Schema expectedSchema) {
+ if (targetColumns == null) {
+ return null;
+ }
+
+ // get the mapping from expected schema to origin schema
+ int[] indexMapping = getIndexMapping(expectedSchema, originSchema);
+ ArrayList<Integer> targetColumnsInExpectedSchema = new ArrayList<>();
+ for (int i = 0; i < targetColumns.length; i++) {
+ if (indexMapping[targetColumns[i]] != UNEXIST_MAPPING) {
+ targetColumnsInExpectedSchema.add(indexMapping[targetColumns[i]]);
+ }
+ }
+ return targetColumnsInExpectedSchema.stream().mapToInt(value -> value).toArray();
+ }
+
+ /**
+ * Get the index of each value of expectedSchema from originSchema.
+ *
+ * @param originSchema
+ * @param expectedSchema
+ * @return indexMapping the value of i-th means the expectedSchema's i-th column is the
+ * originSchema's indexMapping[i]-th column.
+ */
+ public static int[] getIndexMapping(Schema originSchema, Schema expectedSchema) {
+ List<Schema.Column> originColumns = originSchema.getColumns();
+ Map<Integer, Integer> originColumnIdToIndex = new HashMap<>();
+ for (int i = 0; i < originColumns.size(); i++) {
+ originColumnIdToIndex.put(originColumns.get(i).getColumnId(), i);
+ }
+ List<Schema.Column> expectedColumns = expectedSchema.getColumns();
+
+ int[] indexMapping = new int[expectedColumns.size()];
+ for (int i = 0; i < expectedColumns.size(); i++) {
+ Schema.Column expectedColumn = expectedColumns.get(i);
+ indexMapping[i] =
+ originColumnIdToIndex.getOrDefault(
+ expectedColumn.getColumnId(), UNEXIST_MAPPING);
+ Schema.Column originColumn =
+ indexMapping[i] == UNEXIST_MAPPING ? null : originColumns.get(indexMapping[i]);
+
+ if (originColumn != null
+ && !Objects.equals(
+ expectedColumn.getDataType().copy(true),
+ originColumns.get(indexMapping[i]).getDataType().copy(true))) {
+
+ throw new SchemaChangeException(
+ String.format(
+ "Expected datatype of column(id=%s,name=%s) is [%s], while the actual datatype is [%s]",
+ expectedColumn.getColumnId(),
+ expectedColumn.getName(),
+ expectedColumn.getDataType(),
+ originColumn.getDataType()));
+ }
+ }
+
+ return indexMapping;
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java b/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java
index 258b96007..5c8fdb6c5 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java
@@ -74,6 +74,10 @@ public class UnsafeUtils {
return MemoryUtils.UNSAFE.getInt(target, BYTE_ARRAY_BASE_OFFSET + offset);
}
+ public static short getShort(byte[] target, long offset) {
+ return MemoryUtils.UNSAFE.getShort(target, BYTE_ARRAY_BASE_OFFSET + offset);
+ }
+
public static long getLong(byte[] target, long offset) {
return MemoryUtils.UNSAFE.getLong(target, BYTE_ARRAY_BASE_OFFSET + offset);
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
index 1994ba61a..5db5b65ed 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
@@ -25,6 +25,8 @@ import org.apache.fluss.types.DataType;
import java.io.IOException;
+import static org.apache.fluss.metadata.Schema.Column.UNKNOWN_COLUMN_ID;
+
/** Json serializer and deserializer for {@link Schema.Column}. */
@Internal
public class ColumnJsonSerde
@@ -32,6 +34,7 @@ public class ColumnJsonSerde
public static final ColumnJsonSerde INSTANCE = new ColumnJsonSerde();
static final String NAME = "name";
+ static final String ID = "id";
static final String DATA_TYPE = "data_type";
static final String COMMENT = "comment";
@@ -46,6 +49,7 @@ public class ColumnJsonSerde
if (column.getComment().isPresent()) {
generator.writeStringField(COMMENT, column.getComment().get());
}
+ generator.writeNumberField(ID, column.getColumnId());
generator.writeEndObject();
}
@@ -55,11 +59,11 @@ public class ColumnJsonSerde
String columnName = node.required(NAME).asText();
DataType dataType = DataTypeJsonSerde.INSTANCE.deserialize(node.get(DATA_TYPE));
- Schema.Column column = new Schema.Column(columnName, dataType);
- if (node.hasNonNull(COMMENT)) {
- column = column.withComment(node.get(COMMENT).asText());
- }
- return column;
+ return new Schema.Column(
+ columnName,
+ dataType,
+ node.hasNonNull(COMMENT) ? node.get(COMMENT).asText() : null,
+ node.has(ID) ? node.get(ID).asInt() : UNKNOWN_COLUMN_ID);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java
index a4e5fd8ad..527247b6e 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java
@@ -37,6 +37,7 @@ public class SchemaJsonSerde implements JsonSerializer<Schema>, JsonDeserializer
private static final String COLUMNS_NAME = "columns";
private static final String PRIMARY_KEY_NAME = "primary_key";
private static final String VERSION_KEY = "version";
+ private static final String HIGHEST_FIELD_ID = "highest_field_id";
private static final int VERSION = 1;
@Override
@@ -62,6 +63,8 @@ public class SchemaJsonSerde implements JsonSerializer<Schema>, JsonDeserializer
generator.writeEndArray();
}
+ generator.writeNumberField(HIGHEST_FIELD_ID, schema.getHighestFieldId());
+
generator.writeEndObject();
}
@@ -83,6 +86,10 @@ public class SchemaJsonSerde implements JsonSerializer<Schema>, JsonDeserializer
builder.primaryKey(primaryKeys);
}
+ if (node.has(HIGHEST_FIELD_ID)) {
+ builder.highestFieldId(node.get(HIGHEST_FIELD_ID).asInt());
+ }
+
return builder.build();
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java
index d3e9cf2f6..6c8b483a0 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link DefaultKvRecordBatch}. */
@@ -67,7 +68,7 @@ class DefaultKvRecordBatchTest extends KvTestBase {
for (KvRecord record :
kvRecords.records(
KvRecordReadContext.createReadContext(
- KvFormat.COMPACTED, baseRowFieldTypes))) {
+ KvFormat.COMPACTED, new TestingSchemaGetter(1, DATA1_SCHEMA)))) {
assertThat(keyToBytes(record)).isEqualTo(keys.get(i));
assertThat(record.getRow()).isEqualTo(rows.get(i));
i++;
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java
index ee3859be9..8e6774221 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.util.Random;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link DefaultKvRecord}. */
@@ -39,7 +40,8 @@ class DefaultKvRecordTest extends KvTestBase {
@Test
void testBase() throws Exception {
KvRecordReadContext kvRecordReadContext =
- KvRecordReadContext.createReadContext(KvFormat.COMPACTED, baseRowFieldTypes);
+ KvRecordReadContext.createReadContext(
+ KvFormat.COMPACTED, new TestingSchemaGetter(1, DATA1_SCHEMA));
// create row.
CompactedRow row;
@@ -107,7 +109,8 @@ class DefaultKvRecordTest extends KvTestBase {
MemorySegment.wrap(outputView.getCopyOfBuffer()),
0,
schemaId,
- KvRecordReadContext.createReadContext(KvFormat.COMPACTED, colTypes));
+ KvRecordReadContext.createReadContext(
+ KvFormat.COMPACTED, new TestingSchemaGetter(1, DATA1_SCHEMA)));
// check key
assertThat(keyToBytes(kvRecord)).isEqualTo(key);
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
index 70f881e06..fb55be89a 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
@@ -18,6 +18,9 @@
package org.apache.fluss.record;
import org.apache.fluss.memory.UnmanagedPagedOutputView;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.row.TestInternalRowGenerator;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.testutils.DataTestUtils;
@@ -87,10 +90,15 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.isValid()).isTrue();
assertThat(logRecordBatch.schemaId()).isEqualTo(schemaId);
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(
+ new SchemaInfo(
+ Schema.newBuilder().fromRowType(allRowType).build(), schemaId));
// verify record.
int i = 0;
try (LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(allRowType, schemaId);
+ LogRecordReadContext.createIndexedReadContext(
+ allRowType, schemaId, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
while (iter.hasNext()) {
LogRecord record = iter.next();
@@ -125,8 +133,13 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.lastLogOffset()).isEqualTo(0);
assertThat(logRecordBatch.nextLogOffset()).isEqualTo(1);
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(0);
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(
+ new SchemaInfo(
+ Schema.newBuilder().fromRowType(baseRowType).build(), schemaId));
try (LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(baseRowType, schemaId);
+ LogRecordReadContext.createIndexedReadContext(
+ baseRowType, schemaId, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
@@ -153,8 +166,10 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.lastLogOffset()).isEqualTo(100);
assertThat(logRecordBatch.nextLogOffset()).isEqualTo(101);
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(100);
+
try (LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(baseRowType, schemaId);
+ LogRecordReadContext.createIndexedReadContext(
+ baseRowType, schemaId, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java
index 7de046ec7..1e6e51d55 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java
@@ -30,6 +30,7 @@ import java.util.Collections;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V0;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
import static org.apache.fluss.testutils.DataTestUtils.createRecordsWithoutBaseLogOffset;
import static org.assertj.core.api.Assertions.assertThat;
@@ -62,8 +63,10 @@ public class FileLogInputStreamTest extends LogTestBase {
LogRecordBatch recordBatch = batch.loadFullBatch();
+ TestingSchemaGetter schemaGetter = new TestingSchemaGetter(schemaId, DATA1_SCHEMA);
try (LogRecordReadContext readContext =
- LogRecordReadContext.createArrowReadContext(DATA1_ROW_TYPE, schemaId);
+ LogRecordReadContext.createArrowReadContext(
+ DATA1_ROW_TYPE, schemaId, schemaGetter);
CloseableIterator<LogRecord> iterator = recordBatch.records(readContext)) {
assertThat(iterator.hasNext()).isTrue();
LogRecord record = iterator.next();
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
index c19ecc882..2c3ff36ef 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
@@ -18,11 +18,14 @@
package org.apache.fluss.record;
import org.apache.fluss.exception.InvalidColumnProjectionException;
+import org.apache.fluss.exception.SchemaNotExistException;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
@@ -56,77 +59,130 @@ import static org.assertj.core.api.Assertions.fail;
class FileLogProjectionTest {
private @TempDir File tempDir;
+ private TestingSchemaGetter testingSchemaGetter;
+
+ @BeforeEach
+ void beforeEach() {
+ testingSchemaGetter = new TestingSchemaGetter(new SchemaInfo(TestData.DATA1_SCHEMA, 1));
+ testingSchemaGetter.updateLatestSchemaInfo(new SchemaInfo(TestData.DATA2_SCHEMA, 2));
+ }
- // TODO: add tests for nested types
@Test
- void testSetCurrentProjection() {
+ void testSetCurrentProjection() throws Exception {
+ short schemaId = (short) 2;
+ FileLogRecords recordsOfData2RowType =
+ createFileLogRecords(
+ schemaId,
+ LOG_MAGIC_VALUE_V1,
+ TestData.DATA2_ROW_TYPE,
+ TestData.DATA2,
+ TestData.DATA2);
+
FileLogProjection projection = new FileLogProjection();
- projection.setCurrentProjection(
- 1L, TestData.DATA2_ROW_TYPE, DEFAULT_COMPRESSION, new int[] {0, 2});
+ // get schema during running.
+ doProjection(
+ 1L,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {0, 2},
+ recordsOfData2RowType.sizeInBytes());
FileLogProjection.ProjectionInfo info1 = projection.currentProjection;
assertThat(info1).isNotNull();
assertThat(info1.nodesProjection.stream().toArray()).isEqualTo(new int[] {0, 2});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info1.buffersProjection.stream().toArray()).isEqualTo(new int[] {0, 1, 5, 6, 7});
assertThat(projection.projectionsCache).hasSize(1);
- assertThat(projection.projectionsCache.get(1L)).isSameAs(info1);
+ assertThat(
+ projection.projectionsCache.get(
+ new FileLogProjection.ProjectionKey(1L, schemaId)))
+ .isSameAs(info1);
- projection.setCurrentProjection(
- 2L, TestData.DATA2_ROW_TYPE, DEFAULT_COMPRESSION, new int[] {1});
+ doProjection(
+ 2L,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {1},
+ recordsOfData2RowType.sizeInBytes());
FileLogProjection.ProjectionInfo info2 = projection.currentProjection;
assertThat(info2).isNotNull();
assertThat(info2.nodesProjection.stream().toArray()).isEqualTo(new int[] {1});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info2.buffersProjection.stream().toArray()).isEqualTo(new int[] {2, 3, 4});
assertThat(projection.projectionsCache).hasSize(2);
- assertThat(projection.projectionsCache.get(2L)).isSameAs(info2);
+ assertThat(
+ projection.projectionsCache.get(
+ new FileLogProjection.ProjectionKey(2L, schemaId)))
+ .isSameAs(info2);
- projection.setCurrentProjection(
- 1L, TestData.DATA2_ROW_TYPE, DEFAULT_COMPRESSION, new int[] {0, 2});
+ doProjection(
+ 1L,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {0, 2},
+ recordsOfData2RowType.sizeInBytes());
assertThat(projection.currentProjection).isNotNull().isSameAs(info1);
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
+ doProjection(
1L,
- TestData.DATA1_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {1}))
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {1},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
.hasMessage("The schema and projection should be identical for the same table id.");
}
@Test
- void testIllegalSetCurrentProjection() {
+ void testIllegalSetCurrentProjection() throws Exception {
+ long tableId = 1L;
+ short schemaId = (short) 2;
+ FileLogRecords recordsOfData2RowType =
+ createFileLogRecords(
+ schemaId,
+ LOG_MAGIC_VALUE_V1,
+ TestData.DATA2_ROW_TYPE,
+ TestData.DATA2,
+ TestData.DATA2);
FileLogProjection projection = new FileLogProjection();
-
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
- 1L,
- TestData.DATA2_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {3}))
+ doProjection(
+ tableId,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {3},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
- .hasMessage("Projected fields [3] is out of bound for schema with 3 fields.");
+ .hasMessage("Projected field id 3 is not contains in [0, 1, 2]");
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
- 1L,
- TestData.DATA2_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {1, 0}))
+ doProjection(
+ tableId,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {1, 0},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
.hasMessage("The projection indexes should be in field order, but is [1, 0]");
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
- 1L,
- TestData.DATA2_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {0, 0, 0}))
+ doProjection(
+ tableId,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {0, 0, 0},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
.hasMessage(
"The projection indexes should not contain duplicated fields, but is [0, 0, 0]");
@@ -134,28 +190,38 @@ class FileLogProjectionTest {
static Stream<Arguments> projectedFieldsArgs() {
return Stream.of(
- Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V0),
- Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V0),
- Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V0),
- Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V1),
- Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V1),
- Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V1));
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V0, (short) 1),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V0, (short) 1),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V0, (short) 1),
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V1, (short) 1),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V1, (short) 1),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V1, (short) 1),
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V0, (short) 2),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V0, (short) 2),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V0, (short) 2),
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V1, (short) 2),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V1, (short) 2),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V1, (short) 2));
}
@ParameterizedTest
@MethodSource("projectedFieldsArgs")
- void testProject(int[] projectedFields, byte recordBatchMagic) throws Exception {
+ void testProject(int[] projectedFields, byte recordBatchMagic, short schemaId)
+ throws Exception {
+ testingSchemaGetter.updateLatestSchemaInfo(new SchemaInfo(TestData.DATA1_SCHEMA, schemaId));
FileLogRecords fileLogRecords =
createFileLogRecords(
+ schemaId,
recordBatchMagic,
TestData.DATA1_ROW_TYPE,
TestData.DATA1,
TestData.ANOTHER_DATA1);
List<Object[]> results =
doProjection(
+ 1L,
+ schemaId,
new FileLogProjection(),
fileLogRecords,
- TestData.DATA1_ROW_TYPE,
projectedFields,
Integer.MAX_VALUE);
List<Object[]> allData = new ArrayList<>();
@@ -176,7 +242,7 @@ class FileLogProjectionTest {
@ParameterizedTest
@ValueSource(bytes = {LOG_MAGIC_VALUE_V0, LOG_MAGIC_VALUE_V1})
void testIllegalByteOrder(byte recordBatchMagic) throws Exception {
- FileLogRecords fileLogRecords =
+ final FileLogRecords fileLogRecords =
createFileLogRecords(
recordBatchMagic,
TestData.DATA1_ROW_TYPE,
@@ -185,17 +251,38 @@ class FileLogProjectionTest {
FileLogProjection projection = new FileLogProjection();
// overwrite the wrong decoding byte order endian
projection.getLogHeaderBuffer().order(ByteOrder.BIG_ENDIAN);
- // should throw exception.
assertThatThrownBy(
() ->
doProjection(
projection,
fileLogRecords,
- TestData.DATA1_ROW_TYPE,
+ new int[] {0, 1},
+ Integer.MAX_VALUE))
+ .isInstanceOf(SchemaNotExistException.class);
+
+ // The BIG_ENDIAN of schema will be read as 256 wronly. In this case, we use 256 to skip it.
+ short schemaId = 256;
+ testingSchemaGetter.updateLatestSchemaInfo(new SchemaInfo(TestData.DATA2_SCHEMA, schemaId));
+ final FileLogRecords fileLogRecords2 =
+ createFileLogRecords(
+ schemaId,
+ recordBatchMagic,
+ TestData.DATA1_ROW_TYPE,
+ TestData.DATA1,
+ TestData.ANOTHER_DATA1);
+
+ assertThatThrownBy(
+ () ->
+ doProjection(
+ 1L,
+ schemaId,
+ projection,
+ fileLogRecords2,
new int[] {0, 1},
Integer.MAX_VALUE))
.isInstanceOf(EOFException.class)
.hasMessageContaining("Failed to read `arrow header` from file channel");
+ fileLogRecords2.close();
}
@ParameterizedTest
@@ -218,11 +305,7 @@ class FileLogProjectionTest {
int maxBytes = totalSize / i;
List<Object[]> results =
doProjection(
- new FileLogProjection(),
- fileLogRecords,
- TestData.DATA1_ROW_TYPE,
- new int[] {0, 1},
- maxBytes);
+ new FileLogProjection(), fileLogRecords, new int[] {0, 1}, maxBytes);
if (results.isEmpty()) {
hasEmpty = true;
} else if (results.size() == TestData.DATA1.size()) {
@@ -323,13 +406,20 @@ class FileLogProjectionTest {
@SafeVarargs
final FileLogRecords createFileLogRecords(
byte recordBatchMagic, RowType rowType, List<Object[]>... inputs) throws Exception {
+ return createFileLogRecords(DEFAULT_SCHEMA_ID, recordBatchMagic, rowType, inputs);
+ }
+
+ @SafeVarargs
+ final FileLogRecords createFileLogRecords(
+ int schemaId, byte recordBatchMagic, RowType rowType, List<Object[]>... inputs)
+ throws Exception {
FileLogRecords fileLogRecords = FileLogRecords.open(new File(tempDir, "test.tmp"));
long offsetBase = 0L;
for (List<Object[]> input : inputs) {
fileLogRecords.append(
createRecordsWithoutBaseLogOffset(
rowType,
- DEFAULT_SCHEMA_ID,
+ schemaId,
offsetBase,
System.currentTimeMillis(),
recordBatchMagic,
@@ -344,20 +434,33 @@ class FileLogProjectionTest {
private List<Object[]> doProjection(
FileLogProjection projection,
FileLogRecords fileLogRecords,
- RowType rowType,
int[] projectedFields,
int fetchMaxBytes)
throws Exception {
- projection.setCurrentProjection(1L, rowType, DEFAULT_COMPRESSION, projectedFields);
- RowType projectedType = rowType.project(projectedFields);
+ return doProjection(
+ 1L, DEFAULT_SCHEMA_ID, projection, fileLogRecords, projectedFields, fetchMaxBytes);
+ }
+
+ private List<Object[]> doProjection(
+ long tableId,
+ short schemaId,
+ FileLogProjection projection,
+ FileLogRecords fileLogRecords,
+ int[] projectedFields,
+ int fetchMaxBytes)
+ throws Exception {
+ projection.setCurrentProjection(
+ tableId, testingSchemaGetter, DEFAULT_COMPRESSION, projectedFields);
LogRecords project =
projection.project(
fileLogRecords.channel(), 0, fileLogRecords.sizeInBytes(), fetchMaxBytes);
assertThat(project.sizeInBytes()).isLessThanOrEqualTo(fetchMaxBytes);
+ RowType rowType = testingSchemaGetter.getSchema(schemaId).getRowType();
+ RowType projectedType = rowType.project(projectedFields);
List<Object[]> results = new ArrayList<>();
long expectedOffset = 0L;
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, DEFAULT_SCHEMA_ID)) {
+ createArrowReadContext(projectedType, schemaId, testingSchemaGetter)) {
for (LogRecordBatch batch : project.batches()) {
try (CloseableIterator<LogRecord> records = batch.records(context)) {
while (records.hasNext()) {
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java b/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java
index 8e7395ca1..a8f851cee 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java
@@ -17,6 +17,8 @@
package org.apache.fluss.record;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.TestInternalRowGenerator;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.testutils.DataTestUtils;
@@ -75,8 +77,10 @@ final class LogRecordsTest extends LogTestBase {
assertThat(iterator.hasNext()).isTrue();
assertThat(memoryIterator.hasNext()).isTrue();
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(1, Schema.newBuilder().fromRowType(baseRowType).build());
assertIndexedLogRecordBatchAndRowEquals(
- iterator.next(), memoryIterator.next(), baseRowType, rows1);
+ iterator.next(), memoryIterator.next(), baseRowType, rows1, schemaGetter);
assertThat(iterator.hasNext()).isFalse();
assertThat(memoryIterator.hasNext()).isFalse();
@@ -98,8 +102,10 @@ final class LogRecordsTest extends LogTestBase {
assertThat(iterator.hasNext()).isTrue();
assertThat(memoryIterator.hasNext()).isTrue();
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(1, Schema.newBuilder().fromRowType(allRowType).build());
assertIndexedLogRecordBatchAndRowEquals(
- iterator.next(), memoryIterator.next(), allRowType, rows1);
+ iterator.next(), memoryIterator.next(), allRowType, rows1, schemaGetter);
assertThat(iterator.hasNext()).isFalse();
assertThat(memoryIterator.hasNext()).isFalse();
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
index 3183d1acf..108ab2acf 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
@@ -19,6 +19,7 @@ package org.apache.fluss.record;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.memory.MemorySegmentOutputView;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.TestInternalRowGenerator;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.testutils.ListLogRecords;
@@ -71,15 +72,18 @@ public abstract class LogTestBase {
assertLogRecordsListEquals(expected, actual, baseRowType);
}
+ // todo: 在这个基础上可以加更多最简单的schema evolution测试，非常好
protected void assertIndexedLogRecordBatchAndRowEquals(
LogRecordBatch actual,
LogRecordBatch expected,
RowType rowType,
- List<IndexedRow> rows) {
+ List<IndexedRow> rows,
+ SchemaGetter schemaGetter) {
assertRecordBatchHeaderEquals(actual);
LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(rowType, TestData.DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createIndexedReadContext(
+ rowType, TestData.DEFAULT_SCHEMA_ID, schemaGetter);
try (CloseableIterator<LogRecord> actualIter = actual.records(readContext);
CloseableIterator<LogRecord> expectIter = expected.records(readContext)) {
int i = 0;
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
index 5c0df6e36..e72b44f14 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
@@ -58,6 +58,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.row.arrow.ArrowWriter.BUFFER_USAGE_RATIO;
import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals;
import static org.apache.fluss.testutils.DataTestUtils.row;
@@ -278,9 +279,10 @@ public class MemoryLogRecordsArrowBuilderTest {
assertThat(logRecordBatch.lastLogOffset()).isEqualTo(0);
assertThat(logRecordBatch.nextLogOffset()).isEqualTo(1);
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(0);
+ TestingSchemaGetter schemaGetter = TEST_SCHEMA_GETTER;
try (LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
@@ -305,7 +307,7 @@ public class MemoryLogRecordsArrowBuilderTest {
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(100);
try (LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestData.java b/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
index 7752b15b0..874377125 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
@@ -217,4 +217,7 @@ public final class TestData {
.primaryKey("a")
.build();
// ---------------------------- data3 table info end ------------------------------
+
+ public static final TestingSchemaGetter TEST_SCHEMA_GETTER =
+ new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA2_SCHEMA);
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
new file mode 100644
index 000000000..f74f379eb
--- /dev/null
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.record;
+
+import org.apache.fluss.exception.SchemaNotExistException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** A testing schema metadata subscriber. */
+public class TestingSchemaGetter implements SchemaGetter {
+ private SchemaInfo schemaInfo;
+ private Map<Integer, Schema> schemaCaches;
+
+ public TestingSchemaGetter() {
+ this.schemaInfo = null;
+ this.schemaCaches = new HashMap<>();
+ }
+
+ // todo: remove it(简化测试代码）
+ public TestingSchemaGetter(SchemaInfo schemaInfo) {
+ this.schemaInfo = schemaInfo;
+ this.schemaCaches = new HashMap<>();
+ this.schemaCaches.put(schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+
+ public TestingSchemaGetter(int schemaId, Schema schema) {
+ this.schemaInfo = new SchemaInfo(schema, schemaId);
+ this.schemaCaches = new HashMap<>();
+ this.schemaCaches.put(schemaId, schema);
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ if (!schemaCaches.containsKey(schemaId)) {
+ throw new SchemaNotExistException("Schema not exist");
+ }
+ return schemaCaches.get(schemaId);
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ return schemaInfo;
+ }
+
+ @Override
+ public void release() {}
+
+ public void updateLatestSchemaInfo(SchemaInfo schemaInfo) {
+ if (schemaInfo.getSchemaId() >= this.schemaInfo.getSchemaId()) {
+ this.schemaInfo = schemaInfo;
+ }
+ this.schemaCaches.put(schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+}
diff --git a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
index 5e04e57a9..09467d31a 100644
--- a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
@@ -17,11 +17,18 @@
package org.apache.fluss.row;
+import org.apache.fluss.exception.SchemaChangeException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.types.DataTypes;
+
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link ProjectedRow}. */
class ProjectedRowTest {
@@ -159,4 +166,65 @@ class ProjectedRowTest {
assertThat(projectedRow.isNullAt(2)).isFalse();
assertThat(projectedRow.isNullAt(3)).isTrue();
}
+
+ @Test
+ void testProjectedRowsWithDifferentSchema() {
+ Schema schema =
+ Schema.newBuilder()
+ .column("a", DataTypes.BIGINT())
+ .column("b", DataTypes.STRING())
+ .column("c", DataTypes.DECIMAL(10, 1))
+ .build();
+ assertThatThrownBy(
+ () ->
+ ProjectedRow.from(
+ schema,
+ Schema.newBuilder().column("a", DataTypes.INT()).build()))
+ .isExactlyInstanceOf(SchemaChangeException.class)
+ .hasMessage(
+ "Expected datatype of column(id=0,name=a) is [INT], while the actual datatype is [BIGINT]");
+ assertThatThrownBy(
+ () ->
+ ProjectedRow.from(
+ schema,
+ Schema.newBuilder()
+ .fromColumns(
+ Collections.singletonList(
+ new Schema.Column(
+ "f",
+ DataTypes.DECIMAL(20, 1),
+ null,
+ 2)))
+ .build()))
+ .isExactlyInstanceOf(SchemaChangeException.class)
+ .hasMessage(
+ "Expected datatype of column(id=2,name=f) is [DECIMAL(20, 1)], while the actual datatype is [DECIMAL(10, 1)]");
+
+ Schema newSchema =
+ Schema.newBuilder()
+ .fromColumns(
+ Arrays.asList(
+ new Schema.Column("a", DataTypes.BIGINT(), null, 0),
+ // colum e share same id as the column c in the original
+ // schema.
+ new Schema.Column("e", DataTypes.DECIMAL(10, 1), null, 2),
+ // column b has different id the column b the original
+ // schema.
+ new Schema.Column("b", DataTypes.DECIMAL(10, 1), null, 3),
+ new Schema.Column("f", DataTypes.STRING(), null, 4)))
+ .build();
+
+ ProjectedRow projectedRow = ProjectedRow.from(schema, newSchema);
+ projectedRow.replaceRow(
+ GenericRow.of(
+ 1L,
+ BinaryString.fromString("value"),
+ Decimal.fromBigDecimal(new BigDecimal("13145678.1"), 10, 1)));
+ assertThat(projectedRow.getFieldCount()).isEqualTo(4);
+ assertThat(projectedRow.getLong(0)).isEqualTo(1L);
+ assertThat(projectedRow.getDecimal(1, 10, 1).decimalVal)
+ .isEqualTo(new BigDecimal("13145678.1"));
+ assertThat(projectedRow.isNullAt(2)).isTrue();
+ assertThat(projectedRow.isNullAt(3)).isTrue();
+ }
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java b/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
index 2236ebe21..b9d5c4a27 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
@@ -27,6 +27,7 @@ import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.DefaultLogRecordBatch;
@@ -81,6 +82,7 @@ import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK;
import static org.apache.fluss.record.TestData.DEFAULT_MAGIC;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.LogRecordBatchAssert.assertThatLogRecordBatch;
import static org.apache.fluss.utils.FlussPaths.remoteLogDir;
import static org.apache.fluss.utils.FlussPaths.remoteLogSegmentDir;
@@ -153,14 +155,13 @@ public class DataTestUtils {
public static MemoryLogRecords genMemoryLogRecordsByObject(byte magic, List<Object[]> objects)
throws Exception {
+ return genMemoryLogRecordsByObject(DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, magic, objects);
+ }
+
+ public static MemoryLogRecords genMemoryLogRecordsByObject(
+ RowType rowType, int schemaId, byte magic, List<Object[]> objects) throws Exception {
return createRecordsWithoutBaseLogOffset(
- DATA1_ROW_TYPE,
- DEFAULT_SCHEMA_ID,
- 0,
- System.currentTimeMillis(),
- magic,
- objects,
- LogFormat.ARROW);
+ rowType, schemaId, 0, System.currentTimeMillis(), magic, objects, LogFormat.ARROW);
}
public static MemoryLogRecords genMemoryLogRecordsByObject(List<Object[]> objects)
@@ -271,9 +272,14 @@ public class DataTestUtils {
* Generate a KvRecord batch from the values only, whose key will be extracted from the value.
*/
public static KvRecordBatch genKvRecordBatch(Object[]... values) throws Exception {
+ return genKvRecordBatch(DEFAULT_SCHEMA_ID, DATA1_ROW_TYPE, values);
+ }
+
+ public static KvRecordBatch genKvRecordBatch(
+ short schemaId, RowType rowType, Object[]... values) throws Exception {
KvRecordTestUtils.KvRecordBatchFactory kvRecordBatchFactory =
- KvRecordTestUtils.KvRecordBatchFactory.of(DEFAULT_SCHEMA_ID);
- return kvRecordBatchFactory.ofRecords(genKvRecords(values));
+ KvRecordTestUtils.KvRecordBatchFactory.of(schemaId);
+ return kvRecordBatchFactory.ofRecords(genKvRecords(rowType, values));
}
public static KvRecordBatch toKvRecordBatch(List<KvRecord> records) throws Exception {
@@ -344,9 +350,13 @@ public class DataTestUtils {
* the values.
*/
public static List<KvRecord> genKvRecords(Object[]... values) {
+ return genKvRecords(DATA1_ROW_TYPE, values);
+ }
+
+ public static List<KvRecord> genKvRecords(RowType rowType, Object[]... values) {
KvRecordTestUtils.PKBasedKvRecordFactory kvRecordFactory =
KvRecordTestUtils.PKBasedKvRecordFactory.of(
- DATA1_SCHEMA_PK.getRowType(), DATA1_SCHEMA_PK.getPrimaryKeyIndexes());
+ rowType, DATA1_SCHEMA_PK.getPrimaryKeyIndexes());
List<KvRecord> records = new ArrayList<>();
for (Object[] value : values) {
records.add(kvRecordFactory.ofRecord(value));
@@ -358,13 +368,23 @@ public class DataTestUtils {
return getKeyValuePairs(kvRecords.toArray(new KvRecord[0]));
}
+ public static List<Tuple2<byte[], byte[]>> getKeyValuePairs(
+ short schemaId, List<KvRecord> kvRecords) {
+ return getKeyValuePairs(schemaId, kvRecords.toArray(new KvRecord[0]));
+ }
+
public static List<Tuple2<byte[], byte[]>> getKeyValuePairs(KvRecord... kvRecords) {
+ return getKeyValuePairs(DEFAULT_SCHEMA_ID, kvRecords);
+ }
+
+ public static List<Tuple2<byte[], byte[]>> getKeyValuePairs(
+ short schemaId, KvRecord... kvRecords) {
List<Tuple2<byte[], byte[]>> keyValuePairs = new ArrayList<>();
for (KvRecord kvRecord : kvRecords) {
keyValuePairs.add(
Tuple2.of(
BytesUtils.toArray(kvRecord.getKey()),
- ValueEncoder.encodeValue(DEFAULT_SCHEMA_ID, kvRecord.getRow())));
+ ValueEncoder.encodeValue(schemaId, kvRecord.getRow())));
}
return keyValuePairs;
}
@@ -527,7 +547,10 @@ public class DataTestUtils {
}
public static void assertMemoryRecordsEquals(
- RowType rowType, LogRecords records, List<List<Object[]>> expected) {
+ RowType rowType,
+ SchemaGetter schemaGetter,
+ LogRecords records,
+ List<List<Object[]>> expected) {
List<List<Tuple2<ChangeType, Object[]>>> appendOnlyExpectedValue = new ArrayList<>();
for (List<Object[]> expectedRecord : expected) {
List<Tuple2<ChangeType, Object[]>> expectedFieldAndRowKind =
@@ -536,11 +559,13 @@ public class DataTestUtils {
.collect(Collectors.toList());
appendOnlyExpectedValue.add(expectedFieldAndRowKind);
}
- assertMemoryRecordsEqualsWithRowKind(rowType, records, appendOnlyExpectedValue);
+ assertMemoryRecordsEqualsWithRowKind(
+ rowType, schemaGetter, records, appendOnlyExpectedValue);
}
public static void assertMemoryRecordsEqualsWithRowKind(
- RowType rowType,
+ RowType expectRowType,
+ SchemaGetter schemaGetter,
LogRecords records,
List<List<Tuple2<ChangeType, Object[]>>> expected) {
Iterator<LogRecordBatch> iterator = records.batches().iterator();
@@ -548,12 +573,12 @@ public class DataTestUtils {
assertThat(iterator.hasNext()).isTrue();
LogRecordBatch batch = iterator.next();
try (LogRecordReadContext readContext =
- createArrowReadContext(rowType, DEFAULT_SCHEMA_ID);
+ createArrowReadContext(expectRowType, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> logIterator = batch.records(readContext)) {
for (Tuple2<ChangeType, Object[]> expectedFieldAndRowKind : expectedRecord) {
assertThat(logIterator.hasNext()).isTrue();
assertLogRecordsEqualsWithRowKind(
- rowType, logIterator.next(), expectedFieldAndRowKind);
+ expectRowType, logIterator.next(), expectedFieldAndRowKind);
}
assertThat(logIterator.hasNext()).isFalse();
}
@@ -563,9 +588,11 @@ public class DataTestUtils {
public static void assertLogRecordBatchEqualsWithRowKind(
RowType rowType,
+ SchemaGetter schemaGetter,
LogRecordBatch logRecordBatch,
List<Tuple2<ChangeType, Object[]>> expected) {
- try (LogRecordReadContext readContext = createArrowReadContext(rowType, DEFAULT_SCHEMA_ID);
+ try (LogRecordReadContext readContext =
+ createArrowReadContext(rowType, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> logIterator = logRecordBatch.records(readContext)) {
for (Tuple2<ChangeType, Object[]> expectedFieldAndRowKind : expected) {
assertThat(logIterator.hasNext()).isTrue();
@@ -615,17 +642,46 @@ public class DataTestUtils {
public static void assertLogRecordsEquals(
RowType rowType, LogRecords logRecords, List<Object[]> expectedValue) {
+ assertLogRecordsEquals(
+ DEFAULT_SCHEMA_ID, rowType, logRecords, expectedValue, TEST_SCHEMA_GETTER);
+ }
+
+ public static void assertLogRecordsEquals(
+ RowType rowType,
+ LogRecords logRecords,
+ List<Object[]> expectedValue,
+ SchemaGetter schemaGetter) {
+ assertLogRecordsEquals(DEFAULT_SCHEMA_ID, rowType, logRecords, expectedValue, schemaGetter);
+ }
+
+ public static void assertLogRecordsEquals(
+ int schemaId,
+ RowType rowType,
+ LogRecords logRecords,
+ List<Object[]> expectedValue,
+ SchemaGetter schemaGetter) {
List<Tuple2<ChangeType, Object[]>> expectedValueWithRowKind =
expectedValue.stream()
.map(val -> Tuple2.of(ChangeType.APPEND_ONLY, val))
.collect(Collectors.toList());
- assertLogRecordsEqualsWithRowKind(rowType, logRecords, expectedValueWithRowKind);
+ assertLogRecordsEqualsWithRowKind(
+ schemaId, rowType, logRecords, expectedValueWithRowKind, schemaGetter);
}
public static void assertLogRecordsEqualsWithRowKind(
RowType rowType,
LogRecords logRecords,
List<Tuple2<ChangeType, Object[]>> expectedValue) {
+ assertLogRecordsEqualsWithRowKind(
+ DEFAULT_SCHEMA_ID, rowType, logRecords, expectedValue, TEST_SCHEMA_GETTER);
+ }
+
+ public static void assertLogRecordsEqualsWithRowKind(
+ int schemaId,
+ RowType rowType,
+ LogRecords logRecords,
+ List<Tuple2<ChangeType, Object[]>> expectedValue,
+ SchemaGetter schemaGetter) {
DataType[] dataTypes = rowType.getChildren().toArray(new DataType[0]);
InternalRow.FieldGetter[] fieldGetter = new InternalRow.FieldGetter[dataTypes.length];
for (int i = 0; i < dataTypes.length; i++) {
@@ -634,7 +690,7 @@ public class DataTestUtils {
int i = 0;
try (LogRecordReadContext readContext =
- LogRecordReadContext.createArrowReadContext(rowType, DEFAULT_SCHEMA_ID)) {
+ LogRecordReadContext.createArrowReadContext(rowType, schemaId, schemaGetter)) {
for (LogRecordBatch batch : logRecords.batches()) {
try (CloseableIterator<LogRecord> iterator = batch.records(readContext)) {
while (iterator.hasNext()) {
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java
index ca6ee5d4f..320533ffb 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java
@@ -18,6 +18,7 @@
package org.apache.fluss.testutils;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
@@ -35,6 +36,7 @@ public class LogRecordBatchAssert extends AbstractAssert<LogRecordBatchAssert, L
private RowType rowType;
private LogFormat logFormat;
private boolean assertCheckSum = true;
+ private SchemaGetter schemaGetter;
/** Creates assertions for {@link LogRecordBatch}. */
public static LogRecordBatchAssert assertThatLogRecordBatch(LogRecordBatch actual) {
@@ -60,6 +62,11 @@ public class LogRecordBatchAssert extends AbstractAssert<LogRecordBatchAssert, L
return this;
}
+ public LogRecordBatchAssert withSchemaGetter(SchemaGetter schemaGetter) {
+ this.schemaGetter = schemaGetter;
+ return this;
+ }
+
public LogRecordBatchAssert isEqualTo(LogRecordBatch expected) {
if (rowType == null) {
throw new IllegalStateException(
@@ -113,9 +120,9 @@ public class LogRecordBatchAssert extends AbstractAssert<LogRecordBatchAssert, L
private LogRecordReadContext createReadContext(int schemaId) {
if (logFormat != null && logFormat == LogFormat.INDEXED) {
- return LogRecordReadContext.createIndexedReadContext(rowType, schemaId);
+ return LogRecordReadContext.createIndexedReadContext(rowType, schemaId, schemaGetter);
} else {
- return LogRecordReadContext.createArrowReadContext(rowType, schemaId);
+ return LogRecordReadContext.createArrowReadContext(rowType, schemaId, schemaGetter);
}
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
index b487b2138..28df40e34 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
@@ -17,8 +17,10 @@
package org.apache.fluss.testutils;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.types.RowType;
import org.assertj.core.api.AbstractAssert;
@@ -38,6 +40,7 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
private RowType rowType;
private boolean assertCheckSum = true;
+ private SchemaGetter schemaGetter = new TestingSchemaGetter();
private LogRecordsAssert(LogRecords actual) {
super(actual, LogRecordsAssert.class);
@@ -48,6 +51,11 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
return this;
}
+ public LogRecordsAssert withSchemaGetter(SchemaGetter schemaGetter) {
+ this.schemaGetter = schemaGetter;
+ return this;
+ }
+
public LogRecordsAssert assertCheckSum(boolean assertCheckSum) {
this.assertCheckSum = assertCheckSum;
return this;
@@ -69,6 +77,7 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
assertThatLogRecordBatch(actualIter.next())
.withSchema(rowType)
.assertCheckSum(assertCheckSum)
+ .withSchemaGetter(schemaGetter)
.isEqualTo(expectedNext);
}
assertThat(actualIter.hasNext()).isFalse();
diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java
index e34ff49b1..48fc2ac67 100644
--- a/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java
@@ -17,6 +17,7 @@
package org.apache.fluss.utils;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.row.BinaryString;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.ProjectedRow;
@@ -32,8 +33,15 @@ class ProjectionTest {
@Test
void testProjection() {
- Projection projection = Projection.of(new int[] {2, 0, 3});
- assertThat(projection.getProjectionInOrder()).isEqualTo(new int[] {0, 2, 3});
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.BIGINT())
+ .column("f2", DataTypes.STRING())
+ .column("f3", DataTypes.DOUBLE())
+ .build();
+ Projection projection = Projection.of(new int[] {2, 0, 3}, schema);
+ assertThat(projection.getProjectionIdInOrder()).isEqualTo(new int[] {0, 2, 3});
RowType rowType =
projection.projectInOrder(
@@ -59,7 +67,7 @@ class ProjectionTest {
DataTypes.FIELD("f3", DataTypes.DOUBLE())));
GenericRow row = GenericRow.of(0, 1L, BinaryString.fromString("2"), 3.0d);
- ProjectedRow p1 = ProjectedRow.from(projection.getProjectionInOrder());
+ ProjectedRow p1 = ProjectedRow.from(projection.getProjectionIdInOrder());
p1.replaceRow(row);
ProjectedRow p2 = ProjectedRow.from(projection.getReorderingIndexes());
p2.replaceRow(p1);
diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java
index b9924fac5..716e04347 100644
--- a/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java
@@ -21,6 +21,13 @@ import org.apache.fluss.metadata.Schema;
import org.apache.fluss.types.DataTypes;
import org.apache.fluss.types.IntType;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
/** Test for {@link ColumnJsonSerde}. */
public class ColumnJsonSerdeTest extends JsonSerdeTestBase<Schema.Column> {
protected ColumnJsonSerdeTest() {
@@ -29,19 +36,41 @@ public class ColumnJsonSerdeTest extends JsonSerdeTestBase<Schema.Column> {
@Override
protected Schema.Column[] createObjects() {
- Schema.Column[] columns = new Schema.Column[3];
+ Schema.Column[] columns = new Schema.Column[4];
columns[0] = new Schema.Column("a", DataTypes.STRING());
columns[1] = new Schema.Column("b", DataTypes.INT(), "hello b");
columns[2] = new Schema.Column("c", new IntType(false), "hello c");
+ columns[3] = new Schema.Column("d", new IntType(false), "hello c", (short) 2);
return columns;
}
@Override
protected String[] expectedJsons() {
return new String[] {
+ "{\"name\":\"a\",\"data_type\":{\"type\":\"STRING\"},\"id\":-1}",
+ "{\"name\":\"b\",\"data_type\":{\"type\":\"INTEGER\"},\"comment\":\"hello b\",\"id\":-1}",
+ "{\"name\":\"c\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"hello c\",\"id\":-1}",
+ "{\"name\":\"d\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"hello c\",\"id\":2}"
+ };
+ }
+
+ @Test
+ void testDeserializeCompatibility() throws IOException {
+ String[] jsonWithoutColumnId = {
"{\"name\":\"a\",\"data_type\":{\"type\":\"STRING\"}}",
"{\"name\":\"b\",\"data_type\":{\"type\":\"INTEGER\"},\"comment\":\"hello b\"}",
"{\"name\":\"c\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"hello c\"}"
};
+ Schema.Column[] columns = new Schema.Column[3];
+ columns[0] = new Schema.Column("a", DataTypes.STRING());
+ columns[1] = new Schema.Column("b", DataTypes.INT(), "hello b");
+ columns[2] = new Schema.Column("c", new IntType(false), "hello c");
+ for (int i = 0; i < jsonWithoutColumnId.length; i++) {
+ Schema.Column column =
+ JsonSerdeUtils.readValue(
+ jsonWithoutColumnId[i].getBytes(StandardCharsets.UTF_8),
+ ColumnJsonSerde.INSTANCE);
+ assertThat(column).isEqualTo(columns[i]);
+ }
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java
index 4b3ddc4c9..a7b6dca62 100644
--- a/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java
@@ -20,9 +20,14 @@ package org.apache.fluss.utils.json;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.types.DataTypes;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
+import static org.assertj.core.api.Assertions.assertThat;
+
/** Test for {@link SchemaJsonSerde}. */
public class SchemaJsonSerdeTest extends JsonSerdeTestBase<Schema> {
@@ -62,14 +67,12 @@ public class SchemaJsonSerdeTest extends JsonSerdeTestBase<Schema> {
.build();
static final String SCHEMA_JSON_0 =
- "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"CHAR\",\"nullable\":false,\"length\":10},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\",\"c\"]}";
-
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"a is first column\",\"id\":0},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\",\"id\":1},{\"name\":\"c\",\"data_type\":{\"type\":\"CHAR\",\"nullable\":false,\"length\":10},\"comment\":\"c is third column\",\"id\":2}],\"primary_key\":[\"a\",\"c\"],\"highest_field_id\":2}";
static final String SCHEMA_JSON_1 =
- "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false}},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\"]}";
-
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"id\":0},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\",\"id\":1},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\",\"id\":2}],\"primary_key\":[\"a\"],\"highest_field_id\":2}";
// shouldn't contain primary_key fields
static final String SCHEMA_JSON_3 =
- "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"BIGINT\"},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}]}";
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"BIGINT\"},\"comment\":\"a is first column\",\"id\":0},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\",\"id\":1},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\",\"id\":2}],\"highest_field_id\":2}";
SchemaJsonSerdeTest() {
super(SchemaJsonSerde.INSTANCE);
@@ -84,4 +87,31 @@ public class SchemaJsonSerdeTest extends JsonSerdeTestBase<Schema> {
protected String[] expectedJsons() {
return new String[] {SCHEMA_JSON_0, SCHEMA_JSON_1, SCHEMA_JSON_1, SCHEMA_JSON_3};
}
+
+ @Test
+ void testCompatibilityFromJsonLackOfColumnId() {
+ String[] jsons = jsonLackOfColumnId();
+ Schema[] expectedSchema = new Schema[] {SCHEMA_0, SCHEMA_1, SCHEMA_2, SCHEMA_3};
+ for (int i = 0; i < jsons.length; i++) {
+ assertThat(
+ JsonSerdeUtils.readValue(
+ jsons[i].getBytes(StandardCharsets.UTF_8),
+ SchemaJsonSerde.INSTANCE))
+ .isEqualTo(expectedSchema[i]);
+ }
+ }
+
+ protected String[] jsonLackOfColumnId() {
+ String oldSchemaJson1 =
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"CHAR\",\"nullable\":false,\"length\":10},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\",\"c\"]}";
+
+ String oldSchemaJson2 =
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false}},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\"]}";
+
+ // shouldn't contain primary_key fields
+ String oldSchemaJson3 =
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"BIGINT\"},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}]}";
+
+ return new String[] {oldSchemaJson1, oldSchemaJson2, oldSchemaJson2, oldSchemaJson3};
+ }
}
diff --git a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
index 12b05f186..bd1b6af81 100644
--- a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
+++ b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level = OFF
+rootLogger.level = INFO
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
index 003e910ea..314e4b40f 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
@@ -23,17 +23,34 @@ import org.apache.fluss.row.InternalArray;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
+import org.apache.fluss.types.RowType;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.TimestampData;
+import javax.annotation.Nullable;
+
+import java.util.List;
+
/** Wraps a Flink {@link RowData} as a Fluss {@link InternalRow}. */
public class FlinkAsFlussRow implements InternalRow {
private RowData flinkRow;
- public FlinkAsFlussRow() {}
+ /**
+ * Creates a FlinkAsFlussRow with a mapping from Fluss field index to Flink field index. If the
+ * indexMapping is null, flink and flink share same field name.
+ */
+ @Nullable private final int[] indexMapping;
+
+ public FlinkAsFlussRow() {
+ this(null);
+ }
+
+ public FlinkAsFlussRow(int[] indexMapping) {
+ this.indexMapping = indexMapping;
+ }
public FlinkAsFlussRow replace(RowData flinkRow) {
this.flinkRow = flinkRow;
@@ -42,62 +59,94 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public int getFieldCount() {
- return flinkRow.getArity();
+ if (indexMapping == null) {
+ return flinkRow.getArity();
+ } else {
+ return indexMapping.length;
+ }
}
@Override
public boolean isNullAt(int pos) {
- return flinkRow.isNullAt(pos);
+ // if no mapping, means that the field is not found in Flink, just ignore it.
+ if ((indexMapping != null && indexMapping[pos] == -1)) {
+ return true;
+ }
+
+ // If pos is larger than flinkRow.getArity(), it indicates that the Fluss table's data type
+ // is wider than the data provided by Flink.
+ // This often occurs when a schema change happens to the catalog table before job restart.
+ // Only appending columns at the end is compatible, just need to ignore the trailing
+ // columns.
+ // Other types of schema changes (e.g., dropping columns) would lead to incompatibility.
+ // Therefore, Fluss currently only supports appending columns, and here we simply ignore any
+ // extra columns.
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ if (flussPos >= flinkRow.getArity()) {
+ return true;
+ }
+
+ return flinkRow.isNullAt(flussPos);
}
@Override
public boolean getBoolean(int pos) {
- return flinkRow.getBoolean(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getBoolean(flussPos);
}
@Override
public byte getByte(int pos) {
- return flinkRow.getByte(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getByte(flussPos);
}
@Override
public short getShort(int pos) {
- return flinkRow.getShort(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getShort(flussPos);
}
@Override
public int getInt(int pos) {
- return flinkRow.getInt(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getInt(flussPos);
}
@Override
public long getLong(int pos) {
- return flinkRow.getLong(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getLong(flussPos);
}
@Override
public float getFloat(int pos) {
- return flinkRow.getFloat(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getFloat(flussPos);
}
@Override
public double getDouble(int pos) {
- return flinkRow.getDouble(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getDouble(flussPos);
}
@Override
public BinaryString getChar(int pos, int length) {
- return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
}
@Override
public BinaryString getString(int pos) {
- return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
}
@Override
public Decimal getDecimal(int pos, int precision, int scale) {
- return fromFlinkDecimal(flinkRow.getDecimal(pos, precision, scale));
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return fromFlinkDecimal(flinkRow.getDecimal(flussPos, precision, scale));
}
public static Decimal fromFlinkDecimal(DecimalData decimal) {
@@ -112,26 +161,43 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public TimestampNtz getTimestampNtz(int pos, int precision) {
- TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
return TimestampNtz.fromMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public TimestampLtz getTimestampLtz(int pos, int precision) {
- TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
return TimestampLtz.fromEpochMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public byte[] getBinary(int pos, int length) {
- return flinkRow.getBinary(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getBinary(flussPos);
}
@Override
public byte[] getBytes(int pos) {
- return flinkRow.getBinary(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getBinary(flussPos);
+ }
+
+ /** Converts a Flink {@link RowType} to a Fluss {@link RowType}, based on the field name. */
+ public static FlinkAsFlussRow from(
+ org.apache.flink.table.types.logical.RowType flinkRowType, RowType flussRowType) {
+ int[] indexMapping = new int[flussRowType.getFieldCount()];
+ List<String> fieldNames = flussRowType.getFieldNames();
+ for (int i = 0; i < flussRowType.getFieldCount(); i++) {
+ String fieldName = fieldNames.get(i);
+ int fieldIndex = flinkRowType.getFieldIndex(fieldName);
+ indexMapping[i] = fieldIndex;
+ }
+ return new FlinkAsFlussRow(indexMapping);
}
@Override
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java
index b615797a5..1b78292dd 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java
@@ -93,6 +93,8 @@ public class FlinkRowDataChannelComputer<InputT> implements ChannelComputer<Inpu
partitionGetter != null && numBucket % numChannels != 0;
try {
+ // no need to read real database, thus assume to deserialize the fluss row as same as
+ // flink table type.
this.serializationSchema.open(new SerializerInitContextImpl(flussRowType));
} catch (Exception e) {
throw new FlussRuntimeException(e);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
index 91cee80cd..0f3fff210 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
@@ -62,10 +62,17 @@ public interface FlussSerializationSchema<T> extends Serializable {
@PublicEvolving
interface InitializationContext {
/**
- * Returns the target row schema.
+ * Returns the Fluss physical row schema.
*
* @return The schema of the target row.
*/
RowType getRowSchema();
+
+ /**
+ * Returns the Flink table row schema.
+ *
+ * @return The schema of the Flink table row.
+ */
+ org.apache.flink.table.types.logical.RowType getTableRowType();
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
index adbfb7fb3..05e2921fc 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
@@ -63,7 +63,7 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
*/
@Override
public void open(InitializationContext context) throws Exception {
- this.converter = new FlinkAsFlussRow();
+ this.converter = FlinkAsFlussRow.from(context.getTableRowType(), context.getRowSchema());
}
/**
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
index b2cea1382..93b849e58 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
@@ -19,6 +19,7 @@ package org.apache.fluss.flink.sink.serializer;
import org.apache.fluss.types.RowType;
+import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/**
@@ -27,14 +28,26 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull;
*/
public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
- private final RowType rowSchema;
+ private final RowType flussRowSchema;
+ private final org.apache.flink.table.types.logical.RowType flinkRowType;
public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
+ this(rowSchema, toFlinkRowType(rowSchema));
+ }
+
+ public SerializerInitContextImpl(
+ RowType rowSchema, org.apache.flink.table.types.logical.RowType flinkRowType) {
+ this.flussRowSchema = checkNotNull(rowSchema, "flussRowSchema");
+ this.flinkRowType = checkNotNull(flinkRowType, "flinkRowType");
}
@Override
public RowType getRowSchema() {
- return rowSchema;
+ return flussRowSchema;
+ }
+
+ @Override
+ public org.apache.flink.table.types.logical.RowType getTableRowType() {
+ return flinkRowType;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
index 901a8999f..a4469fd32 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
@@ -110,11 +110,15 @@ public abstract class FlinkSinkWriter<InputT> implements SinkWriter<InputT> {
metricGroup, Collections.singleton(MetricNames.WRITER_SEND_LATENCY_MS));
connection = ConnectionFactory.createConnection(flussConfig, flinkMetricRegistry);
table = connection.getTable(tablePath);
- sanityCheck(table.getTableInfo());
+ LOG.info(
+ "Current Fluss Schema is {}, Table RowType is {}",
+ table.getTableInfo().getSchema(),
+ tableRowType);
+ // sanityCheck(table.getTableInfo());
try {
this.serializationSchema.open(
- new SerializerInitContextImpl(table.getTableInfo().getRowType()));
+ new SerializerInitContextImpl(table.getTableInfo().getRowType(), tableRowType));
} catch (Exception e) {
throw new FlussRuntimeException(e);
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
index bc15390fe..0bf9f187c 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
@@ -60,7 +60,6 @@ public class FlinkSource<OUT>
private final boolean hasPrimaryKey;
private final boolean isPartitioned;
private final RowType sourceOutputType;
- @Nullable private final int[] projectedFields;
protected final OffsetsInitializer offsetsInitializer;
protected final long scanPartitionDiscoveryIntervalMs;
private final boolean streaming;
@@ -74,7 +73,6 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -86,7 +84,6 @@ public class FlinkSource<OUT>
hasPrimaryKey,
isPartitioned,
sourceOutputType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
@@ -101,7 +98,6 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -113,7 +109,6 @@ public class FlinkSource<OUT>
this.hasPrimaryKey = hasPrimaryKey;
this.isPartitioned = isPartitioned;
this.sourceOutputType = sourceOutputType;
- this.projectedFields = projectedFields;
this.offsetsInitializer = offsetsInitializer;
this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs;
this.deserializationSchema = deserializationSchema;
@@ -187,6 +182,7 @@ public class FlinkSource<OUT>
context.getUserCodeClassLoader(),
sourceOutputType));
FlinkRecordEmitter<OUT> recordEmitter = new FlinkRecordEmitter<>(deserializationSchema);
+ // recall to projectedFields
return new FlinkSourceReader<>(
elementsQueue,
@@ -194,7 +190,6 @@ public class FlinkSource<OUT>
tablePath,
sourceOutputType,
context,
- projectedFields,
flinkSourceReaderMetrics,
recordEmitter,
lakeSource);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
index ef4b63812..b110e9f88 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
@@ -325,7 +325,6 @@ public class FlinkTableSource
hasPrimaryKey(),
isPartitioned(),
flussRowType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
new RowDataDeserializationSchema(),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
index 427741834..5f0b2ff8c 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
@@ -24,8 +24,6 @@ import org.apache.fluss.flink.source.enumerator.initializer.OffsetsInitializer;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.types.RowType;
-import javax.annotation.Nullable;
-
/**
* A Flink DataStream source implementation for reading data from Fluss tables.
*
@@ -63,7 +61,6 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -75,7 +72,6 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
index f16639753..4891ab256 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
@@ -296,7 +296,6 @@ public class FlussSourceBuilder<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java
index 7c6405004..31238cf72 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java
@@ -19,7 +19,6 @@ package org.apache.fluss.flink.source.deserializer;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.client.table.scanner.ScanRecord;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.record.LogRecord;
import org.apache.fluss.types.RowType;
@@ -28,6 +27,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
+
/**
* A deserialization schema that converts {@link LogRecord} objects to Flink's {@link RowData}
* format.
@@ -89,6 +90,6 @@ public class RowDataDeserializationSchema implements FlussDeserializationSchema<
/** Returns the TypeInformation for the produced {@link RowData} type. */
@Override
public TypeInformation<RowData> getProducedType(RowType rowSchema) {
- return InternalTypeInfo.of(FlinkConversions.toFlinkRowType(rowSchema));
+ return InternalTypeInfo.of(toFlinkRowType(rowSchema));
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
index e1906712a..6e4647f80 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
@@ -27,12 +27,10 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.TableNotExistException;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
import org.apache.fluss.flink.source.lookup.LookupNormalizer.RemainingFilter;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.utils.ExceptionUtils;
import org.apache.flink.table.data.RowData;
@@ -87,6 +85,10 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
+ org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
+ LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
+ // Currently, only primary key and prefix lookup are supported. These keys must be primary
+ // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
@@ -95,8 +97,8 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
} else {
outputRowType = FlinkUtils.projectRowType(flinkRowType, projection);
}
- flussRowToFlinkRowConverter =
- new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
+
+ flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -162,7 +164,7 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
List<RowData> projectedRow = new ArrayList<>();
for (InternalRow row : lookupResult) {
if (row != null) {
- RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
+ RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRow.add(flinkRow);
}
@@ -171,14 +173,6 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
resultFuture.complete(projectedRow);
}
- private InternalRow maybeProject(InternalRow row) {
- if (projection == null) {
- return row;
- }
- // should not reuse objects for async operations
- return ProjectedRow.from(projection).replaceRow(row);
- }
-
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
index 79418f9b2..e9a5ad915 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
@@ -25,7 +25,6 @@ import org.apache.fluss.client.lookup.Lookuper;
import org.apache.fluss.client.table.Table;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
@@ -83,6 +82,10 @@ public class FlinkLookupFunction extends LookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
+ org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
+ LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
+ // Currently, only primary key and prefix lookup are supported. These keys must be primary
+ // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
@@ -94,8 +97,7 @@ public class FlinkLookupFunction extends LookupFunction {
// reuse the projected row
projectedRow = ProjectedRow.from(projection);
}
- flussRowToFlinkRowConverter =
- new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
+ flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -131,8 +133,7 @@ public class FlinkLookupFunction extends LookupFunction {
List<RowData> projectedRows = new ArrayList<>();
for (InternalRow row : lookupRows) {
if (row != null) {
- RowData flinkRow =
- flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
+ RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRows.add(flinkRow);
}
@@ -145,13 +146,6 @@ public class FlinkLookupFunction extends LookupFunction {
}
}
- private InternalRow maybeProject(InternalRow row) {
- if (projectedRow == null) {
- return row;
- }
- return projectedRow.replaceRow(row);
- }
-
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
index 6364532bb..639d52513 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
@@ -40,8 +40,6 @@ import org.apache.flink.api.connector.source.SourceReaderContext;
import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
-import javax.annotation.Nullable;
-
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
@@ -57,7 +55,6 @@ public class FlinkSourceReader<OUT>
TablePath tablePath,
RowType sourceOutputType,
SourceReaderContext context,
- @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
FlinkRecordEmitter<OUT> recordEmitter,
LakeSource<LakeSplit> lakeSource) {
@@ -70,7 +67,6 @@ public class FlinkSourceReader<OUT>
flussConfig,
tablePath,
sourceOutputType,
- projectedFields,
flinkSourceReaderMetrics,
lakeSource),
(ignore) -> {}),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
index fa0cb9e1b..10997e91e 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
@@ -17,6 +17,7 @@
package org.apache.fluss.flink.source.reader;
+import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.client.Connection;
import org.apache.fluss.client.ConnectionFactory;
import org.apache.fluss.client.table.Table;
@@ -57,7 +58,6 @@ import java.io.IOException;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -119,7 +119,6 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
Configuration flussConf,
TablePath tablePath,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
@Nullable LakeSource<LakeSplit> lakeSource) {
this.flinkMetricRegistry =
@@ -129,13 +128,17 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
this.sourceOutputType = sourceOutputType;
this.boundedSplits = new ArrayDeque<>();
this.subscribedBuckets = new HashMap<>();
- this.projectedFields = projectedFields;
this.flinkSourceReaderMetrics = flinkSourceReaderMetrics;
- sanityCheck(table.getTableInfo().getRowType(), projectedFields);
+ this.projectedFields =
+ reCalculateProjectedFields(sourceOutputType, table.getTableInfo().getRowType());
this.logScanner = table.newScan().project(projectedFields).createLogScanner();
this.stoppingOffsets = new HashMap<>();
this.emptyLogSplits = new HashSet<>();
this.lakeSource = lakeSource;
+ LOG.info(
+ "fluss table schema: {}, flink table output type:{}",
+ table.getTableInfo().getSchema(),
+ sourceOutputType);
}
@Override
@@ -213,7 +216,8 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
LakeSnapshotAndFlussLogSplit lakeSnapshotAndFlussLogSplit =
(LakeSnapshotAndFlussLogSplit) sourceSplitBase;
if (lakeSnapshotAndFlussLogSplit.isStreaming()) {
- // is streaming split which has no stopping offset, we need also subscribe
+ // is streaming split which has no stopping offset, we need also
+ // subscribe
// change log
subscribeLog(
lakeSnapshotAndFlussLogSplit,
@@ -560,35 +564,33 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
flinkMetricRegistry.close();
}
- private void sanityCheck(RowType flussTableRowType, @Nullable int[] projectedFields) {
- RowType tableRowType =
- projectedFields != null
- ? flussTableRowType.project(projectedFields)
- : flussTableRowType;
- if (!sourceOutputType.copy(false).equals(tableRowType.copy(false))) {
- // The default nullability of Flink row type and Fluss row type might be not the same,
- // thus we need to compare the row type without nullability here.
-
- final String flussSchemaMsg;
- if (projectedFields == null) {
- flussSchemaMsg = "\nFluss table schema: " + tableRowType;
- } else {
- flussSchemaMsg =
- "\nFluss table schema: "
- + tableRowType
- + " (projection "
- + Arrays.toString(projectedFields)
- + ")";
+ /**
+ * The projected fields for the fluss table from the source output types. Mapping based on
+ * column name rather thn column id.
+ */
+ private static int[] reCalculateProjectedFields(
+ RowType sourceOutputType, RowType flussRowType) {
+ if (sourceOutputType.copy(false).equals(flussRowType.copy(false))) {
+ return null;
+ }
+
+ List<String> fieldNames = sourceOutputType.getFieldNames();
+ int[] projectedFlussFields = new int[fieldNames.size()];
+ for (int i = 0; i < fieldNames.size(); i++) {
+ int fieldIndex = flussRowType.getFieldIndex(fieldNames.get(i));
+ if (fieldIndex == -1) {
+ throw new ValidationException(
+ String.format(
+ "The field %s is not found in the fluss table.",
+ fieldNames.get(i)));
}
- // Throw exception if the schema is the not same, this should rarely happen because we
- // only allow fluss tables derived from fluss catalog. But this can happen if an ALTER
- // TABLE command executed on the fluss table, after the job is submitted but before the
- // SinkFunction is opened.
- throw new ValidationException(
- "The Flink query schema is not matched to Fluss table schema. "
- + "\nFlink query schema: "
- + sourceOutputType
- + flussSchemaMsg);
+ projectedFlussFields[i] = fieldIndex;
}
+ return projectedFlussFields;
+ }
+
+ @VisibleForTesting
+ public int[] getProjectedFields() {
+ return projectedFields;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
index 3256b3f52..74375e24c 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
@@ -58,12 +58,14 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
import static org.apache.flink.table.utils.EncodingUtils.decodeBase64ToBytes;
import static org.apache.flink.table.utils.EncodingUtils.encodeBytesToBase64;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
import static org.apache.fluss.config.FlussConfigUtils.isTableStorageConfig;
import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_KEY;
import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_NUMBER;
@@ -84,6 +86,18 @@ public class FlinkConversions {
private FlinkConversions() {}
+ public static int[] toFlinkRowTypeIndexMapping(
+ RowType sourceFlussRowType,
+ org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
+ int[] indexMapping = new int[targetFlinkRowType.getFieldCount()];
+ for (int i = 0; i < targetFlinkRowType.getFieldCount(); i++) {
+ String fieldName = targetFlinkRowType.getFieldNames().get(i);
+ int index = sourceFlussRowType.getFieldIndex(fieldName);
+ indexMapping[i] = index;
+ }
+ return indexMapping;
+ }
+
/** Convert Fluss's type to Flink's type. */
@VisibleForTesting
public static org.apache.flink.table.types.DataType toFlinkType(DataType flussDataType) {
@@ -394,6 +408,44 @@ public class FlinkConversions {
return Collections.singletonList(
convertSetOption(
(org.apache.flink.table.catalog.TableChange.SetOption) tableChange));
+ } else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.AddColumn) {
+ org.apache.flink.table.catalog.TableChange.AddColumn addColumn =
+ (org.apache.flink.table.catalog.TableChange.AddColumn) tableChange;
+ Column column = addColumn.getColumn();
+ return Collections.singletonList(
+ TableChange.addColumn(
+ column.getName(),
+ toFlussType(column.getDataType()),
+ column.getComment().orElse(null),
+ toFlussColumnPosition(addColumn.getPosition())));
+ } else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.DropColumn) {
+ return Collections.singletonList(
+ TableChange.dropColumn(
+ ((org.apache.flink.table.catalog.TableChange.DropColumn) tableChange)
+ .getColumnName()));
+ } else if (tableChange
+ instanceof org.apache.flink.table.catalog.TableChange.ModifyColumnName) {
+ org.apache.flink.table.catalog.TableChange.ModifyColumnName renameColumn =
+ (org.apache.flink.table.catalog.TableChange.ModifyColumnName) tableChange;
+ return Collections.singletonList(
+ TableChange.renameColumn(
+ renameColumn.getOldColumn().getName(),
+ renameColumn.getNewColumnName()));
+ } else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.ModifyColumn) {
+ org.apache.flink.table.catalog.TableChange.ModifyColumn modifyColumn =
+ (org.apache.flink.table.catalog.TableChange.ModifyColumn) tableChange;
+ checkState(
+ Objects.equals(
+ modifyColumn.getNewColumn().getName(),
+ modifyColumn.getOldColumn().getName()),
+ "Can only modify columns with the same name.");
+ Column newColumn = modifyColumn.getNewColumn();
+ return Collections.singletonList(
+ TableChange.modifyColumn(
+ newColumn.getName(),
+ toFlussType(newColumn.getDataType()),
+ newColumn.getComment().orElse(null),
+ toFlussColumnPosition(modifyColumn.getNewPosition())));
} else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.ResetOption) {
return Collections.singletonList(
convertResetOption(
@@ -408,6 +460,21 @@ public class FlinkConversions {
}
}
+ private static TableChange.ColumnPosition toFlussColumnPosition(
+ org.apache.flink.table.catalog.TableChange.ColumnPosition columnPosition) {
+ if (columnPosition == null) {
+ return TableChange.ColumnPosition.last();
+ } else if (columnPosition instanceof org.apache.flink.table.catalog.TableChange.First) {
+ return TableChange.ColumnPosition.first();
+ } else if (columnPosition instanceof org.apache.flink.table.catalog.TableChange.After) {
+ return TableChange.ColumnPosition.after(
+ ((org.apache.flink.table.catalog.TableChange.After) columnPosition).column());
+ } else {
+ throw new UnsupportedOperationException(
+ String.format("Unsupported column position: %s.", columnPosition));
+ }
+ }
+
private static TableChange.SetOption convertSetOption(
org.apache.flink.table.catalog.TableChange.SetOption flinkSetOption) {
return TableChange.set(flinkSetOption.getKey(), flinkSetOption.getValue());
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
index e52a83a82..4dc5287b0 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
@@ -39,18 +39,32 @@ import org.apache.flink.types.RowKind;
import java.io.Serializable;
import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowKind;
+import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
/** A converter to convert Fluss's {@link InternalRow} to Flink's {@link RowData}. */
public class FlussRowToFlinkRowConverter {
private final FlussDeserializationConverter[] toFlinkFieldConverters;
private final InternalRow.FieldGetter[] flussFieldGetters;
+ private final int[] indexMapping;
+ // todo: remove
public FlussRowToFlinkRowConverter(RowType rowType) {
- this.toFlinkFieldConverters = new FlussDeserializationConverter[rowType.getFieldCount()];
- this.flussFieldGetters = new InternalRow.FieldGetter[rowType.getFieldCount()];
- for (int i = 0; i < rowType.getFieldCount(); i++) {
- toFlinkFieldConverters[i] = createNullableInternalConverter(rowType.getTypeAt(i));
- flussFieldGetters[i] = InternalRow.createFieldGetter(rowType.getTypeAt(i), i);
+ this(rowType, toFlinkRowType(rowType));
+ }
+
+ public FlussRowToFlinkRowConverter(
+ RowType sourceFlussRowType,
+ org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
+ this.indexMapping =
+ FlinkConversions.toFlinkRowTypeIndexMapping(sourceFlussRowType, targetFlinkRowType);
+ this.toFlinkFieldConverters =
+ new FlussDeserializationConverter[sourceFlussRowType.getFieldCount()];
+ this.flussFieldGetters = new InternalRow.FieldGetter[sourceFlussRowType.getFieldCount()];
+ for (int i = 0; i < sourceFlussRowType.getFieldCount(); i++) {
+ toFlinkFieldConverters[i] =
+ createNullableInternalConverter(sourceFlussRowType.getTypeAt(i));
+ flussFieldGetters[i] =
+ InternalRow.createFieldGetter(sourceFlussRowType.getTypeAt(i), i);
}
}
@@ -63,11 +77,18 @@ public class FlussRowToFlinkRowConverter {
}
private RowData toFlinkRowData(InternalRow flussRow, RowKind rowKind) {
- GenericRowData genericRowData = new GenericRowData(toFlinkFieldConverters.length);
+ int targetLength = indexMapping.length;
+ GenericRowData genericRowData = new GenericRowData(targetLength);
genericRowData.setRowKind(rowKind);
- for (int i = 0; i < toFlinkFieldConverters.length; i++) {
- Object flussField = flussFieldGetters[i].getFieldOrNull(flussRow);
- genericRowData.setField(i, toFlinkFieldConverters[i].deserialize(flussField));
+ for (int i = 0; i < targetLength; i++) {
+ int flussIndex = indexMapping[i];
+ if (flussIndex == -1) {
+ genericRowData.setField(i, null);
+ } else {
+ Object flussField = flussFieldGetters[flussIndex].getFieldOrNull(flussRow);
+ genericRowData.setField(
+ i, toFlinkFieldConverters[flussIndex].deserialize(flussField));
+ }
}
return genericRowData;
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java
index 88e16e996..570d7121a 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java
@@ -203,7 +203,7 @@ abstract class FlinkCatalogITCase {
}
@Test
- void testAlterTable() throws Exception {
+ void testAlterTableConfig() throws Exception {
String ddl =
"create table test_alter_table_append_only ("
+ "a string, "
@@ -286,6 +286,25 @@ abstract class FlinkCatalogITCase {
"Property 'paimon.file.format' is not supported to alter which is for datalake table.");
}
+ @Test
+ void testAlterTableSchema() throws Exception {
+ ObjectPath objectPath = new ObjectPath(DEFAULT_DB, "append_only_table");
+ tEnv.executeSql(
+ "create table append_only_table(a int, b STRING) with ('bucket.num' = '10')")
+ .await();
+ tEnv.executeSql("alter table append_only_table add c int").await();
+ CatalogTable table = (CatalogTable) catalog.getTable(objectPath);
+
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ Schema expectedSchema =
+ schemaBuilder
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.STRING())
+ .column("c", DataTypes.INT())
+ .build();
+ assertThat(table.getUnresolvedSchema()).isEqualTo(expectedSchema);
+ }
+
@Test
void testCreateUnSupportedTable() {
// test invalid property
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
index 73ba044d8..1f147751a 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
+import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.offset;
@@ -59,7 +60,7 @@ class FlinkAsFlussRowTest {
TimestampData.fromEpochMillis(1672531200000L, 3),
new byte[] {1, 2, 3},
null);
- row = new FlinkAsFlussRow().replace(flinkRow);
+ row = new FlinkAsFlussRow(IntStream.range(0, 14).toArray()).replace(flinkRow);
}
@Test
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java
index 0ea8ed94f..a22316d70 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java
@@ -165,6 +165,52 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
assertResultsIgnoreOrder(rowIter, expectedRows, true);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testAppendLogDuringAddColumn(boolean compressed) throws Exception {
+ String compressedProperties =
+ compressed
+ ? ",'table.log.format' = 'arrow', 'table.log.arrow.compression.type' = 'zstd'"
+ : "";
+ tEnv.executeSql(
+ "create table sink_test (a int not null, b bigint, c string) with "
+ + "('bucket.num' = '3'"
+ + compressedProperties
+ + ")");
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (1, 3501, 'Tim'), "
+ + "(2, 3502, 'Fabian'), "
+ + "(3, 3503, 'coco') ")
+ .await();
+
+ CloseableIterator<Row> rowIter = tEnv.executeSql("select * from sink_test").collect();
+ // add new column
+ tEnv.executeSql("alter table sink_test add add_column int").await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, "sink_test"), 2);
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (4, 3504, 'jerry', 4), "
+ + "(5, 3505, 'piggy', 5), "
+ + "(6, 3506, 'stave', 6)")
+ .await();
+ List<String> expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim]", "+I[2, 3502, Fabian]",
+ "+I[3, 3503, coco]", "+I[4, 3504, jerry]",
+ "+I[5, 3505, piggy]", "+I[6, 3506, stave]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+
+ // read with new schema.
+ rowIter = tEnv.executeSql("select * from sink_test").collect();
+ expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim, null]", "+I[2, 3502, Fabian, null]",
+ "+I[3, 3503, coco, null]", "+I[4, 3504, jerry, 4]",
+ "+I[5, 3505, piggy, 5]", "+I[6, 3506, stave, 6]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+ }
+
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testAppendLogWithBucketKey(boolean sinkBucketShuffle) throws Exception {
@@ -356,6 +402,45 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
assertResultsIgnoreOrder(rowIter, expectedRows, true);
}
+ @Test
+ void testPutDuringAddColumn() throws Exception {
+ tEnv.executeSql(
+ "create table sink_test (a int not null primary key not enforced, b bigint, c string)");
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (1, 3501, 'Tim'), "
+ + "(2, 3502, 'Fabian'), "
+ + "(3, 3503, 'coco') ")
+ .await();
+
+ CloseableIterator<Row> rowIter = tEnv.executeSql("select * from sink_test").collect();
+ // add new column
+ tEnv.executeSql("alter table sink_test add add_column int").await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, "sink_test"), 2);
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (4, 3504, 'jerry', 4), "
+ + "(5, 3505, 'piggy', 5), "
+ + "(6, 3506, 'stave', 6)")
+ .await();
+ List<String> expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim]", "+I[2, 3502, Fabian]",
+ "+I[3, 3503, coco]", "+I[4, 3504, jerry]",
+ "+I[5, 3505, piggy]", "+I[6, 3506, stave]");
+ // read with old schema
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+
+ // read with new schema.
+ rowIter = tEnv.executeSql("select * from sink_test").collect();
+ expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim, null]", "+I[2, 3502, Fabian, null]",
+ "+I[3, 3503, coco, null]", "+I[4, 3504, jerry, 4]",
+ "+I[5, 3505, piggy, 5]", "+I[6, 3506, stave, 6]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+ }
+
@Test
void testPartialUpsert() throws Exception {
tEnv.executeSql(
@@ -407,6 +492,58 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
assertResultsIgnoreOrder(rowIter, expectedRows, true);
}
+ @Test
+ void testPartialUpsertDuringAddColumn() throws Exception {
+ tEnv.executeSql(
+ "create table sink_test (a int not null primary key not enforced, b bigint, c string) with('bucket.num' = '3')");
+
+ // partial insert
+ tEnv.executeSql("INSERT INTO sink_test(a, b) VALUES (1, 111), (2, 222)").await();
+ tEnv.executeSql("INSERT INTO sink_test(c, a) VALUES ('c1', 1), ('c2', 2)").await();
+
+ CloseableIterator<Row> rowIter = tEnv.executeSql("select * from sink_test").collect();
+ // add new column
+ tEnv.executeSql("alter table sink_test add add_column string").await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, "sink_test"), 2);
+ tEnv.executeSql(
+ "INSERT INTO sink_test(add_column, a ) VALUES ('new_value', 1), ('new_value', 2)")
+ .await();
+
+ // read with old schema
+ List<String> expectedRows =
+ Arrays.asList(
+ "+I[1, 111, null]",
+ "+I[2, 222, null]",
+ "-U[1, 111, null]",
+ "+U[1, 111, c1]",
+ "-U[2, 222, null]",
+ "+U[2, 222, c2]",
+ "-U[1, 111, c1]",
+ "+U[1, 111, c1]",
+ "-U[2, 222, c2]",
+ "+U[2, 222, c2]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+
+ // read with new schema
+ CloseableIterator<Row> newSchemaRowIter =
+ tEnv.executeSql(
+ "select * from sink_test /*+ OPTIONS('scan.startup.mode' = 'earliest') */ ")
+ .collect();
+ expectedRows =
+ Arrays.asList(
+ "+I[1, 111, null, null]",
+ "+I[2, 222, null, null]",
+ "-U[1, 111, null, null]",
+ "+U[1, 111, c1, null]",
+ "-U[2, 222, null, null]",
+ "+U[2, 222, c2, null]",
+ "-U[1, 111, c1, null]",
+ "+U[1, 111, c1, new_value]",
+ "-U[2, 222, c2, null]",
+ "+U[2, 222, c2, new_value]");
+ assertResultsIgnoreOrder(newSchemaRowIter, expectedRows, true);
+ }
+
@Test
void testFirstRowMergeEngine() throws Exception {
tEnv.executeSql(
@@ -676,6 +813,21 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
"-U[4, 3504, Seattle]",
"+U[4, 3504, New York]");
assertResultsIgnoreOrder(changelogIter, expectedRows, true);
+
+ // test schema evolution.
+ tBatchEnv
+ .executeSql(String.format("alter table %s add new_added_column int", tableName))
+ .await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, tableName), 2);
+ tBatchEnv
+ .executeSql("UPDATE " + tableName + " SET new_added_column = 2 WHERE a = 4")
+ .await();
+ CloseableIterator<Row> row5 =
+ tBatchEnv
+ .executeSql(String.format("select * from %s WHERE a = 4", tableName))
+ .collect();
+ expected = Collections.singletonList("+I[4, 3504, New York, 2]");
+ assertResultsIgnoreOrder(row5, expected, true);
}
@Test
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java
index f65da98e6..9ac81ba6d 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java
@@ -24,13 +24,14 @@ import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.NetworkException;
import org.apache.fluss.flink.sink.serializer.RowDataSerializationSchema;
-import org.apache.fluss.flink.sink.serializer.SerializerInitContextImpl;
import org.apache.fluss.flink.utils.FlinkTestBase;
import org.apache.fluss.metadata.DatabaseDescriptor;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.server.testutils.FlussClusterExtension;
+import org.apache.fluss.types.DataTypes;
import org.apache.flink.api.common.operators.MailboxExecutor;
import org.apache.flink.api.connector.sink2.SinkWriter;
@@ -50,9 +51,9 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import java.util.Collections;
import java.util.function.BiConsumer;
-import static org.apache.fluss.flink.utils.FlinkConversions.toFlussRowType;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -106,6 +107,52 @@ public class FlinkSinkWriterTest extends FlinkTestBase {
flinkSinkWriter.close();
}
+ @Test
+ void testFlussSchemaChange() throws Exception {
+ admin.createDatabase(
+ DEFAULT_SINK_TABLE_PATH.getDatabaseName(), DatabaseDescriptor.EMPTY, true);
+ createTable(DEFAULT_SINK_TABLE_PATH, TABLE_DESCRIPTOR);
+ admin.alterTable(
+ DEFAULT_SINK_TABLE_PATH,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+
+ Configuration clientConfig = FLUSS_CLUSTER_EXTENSION.getClientConfig();
+
+ MockWriterInitContext mockWriterInitContext =
+ new MockWriterInitContext(new InterceptingOperatorMetricGroup());
+ try (FlinkSinkWriter<RowData> writer =
+ createSinkWriter(clientConfig, mockWriterInitContext.getMailboxExecutor())) {
+ writer.initialize(mockWriterInitContext.metricGroup());
+ // case1: write data which is matched flink schema.
+ writer.write(
+ GenericRowData.of(1, StringData.fromString("a")), new MockSinkWriterContext());
+ writer.flush(false);
+
+ // case2: write data which lacks the last column of flink schema
+ writer.write(GenericRowData.of(1), new MockSinkWriterContext());
+ writer.flush(false);
+
+ // case3: write data which has reorder column of flink schema
+ assertThatThrownBy(
+ () ->
+ writer.write(
+ GenericRowData.of(StringData.fromString("a"), 1),
+ new MockSinkWriterContext()))
+ .rootCause()
+ .isExactlyInstanceOf(ClassCastException.class)
+ .hasMessageContaining(
+ "class org.apache.flink.table.data.binary.BinaryStringData cannot be cast to class java.lang.Integer");
+ writer.flush(false);
+ }
+ }
+
@Test
void testWriteExceptionWhenFlussUnavailable() throws Exception {
testExceptionWhenFlussUnavailable(
@@ -210,7 +257,6 @@ public class FlinkSinkWriterTest extends FlinkTestBase {
new String[] {"id", "name"});
RowDataSerializationSchema serializationSchema =
new RowDataSerializationSchema(true, false);
- serializationSchema.open(new SerializerInitContextImpl(toFlussRowType(tableRowType)));
return new AppendSinkWriter<>(
DEFAULT_SINK_TABLE_PATH,
configuration,
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java
index f235afc56..8a5f3b81d 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java
@@ -37,7 +37,9 @@ import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@@ -125,7 +127,7 @@ public class FlussDeserializationSchemaTest {
@Test
public void testJsonStringDeserialize() throws Exception {
- List<DataField> fields =
+ List<DataField> sourceFields =
Arrays.asList(
new DataField("char", DataTypes.CHAR(64)),
new DataField("string", DataTypes.STRING()),
@@ -145,7 +147,12 @@ public class FlussDeserializationSchemaTest {
new DataField("timestampWithLocalTimeZone", DataTypes.TIMESTAMP_LTZ()),
new DataField("nullVal", DataTypes.STRING()));
- RowType rowType = new RowType(fields);
+ RowType sourceRowType = new RowType(sourceFields);
+
+ List<DataField> targetFields = new ArrayList<>(sourceFields);
+ targetFields.add(new DataField("new_column", DataTypes.STRING()));
+ Collections.reverse(targetFields);
+ RowType targetRowType = new RowType(targetFields);
// Mon Apr 21 2025 10:00:00 GMT+0000
long testTimestampInSeconds = 1745229600;
@@ -174,7 +181,7 @@ public class FlussDeserializationSchemaTest {
// Create deserializer
JsonStringDeserializationSchema deserializer = new JsonStringDeserializationSchema();
// Test deserialization
- deserializer.open(new DeserializerInitContextImpl(null, null, rowType));
+ deserializer.open(new DeserializerInitContextImpl(null, null, sourceRowType));
String result = deserializer.deserialize(scanRecord);
String rowJson =
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
index 297c6a16d..596780302 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
@@ -21,13 +21,19 @@ import org.apache.fluss.client.table.Table;
import org.apache.fluss.client.table.writer.UpsertWriter;
import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkTestBase;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.types.DataTypes;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.functions.AsyncLookupFunction;
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.util.Collector;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.util.ArrayList;
import java.util.Collection;
@@ -44,10 +50,11 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link FlinkLookupFunction} and {@link FlinkAsyncLookupFunction}. */
class FlinkLookupFunctionTest extends FlinkTestBase {
- @Test
- void testSyncLookupEval() throws Exception {
- TablePath tablePath = TablePath.of(DEFAULT_DB, "sync-lookup-table");
- prepareData(tablePath, 3);
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testSyncLookupEval(boolean schemaNotMatch) throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "sync-lookup-table-" + schemaNotMatch);
+ prepareData(tablePath, 3, schemaNotMatch);
RowType flinkRowType =
FlinkConversions.toFlinkRowType(DEFAULT_PK_TABLE_SCHEMA.getRowType());
@@ -80,11 +87,12 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
lookupFunction.close();
}
- @Test
- void testAsyncLookupEval() throws Exception {
- TablePath tablePath = TablePath.of(DEFAULT_DB, "async-lookup-table");
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testAsyncLookupEval(boolean schemaNotMatch) throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "async-lookup-table-" + schemaNotMatch);
int rows = 3;
- prepareData(tablePath, rows);
+ prepareData(tablePath, rows, schemaNotMatch);
RowType flinkRowType =
FlinkConversions.toFlinkRowType(DEFAULT_PK_TABLE_SCHEMA.getRowType());
@@ -133,14 +141,107 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
"4: null");
}
- private void prepareData(TablePath tablePath, int rows) throws Exception {
- createTable(tablePath, DEFAULT_PK_TABLE_DESCRIPTOR);
+ @Test
+ void testSchemaChange() throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "sync-add-column");
+ prepareData(tablePath, 3, false);
+
+ RowType flinkRowType =
+ FlinkConversions.toFlinkRowType(DEFAULT_PK_TABLE_SCHEMA.getRowType());
+ FlinkLookupFunction lookupFunction =
+ new FlinkLookupFunction(
+ clientConf,
+ tablePath,
+ flinkRowType,
+ createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ null);
+
+ ListOutputCollector collector = new ListOutputCollector();
+ lookupFunction.setCollector(collector);
+ lookupFunction.open(null);
+
+ // alter table after lookup function is created.
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "new_column",
+ DataTypes.INT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(tablePath, 2);
+
+ try (Table table = conn.getTable(tablePath)) {
+ UpsertWriter upsertWriter = table.newUpsert().createWriter();
+ upsertWriter.upsert(row(3, "name3", 3));
+ upsertWriter.flush();
+ }
+
+ // look up
+ for (int i = 0; i < 5; i++) {
+ lookupFunction.eval(i);
+ }
+
+ // collect the result and check
+ List<String> result =
+ collector.getOutputs().stream()
+ .map(RowData::toString)
+ .sorted()
+ .collect(Collectors.toList());
+ assertThat(result)
+ .containsExactly("+I(0,name0)", "+I(1,name1)", "+I(2,name2)", "+I(3,name3)");
+ lookupFunction.close();
+
+ // start lookup job after schema change.
+ lookupFunction =
+ new FlinkLookupFunction(
+ clientConf,
+ tablePath,
+ flinkRowType,
+ createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ null);
+ collector = new ListOutputCollector();
+ lookupFunction.setCollector(collector);
+ lookupFunction.open(null);
+ for (int i = 0; i < 5; i++) {
+ lookupFunction.eval(i);
+ }
+ result =
+ collector.getOutputs().stream()
+ .map(RowData::toString)
+ .sorted()
+ .collect(Collectors.toList());
+ assertThat(result)
+ .containsExactly("+I(0,name0)", "+I(1,name1)", "+I(2,name2)", "+I(3,name3)");
+ lookupFunction.close();
+ }
+
+ private void prepareData(TablePath tablePath, int rows, boolean schemaNotMatch)
+ throws Exception {
+ TableDescriptor descriptor = DEFAULT_PK_TABLE_DESCRIPTOR;
+ if (schemaNotMatch) {
+ descriptor =
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .primaryKey("id")
+ .column("name", DataTypes.STRING())
+ .column("id", DataTypes.INT())
+ .column("age", DataTypes.INT())
+ .build())
+ .distributedBy(DEFAULT_BUCKET_NUM, "id")
+ .build();
+ }
+
+ createTable(tablePath, descriptor);
// first write some data to the table
try (Table table = conn.getTable(tablePath)) {
UpsertWriter upsertWriter = table.newUpsert().createWriter();
for (int i = 0; i < rows; i++) {
- upsertWriter.upsert(row(i, "name" + i));
+ upsertWriter.upsert(schemaNotMatch ? row("name" + i, i, i) : row(i, "name" + i));
}
upsertWriter.flush();
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
index 6634236aa..88e2cb539 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
@@ -178,7 +178,6 @@ class FlinkSourceReaderTest extends FlinkTestBase {
tablePath,
sourceOutputType,
context,
- null,
new FlinkSourceReaderMetrics(context.metricGroup()),
recordEmitter,
null);
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
index 19aa0f7a9..c30c9cf72 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
@@ -85,35 +85,23 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
clientConf,
tablePath1,
DataTypes.ROW(
- DataTypes.FIELD("id", DataTypes.BIGINT()),
- DataTypes.FIELD("name", DataTypes.STRING()),
- DataTypes.FIELD("age", DataTypes.INT())),
- null,
+ DataTypes.FIELD("name2", DataTypes.STRING()),
+ DataTypes.FIELD("id", DataTypes.BIGINT())),
createMockSourceReaderMetrics(),
null))
.isInstanceOf(ValidationException.class)
- .hasMessage(
- "The Flink query schema is not matched to Fluss table schema. \n"
- + "Flink query schema: ROW<`id` BIGINT, `name` STRING, `age` INT>\n"
- + "Fluss table schema: ROW<`id` BIGINT NOT NULL, `name` STRING, `age` INT>");
-
- assertThatThrownBy(
- () ->
- new FlinkSourceSplitReader(
- clientConf,
- tablePath1,
- DataTypes.ROW(
- DataTypes.FIELD(
- "id", DataTypes.BIGINT().copy(false)),
- DataTypes.FIELD("name", DataTypes.STRING())),
- new int[] {1, 0},
- createMockSourceReaderMetrics(),
- null))
- .isInstanceOf(ValidationException.class)
- .hasMessage(
- "The Flink query schema is not matched to Fluss table schema. \n"
- + "Flink query schema: ROW<`id` BIGINT NOT NULL, `name` STRING>\n"
- + "Fluss table schema: ROW<`name` STRING, `id` BIGINT NOT NULL> (projection [1, 0])");
+ .hasMessage("The field name2 is not found in the fluss table.");
+
+ FlinkSourceSplitReader flinkSourceSplitReader =
+ new FlinkSourceSplitReader(
+ clientConf,
+ tablePath1,
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("id", DataTypes.BIGINT().copy(false))),
+ createMockSourceReaderMetrics(),
+ null);
+ assertThat(flinkSourceSplitReader.getProjectedFields()).isEqualTo(new int[] {1, 0});
}
@Test
@@ -396,7 +384,7 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
private FlinkSourceSplitReader createSplitReader(TablePath tablePath, RowType rowType) {
return new FlinkSourceSplitReader(
- clientConf, tablePath, rowType, null, createMockSourceReaderMetrics(), null);
+ clientConf, tablePath, rowType, createMockSourceReaderMetrics(), null);
}
private FlinkSourceReaderMetrics createMockSourceReaderMetrics() {
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java
index 6f0ab20f4..08dc96468 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java
@@ -132,7 +132,7 @@ class TieringCommitOperatorTest extends FlinkTestBase {
tablePath2, t2b1, 2, 22, 32L, numberOfWriteResults));
verifyNoLakeSnapshot(tablePath2);
- // add table1, bucket2
+ // addColumn table1, bucket2
TableBucket t1b2 = new TableBucket(tableId1, 2);
committerOperator.processElement(
createTableBucketWriteResultStreamRecord(
@@ -148,7 +148,7 @@ class TieringCommitOperatorTest extends FlinkTestBase {
expectedMaxTimestamps.put(t1b2, 23L);
verifyLakeSnapshot(tablePath1, tableId1, 1, expectedLogEndOffsets, expectedMaxTimestamps);
- // add table2, bucket2
+ // addColumn table2, bucket2
TableBucket t2b2 = new TableBucket(tableId2, 2);
committerOperator.processElement(
createTableBucketWriteResultStreamRecord(
diff --git a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
index 5aad1d5bf..9c86a92c1 100644
--- a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
+++ b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
@@ -79,7 +79,8 @@ public class HadoopUtils {
}
// Approach 3: Fluss configuration
- // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop conf
+ // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop
+ // conf
for (String key : flussConfiguration.keySet()) {
for (String prefix : FLUSS_CONFIG_PREFIXES) {
if (key.startsWith(prefix)) {
@@ -108,7 +109,7 @@ public class HadoopUtils {
}
/**
- * Search Hadoop configuration files in the given path, and add them to the configuration if
+ * Search Hadoop configuration files in the given path, andadd them to the configuration if
* found.
*/
private static boolean addHadoopConfIfFound(
diff --git a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
index 4b6d9de71..133c23322 100644
--- a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
+++ b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
@@ -147,7 +147,7 @@ class IcebergRewriteITCase extends FlinkIcebergTieringTestBase {
// add pos-delete and trigger compaction
rows = Arrays.asList(row(4, "v1"), row(4, "v2"));
flussRows.add(writeIcebergTableRecords(t1, t1Bucket, 6, false, rows).get(1));
- // rewritten files should fail to commit due to conflict, add check here
+ // rewritten files should fail to commit due to conflict,add check here
checkRecords(getIcebergRecords(t1), flussRows);
// 4 data file and 1 delete file
checkFileStatusInIcebergTable(t1, 4, true);
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
index a09a50c09..7f4d6e84e 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
@@ -69,7 +69,8 @@ final class ServerConnection {
private final ServerNode node;
- // TODO: add max inflight requests limit like Kafka's "max.in.flight.requests.per.connection"
+ // TODO: add max inflight requests limit like Kafka's
+ // "max.in.flight.requests.per.connection"
private final Map<Integer, InflightRequest> inflightRequests = MapUtils.newConcurrentHashMap();
private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
private final ConnectionMetrics connectionMetrics;
diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto
index ca41ebcb0..9feb38dc1 100644
--- a/fluss-rpc/src/main/proto/FlussApi.proto
+++ b/fluss-rpc/src/main/proto/FlussApi.proto
@@ -113,9 +113,12 @@ message AlterTableRequest {
required PbTablePath table_path = 1;
required bool ignore_if_not_exists = 2;
repeated PbAlterConfig config_changes = 3;
+ repeated PbAddColumn add_columns = 4;
+ repeated PbDropColumn drop_columns = 5;
+ repeated PbRenameColumn rename_columns = 6;
+ repeated PbModifyColumn modify_columns = 7;
}
-
message AlterTableResponse {
}
@@ -915,6 +918,32 @@ message PbAlterConfig {
required int32 op_type = 3; // SET=0, DELETE=1, APPEND=2, SUBTRACT=3
}
+message PbAddColumn{
+ required string column_name = 1;
+ required bytes data_type_json = 2;
+ optional string comment = 3;
+ required int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
+}
+
+message PbDropColumn{
+ required string column_name = 1;
+}
+
+message PbRenameColumn{
+ required string old_column_name = 1;
+ required string new_column_name = 2;
+}
+
+
+message PbModifyColumn{
+ required string column_name = 1;
+ required bytes data_type_json = 2;
+ optional string comment = 3;
+ optional int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
+}
+
+
+
message PbDescribeConfig{
required string config_key = 1;
optional string config_value = 2;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
index 82f3d5ec2..ededf510f 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
@@ -384,7 +384,7 @@ public abstract class RpcServiceBase extends RpcGatewayService implements AdminR
@Override
public CompletableFuture<GetFileSystemSecurityTokenResponse> getFileSystemSecurityToken(
GetFileSystemSecurityTokenRequest request) {
- // TODO: add ACL for per-table in https://github.com/apache/fluss/issues/752
+ // TODO:add ACL for per-table in https://github.com/apache/fluss/issues/752
try {
// In order to avoid repeatedly obtaining security token, cache it for a while.
long currentTimeMs = System.currentTimeMillis();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
index b7c92289b..def57231d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
@@ -148,7 +148,7 @@ public class CompletedSnapshotStoreManager {
() -> getAllSnapshotSize(bucket));
} else {
LOG.warn(
- "Failed to add bucketMetricGroup for tableBucket {} when creating completed snapshot.",
+ "Failed toadd bucketMetricGroup for tableBucket {} when creating completed snapshot.",
bucket);
}
return snapshotStore;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index b3fed108b..fbe0986b1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -31,6 +31,7 @@ import org.apache.fluss.exception.InvalidUpdateVersionException;
import org.apache.fluss.exception.TabletServerNotAvailableException;
import org.apache.fluss.exception.UnknownTableOrBucketException;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableBucketReplica;
import org.apache.fluss.metadata.TableInfo;
@@ -62,6 +63,7 @@ import org.apache.fluss.server.coordinator.event.FencedCoordinatorEvent;
import org.apache.fluss.server.coordinator.event.NewTabletServerEvent;
import org.apache.fluss.server.coordinator.event.NotifyKvSnapshotOffsetEvent;
import org.apache.fluss.server.coordinator.event.NotifyLeaderAndIsrResponseReceivedEvent;
+import org.apache.fluss.server.coordinator.event.SchemaChangeEvent;
import org.apache.fluss.server.coordinator.event.watcher.TableChangeWatcher;
import org.apache.fluss.server.coordinator.event.watcher.TabletServerChangeWatcher;
import org.apache.fluss.server.coordinator.statemachine.ReplicaStateMachine;
@@ -499,6 +501,9 @@ public class CoordinatorEventProcessor implements EventProcessor {
processDropTable((DropTableEvent) event);
} else if (event instanceof DropPartitionEvent) {
processDropPartition((DropPartitionEvent) event);
+ } else if (event instanceof SchemaChangeEvent) {
+ SchemaChangeEvent schemaChangeEvent = (SchemaChangeEvent) event;
+ processSchemaChange(schemaChangeEvent);
} else if (event instanceof NotifyLeaderAndIsrResponseReceivedEvent) {
processNotifyLeaderAndIsrResponseReceivedEvent(
(NotifyLeaderAndIsrResponseReceivedEvent) event);
@@ -551,6 +556,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
private void processCreateTable(CreateTableEvent createTableEvent) {
long tableId = createTableEvent.getTableInfo().getTableId();
+ int schemaId = createTableEvent.getTableInfo().getSchemaId();
// skip the table if it already exists
if (coordinatorContext.containsTableId(tableId)) {
return;
@@ -595,6 +601,44 @@ public class CoordinatorEventProcessor implements EventProcessor {
}
}
+ private void processSchemaChange(SchemaChangeEvent schemaChangeEvent) {
+ TablePath tablePath = schemaChangeEvent.getTablePath();
+ SchemaInfo schemaInfo = schemaChangeEvent.getSchemaInfo();
+ long tableId = coordinatorContext.getTableIdByPath(tablePath);
+
+ // if table is created, will create schema first. In this case, schema id will be included
+ // in CreateTableEvent.
+ if (tableId == TableInfo.UNKNOWN_TABLE_ID) {
+ return;
+ }
+
+ TableInfo oldTableInfo = coordinatorContext.getTableInfoById(tableId);
+ if (oldTableInfo.getSchemaId() == schemaInfo.getSchemaId()) {
+ return;
+ }
+
+ coordinatorContext.putTableInfo(
+ new TableInfo(
+ tablePath,
+ tableId,
+ schemaInfo.getSchemaId(),
+ schemaInfo.getSchema(),
+ oldTableInfo.getBucketKeys(),
+ oldTableInfo.getPartitionKeys(),
+ oldTableInfo.getNumBuckets(),
+ oldTableInfo.getProperties(),
+ oldTableInfo.getCustomProperties(),
+ oldTableInfo.getComment().orElse(null),
+ oldTableInfo.getCreatedTime(),
+ System.currentTimeMillis()));
+
+ updateTabletServerMetadataCache(
+ new HashSet<>(coordinatorContext.getLiveTabletServers().values()),
+ tableId,
+ null,
+ null);
+ }
+
private void processCreatePartition(CreatePartitionEvent createPartitionEvent) {
long partitionId = createPartitionEvent.getPartitionId();
// skip the partition if it already exists
@@ -1310,6 +1354,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
aliveTabletServers.stream().map(ServerInfo::id).collect(Collectors.toSet());
Set<Long> tablesToBeDeleted = coordinatorContext.getTablesToBeDeleted();
+
tablesToBeDeleted.forEach(
tableId ->
coordinatorRequestBatch.addUpdateMetadataRequestForTabletServers(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java
index de6778d35..982995d21 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java
@@ -279,8 +279,10 @@ public class CoordinatorRequestBatch {
* <li>case7: Leader and isr is changed for these input tableBuckets
* <li>case8: One newly tabletServer added into cluster
* <li>case9: One tabletServer is removed from cluster
+ * <li>case10: schemaId is changed after table is created.
* </ol>
*/
+ // todo: improve this with different phase enum.
public void addUpdateMetadataRequestForTabletServers(
Set<Integer> tabletServers,
@Nullable Long tableId,
@@ -298,7 +300,7 @@ public class CoordinatorRequestBatch {
.computeIfAbsent(tableId, k -> new HashMap<>())
.put(partitionId, Collections.emptyList());
} else {
- // case3, case4
+ // case3, case4, case10
updateMetadataRequestBucketMap.put(tableId, Collections.emptyList());
}
} else {
@@ -306,6 +308,7 @@ public class CoordinatorRequestBatch {
for (TableBucket tableBucket : tableBuckets) {
long currentTableId = tableBucket.getTableId();
Long currentPartitionId = tableBucket.getPartitionId();
+ // case1, case2
Optional<LeaderAndIsr> bucketLeaderAndIsr =
coordinatorContext.getBucketLeaderAndIsr(tableBucket);
Integer leaderEpoch =
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java
index 54480d663..73a760337 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java
@@ -135,7 +135,8 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getCommitRemot
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getPartitionSpec;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeCreateAclsResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeDropAclsResponse;
-import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toTableChanges;
+import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toAlterTableConfigChanges;
+import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toAlterTableSchemaChanges;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toTablePath;
import static org.apache.fluss.server.utils.TableAssignmentUtils.generateAssignment;
import static org.apache.fluss.utils.PartitionUtils.validatePartitionSpec;
@@ -312,19 +313,35 @@ public final class CoordinatorService extends RpcServiceBase implements Coordina
authorizer.authorize(currentSession(), OperationType.ALTER, Resource.table(tablePath));
}
- List<TableChange> tableChanges = toTableChanges(request.getConfigChangesList());
- TablePropertyChanges tablePropertyChanges = toTablePropertyChanges(tableChanges);
+ List<TableChange> alterTableConfigChanges =
+ toAlterTableConfigChanges(request.getConfigChangesList());
+ TablePropertyChanges tablePropertyChanges = toTablePropertyChanges(alterTableConfigChanges);
+ List<TableChange> alterSchemaChanges = toAlterTableSchemaChanges(request);
- LakeCatalogDynamicLoader.LakeCatalogContainer lakeCatalogContainer =
- lakeCatalogDynamicLoader.getLakeCatalogContainer();
- metadataManager.alterTableProperties(
- tablePath,
- tableChanges,
- tablePropertyChanges,
- request.isIgnoreIfNotExists(),
- lakeCatalogContainer.getLakeCatalog(),
- lakeTableTieringManager,
- new DefaultLakeCatalogContext(false, currentSession().getPrincipal()));
+ if (!alterSchemaChanges.isEmpty() && !alterTableConfigChanges.isEmpty()) {
+ // Only support one of alterTableConfigChanges and alterSchemaChanges for atomic change.
+ throw new InvalidAlterTableException(
+ "Table alteration can only be applied to one of the following: "
+ + "table properties or table schema.");
+ }
+
+ if (!alterSchemaChanges.isEmpty()) {
+ metadataManager.alterTableSchema(
+ tablePath, alterSchemaChanges, request.isIgnoreIfNotExists());
+ }
+
+ if (!alterTableConfigChanges.isEmpty()) {
+ LakeCatalogDynamicLoader.LakeCatalogContainer lakeCatalogContainer =
+ lakeCatalogDynamicLoader.getLakeCatalogContainer();
+ metadataManager.alterTableProperties(
+ tablePath,
+ alterTableConfigChanges,
+ tablePropertyChanges,
+ request.isIgnoreIfNotExists(),
+ lakeCatalogContainer.getLakeCatalog(),
+ lakeTableTieringManager,
+ new DefaultLakeCatalogContext(false, currentSession().getPrincipal()));
+ }
return CompletableFuture.completedFuture(new AlterTableResponse());
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
index ed544a8d1..19b13aaa9 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
@@ -38,6 +38,7 @@ import org.apache.fluss.lake.lakestorage.LakeCatalog;
import org.apache.fluss.metadata.DatabaseDescriptor;
import org.apache.fluss.metadata.DatabaseInfo;
import org.apache.fluss.metadata.ResolvedPartitionSpec;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
@@ -320,6 +321,37 @@ public class MetadataManager {
"Fail to create table " + tablePath);
}
+ public void alterTableSchema(
+ TablePath tablePath, List<TableChange> schemaChanges, boolean ignoreIfNotExists)
+ throws TableNotExistException, TableNotPartitionedException {
+ try {
+
+ TableInfo table = getTable(tablePath);
+
+ // validate the table column changes
+ if (!schemaChanges.isEmpty()) {
+ UpdateSchema schemaUpdate = new SchemaUpdate(table);
+ for (TableChange schemaChange : schemaChanges) {
+ schemaUpdate = schemaUpdate.applySchemaChange(schemaChange);
+ }
+ Schema newSchema = schemaUpdate.getSchema();
+ // update the schema
+ zookeeperClient.registerSchema(tablePath, newSchema);
+ }
+ } catch (Exception e) {
+ if (e instanceof TableNotExistException) {
+ if (ignoreIfNotExists) {
+ return;
+ }
+ throw (TableNotExistException) e;
+ } else if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ } else {
+ throw new FlussRuntimeException("Failed to alter table schema: " + tablePath, e);
+ }
+ }
+ }
+
public void alterTableProperties(
TablePath tablePath,
List<TableChange> tableChanges,
@@ -386,7 +418,8 @@ public class MetadataManager {
} else if (e instanceof RuntimeException) {
throw (RuntimeException) e;
} else {
- throw new FlussRuntimeException("Failed to alter table: " + tablePath, e);
+ throw new FlussRuntimeException(
+ "Failed to alter table properties: " + tablePath, e);
}
}
}
@@ -452,7 +485,7 @@ public class MetadataManager {
if (toEnableDataLake) {
TableInfo newTableInfo = newTableRegistration.toTableInfo(tablePath, schemaInfo);
- // if the table is lake table, we need to add it to lake table tiering manager
+ // if the table is lake table, we need toadd it to lake table tiering manager
lakeTableTieringManager.addNewLakeTable(newTableInfo);
} else if (toDisableDataLake) {
lakeTableTieringManager.removeLakeTable(newTableRegistration.tableId);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
new file mode 100644
index 000000000..1cdc702d7
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.coordinator;
+
+import org.apache.fluss.exception.SchemaChangeException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableInfo;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** Schema update. */
+public class SchemaUpdate implements UpdateSchema {
+ private final List<Schema.Column> columns;
+ private final AtomicInteger highestFieldId;
+ private final List<String> primaryKeys;
+ private final List<String> bucketKeys;
+ private final List<String> partitionKeys;
+ private final Map<String, Schema.Column> existedColumns;
+
+ public SchemaUpdate(TableInfo tableInfo) {
+ this.columns = new ArrayList<>();
+ this.existedColumns = new HashMap<>();
+ this.highestFieldId = new AtomicInteger(tableInfo.getSchema().getHighestFieldId());
+ this.primaryKeys = tableInfo.getPrimaryKeys();
+ this.bucketKeys = tableInfo.getBucketKeys();
+ this.partitionKeys = tableInfo.getPartitionKeys();
+ this.columns.addAll(tableInfo.getSchema().getColumns());
+ for (Schema.Column column : columns) {
+ existedColumns.put(column.getName(), column);
+ }
+ }
+
+ @Override
+ public Schema getSchema() {
+ Schema.Builder builder =
+ Schema.newBuilder()
+ .fromColumns(columns)
+ .highestFieldId((short) highestFieldId.get());
+ if (!primaryKeys.isEmpty()) {
+ builder.primaryKey(primaryKeys);
+ }
+
+ return builder.build();
+ }
+
+ @Override
+ public UpdateSchema applySchemaChange(TableChange columnChange) {
+ if (columnChange instanceof TableChange.AddColumn) {
+ return addColumn((TableChange.AddColumn) columnChange);
+ } else if (columnChange instanceof TableChange.ModifyColumn) {
+ return modifiedColumn((TableChange.ModifyColumn) columnChange);
+ } else if (columnChange instanceof TableChange.RenameColumn) {
+ return renameColumn((TableChange.RenameColumn) columnChange);
+ } else if (columnChange instanceof TableChange.DropColumn) {
+ return dropColumn((TableChange.DropColumn) columnChange);
+ }
+ throw new IllegalArgumentException(
+ "Unknown column change type " + columnChange.getClass().getName());
+ }
+
+ private UpdateSchema addColumn(TableChange.AddColumn addColumn) {
+ if (existedColumns.containsKey(addColumn.getName())) {
+ throw new IllegalArgumentException(
+ "Column " + addColumn.getName() + " already exists.");
+ }
+
+ TableChange.ColumnPosition position = addColumn.getPosition();
+ if (position != TableChange.ColumnPosition.last()) {
+ throw new IllegalArgumentException("Only support addColumn column at last now.");
+ }
+
+ if (!addColumn.getDataType().isNullable()) {
+ throw new IllegalArgumentException(
+ "Column " + addColumn.getName() + " must be nullable.");
+ }
+
+ Schema.Column newColumn =
+ new Schema.Column(
+ addColumn.getName(),
+ addColumn.getDataType(),
+ addColumn.getComment(),
+ (byte) highestFieldId.incrementAndGet());
+ columns.add(newColumn);
+ existedColumns.put(newColumn.getName(), newColumn);
+ return this;
+ }
+
+ private UpdateSchema dropColumn(TableChange.DropColumn dropColumn) {
+ throw new SchemaChangeException("Not support drop column now.");
+ }
+
+ private UpdateSchema modifiedColumn(TableChange.ModifyColumn modifyColumn) {
+ throw new SchemaChangeException("Not support modify column now.");
+ }
+
+ private UpdateSchema renameColumn(TableChange.RenameColumn renameColumn) {
+ throw new SchemaChangeException("Not support rename column now.");
+ }
+}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
similarity index 56%
copy from fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
copy to fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
index b2cea1382..1d51704ac 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
@@ -15,26 +15,14 @@
* limitations under the License.
*/
-package org.apache.fluss.flink.sink.serializer;
+package org.apache.fluss.server.coordinator;
-import org.apache.fluss.types.RowType;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
+/** Schema update. */
+public interface UpdateSchema {
+ Schema getSchema();
-/**
- * Contextual information that can be used during initialization of {@link
- * FlussSerializationSchema}.
- */
-public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
-
- private final RowType rowSchema;
-
- public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
- }
-
- @Override
- public RowType getRowSchema() {
- return rowSchema;
- }
+ UpdateSchema applySchemaChange(TableChange columnChange);
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/SchemaChangeEvent.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/SchemaChangeEvent.java
new file mode 100644
index 000000000..7f7f7362b
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/SchemaChangeEvent.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.coordinator.event;
+
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TablePath;
+
+import java.util.Objects;
+
+/** An event for schema change. */
+public class SchemaChangeEvent implements CoordinatorEvent {
+ private final TablePath tablePath;
+ private final SchemaInfo schemaInfo;
+
+ public SchemaChangeEvent(TablePath tablePath, SchemaInfo schemaInfo) {
+ this.tablePath = tablePath;
+ this.schemaInfo = schemaInfo;
+ }
+
+ public TablePath getTablePath() {
+ return tablePath;
+ }
+
+ public SchemaInfo getSchemaInfo() {
+ return schemaInfo;
+ }
+
+ @Override
+ public String toString() {
+ return "SchemaChangeEvent{" + "tablePath=" + tablePath + ", schemaInfo=" + schemaInfo + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SchemaChangeEvent)) {
+ return false;
+ }
+ SchemaChangeEvent that = (SchemaChangeEvent) o;
+ return Objects.equals(tablePath, that.tablePath)
+ && Objects.equals(schemaInfo, that.schemaInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tablePath, schemaInfo);
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
index c8ee481f6..acfb7e689 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
@@ -29,16 +29,19 @@ import org.apache.fluss.server.coordinator.event.CreateTableEvent;
import org.apache.fluss.server.coordinator.event.DropPartitionEvent;
import org.apache.fluss.server.coordinator.event.DropTableEvent;
import org.apache.fluss.server.coordinator.event.EventManager;
+import org.apache.fluss.server.coordinator.event.SchemaChangeEvent;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.data.PartitionAssignment;
import org.apache.fluss.server.zk.data.TableAssignment;
import org.apache.fluss.server.zk.data.TableRegistration;
import org.apache.fluss.server.zk.data.ZkData.DatabasesZNode;
import org.apache.fluss.server.zk.data.ZkData.PartitionZNode;
+import org.apache.fluss.server.zk.data.ZkData.SchemaZNode;
import org.apache.fluss.server.zk.data.ZkData.TableZNode;
import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCache;
import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener;
+import org.apache.fluss.utils.types.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,6 +105,14 @@ public class TableChangeWatcher {
physicalTablePath.getTablePath(),
physicalTablePath.getPartitionName(),
newData);
+ break;
+ }
+
+ Tuple2<TablePath, Integer> tablePathIntegerTuple2 =
+ SchemaZNode.parsePath(newData.getPath());
+ if (tablePathIntegerTuple2 != null) {
+ processSchemaChange(
+ tablePathIntegerTuple2.f0, tablePathIntegerTuple2.f1);
}
}
break;
@@ -231,4 +242,31 @@ public class TableChangeWatcher {
tablePath, tableId, partitionId, partitionName, partitionAssignment));
}
}
+
+ private void processSchemaChange(TablePath tablePath, int schemaId) {
+
+ try {
+ int currentSchemaId = zooKeeperClient.getCurrentSchemaId(tablePath);
+ SchemaInfo schemaInfo;
+ if (schemaId != currentSchemaId) {
+ LOG.warn(
+ "Schema id {} is not equal to current schema id {}. Skipping schema change processing.",
+ schemaId,
+ currentSchemaId);
+ return;
+ }
+
+ Optional<SchemaInfo> optSchema = zooKeeperClient.getSchemaById(tablePath, schemaId);
+ if (!optSchema.isPresent()) {
+ LOG.error("No schema for table {} in zookeeper.", tablePath);
+ return;
+ } else {
+ schemaInfo = optSchema.get();
+ }
+
+ eventManager.put(new SchemaChangeEvent(tablePath, schemaInfo));
+ } catch (Exception e) {
+ LOG.error("Fail to get current schema id for table {}.", tablePath, e);
+ }
+ }
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
index 460562bc1..fad96a596 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
@@ -55,7 +55,7 @@ public class BatchRegisterLeadAndIsr {
tableBucket, leaderAndIsr, partitionName, liveReplicas));
} else {
throw new IllegalArgumentException(
- "Try to add a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "Try toadd a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "batch tableId="
+ tableId
+ " batch partitionId:"
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java
index 2d2764499..d1e24f6bc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java
@@ -28,7 +28,7 @@ import org.apache.fluss.memory.LazyMemorySegmentPool;
import org.apache.fluss.memory.MemorySegmentPool;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
-import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -161,7 +161,7 @@ public final class KvManager extends TabletManagerBase {
TableBucket tableBucket,
LogTablet logTablet,
KvFormat kvFormat,
- Schema schema,
+ SchemaGetter schemaGetter,
TableConfig tableConfig,
ArrowCompressionInfo arrowCompressionInfo)
throws Exception {
@@ -173,8 +173,7 @@ public final class KvManager extends TabletManagerBase {
}
File tabletDir = getOrCreateTabletDir(tablePath, tableBucket);
-
- RowMerger merger = RowMerger.create(tableConfig, schema, kvFormat);
+ RowMerger merger = RowMerger.create(tableConfig, kvFormat);
KvTablet tablet =
KvTablet.create(
logTablet,
@@ -184,9 +183,9 @@ public final class KvManager extends TabletManagerBase {
arrowBufferAllocator,
memorySegmentPool,
kvFormat,
- schema,
merger,
- arrowCompressionInfo);
+ arrowCompressionInfo,
+ schemaGetter);
currentKvs.put(tableBucket, tablet);
LOG.info(
@@ -256,7 +255,7 @@ public final class KvManager extends TabletManagerBase {
}
}
- public KvTablet loadKv(File tabletDir) throws Exception {
+ public KvTablet loadKv(File tabletDir, SchemaGetter schemaGetter) throws Exception {
Tuple2<PhysicalTablePath, TableBucket> pathAndBucket = FlussPaths.parseTabletDir(tabletDir);
PhysicalTablePath physicalTablePath = pathAndBucket.f0;
TableBucket tableBucket = pathAndBucket.f1;
@@ -277,11 +276,10 @@ public final class KvManager extends TabletManagerBase {
// TODO: we should support recover schema from disk to decouple put and schema.
TablePath tablePath = physicalTablePath.getTablePath();
TableInfo tableInfo = getTableInfo(zkClient, tablePath);
+
RowMerger rowMerger =
RowMerger.create(
- tableInfo.getTableConfig(),
- tableInfo.getSchema(),
- tableInfo.getTableConfig().getKvFormat());
+ tableInfo.getTableConfig(), tableInfo.getTableConfig().getKvFormat());
KvTablet kvTablet =
KvTablet.create(
physicalTablePath,
@@ -293,9 +291,9 @@ public final class KvManager extends TabletManagerBase {
arrowBufferAllocator,
memorySegmentPool,
tableInfo.getTableConfig().getKvFormat(),
- tableInfo.getSchema(),
rowMerger,
- tableInfo.getTableConfig().getArrowCompressionInfo());
+ tableInfo.getTableConfig().getArrowCompressionInfo(),
+ schemaGetter);
if (this.currentKvs.containsKey(tableBucket)) {
throw new IllegalStateException(
String.format(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
index 9057598a1..1f1c4192d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
@@ -17,9 +17,9 @@
package org.apache.fluss.server.kv;
-import org.apache.fluss.exception.KvStorageException;
import org.apache.fluss.metadata.DataLakeFormat;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -62,6 +62,7 @@ public class KvRecoverHelper {
private KeyEncoder keyEncoder;
private RowEncoder rowEncoder;
+ private final SchemaGetter schemaGetter;
private InternalRow.FieldGetter[] currentFieldGetters;
@@ -71,11 +72,22 @@ public class KvRecoverHelper {
long recoverPointOffset,
KvRecoverContext recoverContext,
KvFormat kvFormat) {
+ throw new UnsupportedOperationException();
+ }
+
+ public KvRecoverHelper(
+ KvTablet kvTablet,
+ LogTablet logTablet,
+ long recoverPointOffset,
+ KvRecoverContext recoverContext,
+ KvFormat kvFormat,
+ SchemaGetter schemaGetter) {
this.kvTablet = kvTablet;
this.logTablet = logTablet;
this.recoverPointOffset = recoverPointOffset;
this.recoverContext = recoverContext;
this.kvFormat = kvFormat;
+ this.schemaGetter = schemaGetter;
}
public void recover() throws Exception {
@@ -89,6 +101,8 @@ public class KvRecoverHelper {
// after the corresponding log offset is acked(when high watermark is advanced to the
// offset)
+ initSchema(schemaGetter.getLatestSchemaInfo().getSchemaId());
+
long nextLogOffset = recoverPointOffset;
// read to high watermark
try (KvBatchWriter kvBatchWriter = kvTablet.createKvBatchWriter()) {
@@ -138,20 +152,24 @@ public class KvRecoverHelper {
}
for (LogRecordBatch logRecordBatch : logRecords.batches()) {
- short schemaId = logRecordBatch.schemaId();
- if (currentSchemaId == null) {
- initSchema(schemaId);
- } else if (currentSchemaId != schemaId) {
- throw new KvStorageException(
- String.format(
- "Can't recover kv tablet for table bucket from log %s since the schema changes from schema id %d to schema id %d. "
- + "Currently, schema change is not supported.",
- recoverContext.tableBucket, currentSchemaId, schemaId));
- }
+ // short schemaId = logRecordBatch.schemaId();
+ // if (currentSchemaId == null) {
+ // initSchema(schemaId);
+ // } else if (currentSchemaId != schemaId) {
+ // throw new KvStorageException(
+ // String.format(
+ // "Can't recover kv tablet for table bucket from
+ // log %s since the schema changes from schema id %d to schema id %d. "
+ // + "Currently, schema change is not
+ // supported.",
+ // recoverContext.tableBucket, currentSchemaId,
+ // schemaId));
+ // }
+ // todo: currentRowType和currentSchemaId无法对齐
try (LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- currentRowType, currentSchemaId);
+ currentRowType, currentSchemaId, schemaGetter);
CloseableIterator<LogRecord> logRecordIter =
logRecordBatch.records(readContext)) {
while (logRecordIter.hasNext()) {
@@ -164,7 +182,8 @@ public class KvRecoverHelper {
// the log row format may not compatible with kv row format,
// e.g, arrow vs. compacted, thus needs a conversion here.
BinaryRow row = toKvRow(logRecord.getRow());
- value = ValueEncoder.encodeValue(schemaId, row);
+ // todo: short value是否会有问题，感觉可以先check一下
+ value = ValueEncoder.encodeValue(currentSchemaId.shortValue(), row);
}
resumeRecordConsumer.accept(
new KeyValueAndLogOffset(key, value, logRecord.logOffset()));
@@ -203,7 +222,7 @@ public class KvRecoverHelper {
// kv tablet's table id or not. If not equal, it means other table with same
// table path has been created, so the kv tablet's table is consider to be
// deleted. We can ignore the restore operation
- currentRowType = tableInfo.getRowType();
+ currentRowType = schemaGetter.getSchema(schemaId).getRowType();
DataType[] dataTypes = currentRowType.getChildren().toArray(new DataType[0]);
currentSchemaId = schemaId;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
index f91d5cc59..176e749f5 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
@@ -23,19 +23,23 @@ import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.DeletionDisabledException;
import org.apache.fluss.exception.KvStorageException;
+import org.apache.fluss.exception.SchemaNotExistException;
import org.apache.fluss.memory.MemorySegmentPool;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.KvRecord;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.record.KvRecordReadContext;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.arrow.ArrowWriterPool;
import org.apache.fluss.row.arrow.ArrowWriterProvider;
import org.apache.fluss.row.encode.ValueDecoder;
@@ -57,7 +61,6 @@ import org.apache.fluss.server.log.LogTablet;
import org.apache.fluss.server.metrics.group.TabletServerMetricGroup;
import org.apache.fluss.server.utils.FatalErrorHandler;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.BytesUtils;
import org.apache.fluss.utils.FileUtils;
@@ -80,6 +83,7 @@ import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.fluss.utils.SchemaUtil.getTargetColumns;
import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock;
import static org.apache.fluss.utils.concurrent.LockUtils.inWriteLock;
@@ -104,11 +108,12 @@ public final class KvTablet {
private final ReadWriteLock kvLock = new ReentrantReadWriteLock();
private final LogFormat logFormat;
private final KvFormat kvFormat;
- private final Schema schema;
// defines how to merge rows on the same primary key
private final RowMerger rowMerger;
private final ArrowCompressionInfo arrowCompressionInfo;
+ private final SchemaGetter schemaGetter;
+
/**
* The kv data in pre-write buffer whose log offset is less than the flushedLogOffset has been
* flushed into kv.
@@ -130,9 +135,9 @@ public final class KvTablet {
BufferAllocator arrowBufferAllocator,
MemorySegmentPool memorySegmentPool,
KvFormat kvFormat,
- Schema schema,
RowMerger rowMerger,
- ArrowCompressionInfo arrowCompressionInfo) {
+ ArrowCompressionInfo arrowCompressionInfo,
+ SchemaGetter schemaGetter) {
this.physicalPath = physicalPath;
this.tableBucket = tableBucket;
this.logTablet = logTablet;
@@ -144,9 +149,9 @@ public final class KvTablet {
this.arrowWriterProvider = new ArrowWriterPool(arrowBufferAllocator);
this.memorySegmentPool = memorySegmentPool;
this.kvFormat = kvFormat;
- this.schema = schema;
this.rowMerger = rowMerger;
this.arrowCompressionInfo = arrowCompressionInfo;
+ this.schemaGetter = schemaGetter;
}
public static KvTablet create(
@@ -157,9 +162,9 @@ public final class KvTablet {
BufferAllocator arrowBufferAllocator,
MemorySegmentPool memorySegmentPool,
KvFormat kvFormat,
- Schema schema,
RowMerger rowMerger,
- ArrowCompressionInfo arrowCompressionInfo)
+ ArrowCompressionInfo arrowCompressionInfo,
+ SchemaGetter schemaGetter)
throws IOException {
Tuple2<PhysicalTablePath, TableBucket> tablePathAndBucket =
FlussPaths.parseTabletDir(kvTabletDir);
@@ -173,9 +178,9 @@ public final class KvTablet {
arrowBufferAllocator,
memorySegmentPool,
kvFormat,
- schema,
rowMerger,
- arrowCompressionInfo);
+ arrowCompressionInfo,
+ schemaGetter);
}
public static KvTablet create(
@@ -188,9 +193,9 @@ public final class KvTablet {
BufferAllocator arrowBufferAllocator,
MemorySegmentPool memorySegmentPool,
KvFormat kvFormat,
- Schema schema,
RowMerger rowMerger,
- ArrowCompressionInfo arrowCompressionInfo)
+ ArrowCompressionInfo arrowCompressionInfo,
+ SchemaGetter schemaGetter)
throws IOException {
RocksDBKv kv = buildRocksDBKv(serverConf, kvTabletDir);
return new KvTablet(
@@ -205,9 +210,9 @@ public final class KvTablet {
arrowBufferAllocator,
memorySegmentPool,
kvFormat,
- schema,
rowMerger,
- arrowCompressionInfo);
+ arrowCompressionInfo,
+ schemaGetter);
}
private static RocksDBKv buildRocksDBKv(Configuration configuration, File kvDir)
@@ -259,26 +264,58 @@ public final class KvTablet {
kvLock,
() -> {
rocksDBKv.checkIfRocksDBClosed();
- short schemaId = kvRecords.schemaId();
- RowMerger currentMerger = rowMerger.configureTargetColumns(targetColumns);
- RowType rowType = schema.getRowType();
- WalBuilder walBuilder = createWalBuilder(schemaId, rowType);
+
+ SchemaInfo schemaInfo = schemaGetter.getLatestSchemaInfo();
+ Schema latestSchema = schemaInfo.getSchema();
+ // In the following scenario, we need to support schema migration:
+ // schema1: old data's schema -> valueDecoder
+ // schema2: new data's schema
+ // schema3: current schema (also the latest schema)
+ // schema1 <= schema2 <= current schema id
+ short latestSchemaId = (short) schemaInfo.getSchemaId();
+ short schemaIdOfNewData = kvRecords.schemaId();
+ if (schemaIdOfNewData > latestSchemaId || schemaIdOfNewData < 0) {
+ throw new SchemaNotExistException(
+ "Invalid schema id: "
+ + schemaIdOfNewData
+ + ", latest schema id: "
+ + latestSchemaId);
+ }
+ Schema schemaOfNewData = schemaGetter.getSchema(schemaIdOfNewData);
+ RowMerger currentMerger =
+ rowMerger.configureTargetColumns(
+ schemaIdOfNewData == latestSchemaId
+ ? targetColumns
+ : getTargetColumns(
+ targetColumns, schemaOfNewData, latestSchema),
+ latestSchemaId,
+ latestSchema);
+ RowType currentRowType = latestSchema.getRowType();
+ WalBuilder walBuilder = createWalBuilder(latestSchemaId, currentRowType);
walBuilder.setWriterState(kvRecords.writerId(), kvRecords.batchSequence());
// get offset to track the offset corresponded to the kv record
long logEndOffsetOfPrevBatch = logTablet.localLogEndOffset();
- DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
try {
long logOffset = logEndOffsetOfPrevBatch;
// TODO: reuse the read context and decoder
KvRecordBatch.ReadContext readContext =
- KvRecordReadContext.createReadContext(kvFormat, fieldTypes);
- ValueDecoder valueDecoder =
- new ValueDecoder(readContext.getRowDecoder(schemaId));
+ KvRecordReadContext.createReadContext(kvFormat, schemaGetter);
+
+ // replace new row with current schema if the schema id is older than now
+ ProjectedRow projectedRow =
+ ProjectedRow.from(schemaOfNewData, latestSchema);
+ ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat);
for (KvRecord kvRecord : kvRecords.records(readContext)) {
+
byte[] keyBytes = BytesUtils.toArray(kvRecord.getKey());
KvPreWriteBuffer.Key key = KvPreWriteBuffer.Key.of(keyBytes);
- if (kvRecord.getRow() == null) {
+ InternalRow row = kvRecord.getRow();
+ if (row != null && schemaIdOfNewData < latestSchemaId) {
+ row = projectedRow.replaceRow(row);
+ }
+
+ if (row == null) {
DeleteBehavior deleteBehavior = rowMerger.deleteBehavior();
if (deleteBehavior == DeleteBehavior.IGNORE) {
// skip delete rows if the merger doesn't support yet
@@ -290,14 +327,26 @@ public final class KvTablet {
}
// it's for deletion
byte[] oldValue = getFromBufferOrKv(key);
+
if (oldValue == null) {
// there might be large amount of such deletion, so we don't log
LOG.debug(
"The specific key can't be found in kv tablet although the kv record is for deletion, "
+ "ignore it directly as it doesn't exist in the kv tablet yet.");
} else {
- BinaryRow oldRow = valueDecoder.decodeValue(oldValue).row;
- BinaryRow newRow = currentMerger.delete(oldRow);
+ ValueDecoder.Value oldRowAndSchemaId =
+ valueDecoder.decodeValue(oldValue);
+ InternalRow oldRow = oldRowAndSchemaId.row;
+ if (oldRow != null
+ && oldRowAndSchemaId.schemaId != latestSchemaId) {
+ // todo: 后续想办法复用对应的projected row.
+ Schema schemaOfOldData =
+ schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
+ oldRow =
+ ProjectedRow.from(schemaOfOldData, latestSchema)
+ .replaceRow(oldRow);
+ }
+ InternalRow newRow = currentMerger.delete(oldRow);
// if newRow is null, it means the row should be deleted
if (newRow == null) {
walBuilder.append(ChangeType.DELETE, oldRow);
@@ -308,7 +357,11 @@ public final class KvTablet {
walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
kvPreWriteBuffer.put(
key,
- ValueEncoder.encodeValue(schemaId, newRow),
+ ValueEncoder.encodeRow(
+ latestSchemaId,
+ kvFormat,
+ currentRowType,
+ newRow),
logOffset + 1);
logOffset += 2;
}
@@ -318,32 +371,53 @@ public final class KvTablet {
byte[] oldValue = getFromBufferOrKv(key);
// it's update
if (oldValue != null) {
- BinaryRow oldRow = valueDecoder.decodeValue(oldValue).row;
- BinaryRow newRow =
- currentMerger.merge(oldRow, kvRecord.getRow());
+ ValueDecoder.Value oldRowAndSchemaId =
+ valueDecoder.decodeValue(oldValue);
+ InternalRow oldRow = oldRowAndSchemaId.row;
+ if (oldRow != null
+ && oldRowAndSchemaId.schemaId != latestSchemaId) {
+ // todo: 后续想办法复用对应的projected row.
+ Schema schemaOfOldData =
+ schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
+ oldRow =
+ ProjectedRow.from(schemaOfOldData, latestSchema)
+ .replaceRow(oldRow);
+ }
+
+ InternalRow newRow = currentMerger.merge(oldRow, row);
if (newRow == oldRow) {
// newRow is the same to oldRow, means nothing
// happens (no update/delete), and input should be ignored
continue;
}
+
walBuilder.append(ChangeType.UPDATE_BEFORE, oldRow);
walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
// logOffset is for -U, logOffset + 1 is for +U, we need to use
// the log offset for +U
kvPreWriteBuffer.put(
key,
- ValueEncoder.encodeValue(schemaId, newRow),
+ ValueEncoder.encodeRow(
+ latestSchemaId,
+ kvFormat,
+ currentRowType,
+ newRow),
logOffset + 1);
logOffset += 2;
} else {
// it's insert
- // TODO: we should add guarantees that all non-specified columns
+ // TODO: we shouldadd guarantees that all non-specified
+ // columns
// of the input row are set to null.
- BinaryRow newRow = kvRecord.getRow();
+ InternalRow newRow = row;
walBuilder.append(ChangeType.INSERT, newRow);
kvPreWriteBuffer.put(
key,
- ValueEncoder.encodeValue(schemaId, newRow),
+ ValueEncoder.encodeRow(
+ latestSchemaId,
+ kvFormat,
+ currentRowType,
+ newRow),
logOffset++);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
index 70063afc8..3ce22854c 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
@@ -42,13 +42,13 @@ public class PartialUpdaterCache {
// TODO: extend to tableId and schemaId when the cache is shared across all tables
public PartialUpdater getOrCreatePartialUpdater(
- KvFormat kvFormat, Schema schema, int[] targetColumns) {
+ KvFormat kvFormat, short schemaId, Schema schema, int[] targetColumns) {
return rowPartialUpdaters.get(
- getPartialUpdaterKey(targetColumns),
+ getPartialUpdaterKey(targetColumns, schemaId),
k -> new PartialUpdater(kvFormat, schema, targetColumns));
}
- private String getPartialUpdaterKey(int[] targetColumns) {
- return Arrays.toString(targetColumns);
+ private String getPartialUpdaterKey(int[] targetColumns, int schemaId) {
+ return schemaId + "_" + Arrays.toString(targetColumns);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
index 049d2859a..af3500c59 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
import org.apache.fluss.server.kv.partialupdate.PartialUpdater;
import org.apache.fluss.server.kv.partialupdate.PartialUpdaterCache;
@@ -34,12 +34,9 @@ public class DefaultRowMerger implements RowMerger {
private final PartialUpdaterCache partialUpdaterCache;
private final KvFormat kvFormat;
- private final Schema schema;
private final DeleteBehavior deleteBehavior;
- public DefaultRowMerger(
- Schema schema, KvFormat kvFormat, @Nullable DeleteBehavior deleteBehavior) {
- this.schema = schema;
+ public DefaultRowMerger(KvFormat kvFormat, @Nullable DeleteBehavior deleteBehavior) {
this.kvFormat = kvFormat;
// for compatibility, default to ALLOW if not specified
this.deleteBehavior = deleteBehavior != null ? deleteBehavior : DeleteBehavior.ALLOW;
@@ -49,14 +46,14 @@ public class DefaultRowMerger implements RowMerger {
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
// always retain the new row (latest row)
return newRow;
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
// returns null to indicate the row is deleted
return null;
}
@@ -67,13 +64,15 @@ public class DefaultRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(@Nullable int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ @Nullable int[] targetColumns, short schemaId, Schema schema) {
if (targetColumns == null) {
return this;
} else {
// this also sanity checks the validity of the partial update
PartialUpdater partialUpdater =
- partialUpdaterCache.getOrCreatePartialUpdater(kvFormat, schema, targetColumns);
+ partialUpdaterCache.getOrCreatePartialUpdater(
+ kvFormat, schemaId, schema, targetColumns);
return new PartialUpdateRowMerger(partialUpdater, deleteBehavior);
}
}
@@ -91,20 +90,21 @@ public class DefaultRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ int[] targetColumns, short schemaId, Schema schema) {
throw new IllegalStateException(
"PartialUpdateRowMerger does not support reconfigure target merge columns.");
}
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
return partialUpdater.updateRow(oldRow, newRow);
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
return partialUpdater.deleteRow(oldRow);
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
index 3271f0920..79592e177 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
@@ -19,7 +19,8 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.row.InternalRow;
import javax.annotation.Nullable;
@@ -43,14 +44,14 @@ public class FirstRowRowMerger implements RowMerger {
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
// always retain the old row (first row)
return oldRow;
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the first_row merge engine.");
}
@@ -61,7 +62,8 @@ public class FirstRowRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(@Nullable int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ @Nullable int[] targetColumns, short schemaId, Schema schema) {
if (targetColumns == null) {
return this;
} else {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
index b942cf181..5f3414574 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
@@ -23,7 +23,7 @@ import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
import javax.annotation.Nullable;
@@ -40,7 +40,7 @@ public interface RowMerger {
* @return the merged row, if the returned row is the same to the old row, then nothing happens
* to the row (no update, no delete).
*/
- BinaryRow merge(BinaryRow oldRow, BinaryRow newRow);
+ InternalRow merge(InternalRow oldRow, InternalRow newRow);
/**
* Merge the old row with a delete row.
@@ -52,7 +52,7 @@ public interface RowMerger {
* @return the merged row, or null if the row is deleted.
*/
@Nullable
- BinaryRow delete(BinaryRow oldRow);
+ InternalRow delete(InternalRow oldRow);
/**
* The behavior of delete operations on primary key tables.
@@ -62,10 +62,10 @@ public interface RowMerger {
DeleteBehavior deleteBehavior();
/** Dynamically configure the target columns to merge and return the effective merger. */
- RowMerger configureTargetColumns(@Nullable int[] targetColumns);
+ RowMerger configureTargetColumns(@Nullable int[] targetColumns, short schemaId, Schema schema);
/** Create a row merger based on the given configuration. */
- static RowMerger create(TableConfig tableConf, Schema schema, KvFormat kvFormat) {
+ static RowMerger create(TableConfig tableConf, KvFormat kvFormat) {
Optional<MergeEngineType> mergeEngineType = tableConf.getMergeEngineType();
@Nullable DeleteBehavior deleteBehavior = tableConf.getDeleteBehavior().orElse(null);
@@ -81,14 +81,13 @@ public interface RowMerger {
"'%s' must be set for versioned merge engine.",
ConfigOptions.TABLE_MERGE_ENGINE_VERSION_COLUMN.key()));
}
- return new VersionedRowMerger(
- schema.getRowType(), versionColumn.get(), deleteBehavior);
+ return new VersionedRowMerger(versionColumn.get(), deleteBehavior);
default:
throw new IllegalArgumentException(
"Unsupported merge engine type: " + mergeEngineType.get());
}
} else {
- return new DefaultRowMerger(schema, kvFormat, deleteBehavior);
+ return new DefaultRowMerger(kvFormat, deleteBehavior);
}
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
index 11901b34b..9706230b5 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
@@ -19,7 +19,8 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
import org.apache.fluss.types.DataType;
@@ -45,31 +46,32 @@ public class VersionedRowMerger implements RowMerger {
private static final TimestampLtz MIN_TIMESTAMP_LTZ =
TimestampLtz.fromEpochMillis(Long.MIN_VALUE, 0);
- private final Comparator<BinaryRow> versionComparator;
-
+ private final String versionColumnName;
private final DeleteBehavior deleteBehavior;
- public VersionedRowMerger(
- RowType schema, String versionColumnName, @Nullable DeleteBehavior deleteBehavior) {
- this.versionComparator = createVersionComparator(schema, versionColumnName);
+ private int schemaId;
+ private Comparator<InternalRow> versionComparator;
+
+ public VersionedRowMerger(String versionColumnName, @Nullable DeleteBehavior deleteBehavior) {
if (deleteBehavior == DeleteBehavior.ALLOW) {
throw new IllegalArgumentException(
"DELETE is not supported for the versioned merge engine.");
}
// for compatibility, default to IGNORE if not specified
this.deleteBehavior = deleteBehavior != null ? deleteBehavior : DeleteBehavior.IGNORE;
+ this.versionColumnName = versionColumnName;
}
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
// return newRow if newRow's version is larger or equal than oldRow's version
return versionComparator.compare(oldRow, newRow) <= 0 ? newRow : oldRow;
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the versioned merge engine.");
}
@@ -80,8 +82,14 @@ public class VersionedRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(@Nullable int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ @Nullable int[] targetColumns, short schemaId, Schema schema) {
if (targetColumns == null) {
+ if (schemaId != this.schemaId) {
+ this.schemaId = schemaId;
+ this.versionComparator =
+ createVersionComparator(schema.getRowType(), versionColumnName);
+ }
return this;
} else {
throw new UnsupportedOperationException(
@@ -90,7 +98,7 @@ public class VersionedRowMerger implements RowMerger {
}
/** Create a comparator for version column. */
- public static Comparator<BinaryRow> createVersionComparator(
+ public static Comparator<InternalRow> createVersionComparator(
RowType schema, String versionColumnName) {
int columnIndex = schema.getFieldIndex(versionColumnName);
checkArgument(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
index 3e93cc46a..b8d941221 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
@@ -99,7 +99,7 @@ public class CompletedSnapshotStore {
* Synchronously writes the new snapshots to snapshot handle store and asynchronously removes
* older ones.
*
- * @param snapshot Completed snapshot to add.
+ * @param snapshot Completed snapshot toadd.
*/
@VisibleForTesting
CompletedSnapshot addSnapshotAndSubsumeOldestOne(
@@ -116,7 +116,7 @@ public class CompletedSnapshotStore {
completedSnapshotHandleStore.add(
snapshot.getTableBucket(), snapshot.getSnapshotID(), completedSnapshotHandle);
- // Now add the new one. If it fails, we don't want to lose existing data.
+ // Nowadd the new one. If it fails, we don't want to lose existing data.
completedSnapshots.addLast(snapshot);
// Remove completed snapshot from queue and snapshotStateHandleStore, not discard.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
index fe517f268..e09c93b22 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
@@ -328,7 +328,7 @@ public class SnapshotLocation {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: may add entropy injection?
+ // todo: mayadd entropy injection?
this.kvFilePath = createFilePath();
this.outStream = fs.create(kvFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
index 2af972ae7..847bfa883 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
@@ -19,9 +19,9 @@ package org.apache.fluss.server.log;
import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.compression.ArrowCompressionInfo;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.rpc.messages.FetchLogRequest;
-import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -66,7 +66,7 @@ public final class FetchParams {
private final int minFetchBytes;
private final long maxWaitMs;
- // TODO: add more params like epoch etc.
+ // TODO:add more params like epoch etc.
public FetchParams(int replicaId, int maxFetchBytes) {
this(replicaId, true, maxFetchBytes, DEFAULT_MIN_FETCH_BYTES, DEFAULT_MAX_WAIT_MS);
@@ -97,7 +97,7 @@ public final class FetchParams {
long tableId,
long fetchOffset,
int maxFetchBytes,
- RowType schema,
+ SchemaGetter schemaGetter,
ArrowCompressionInfo compressionInfo,
@Nullable int[] projectedFields) {
this.fetchOffset = fetchOffset;
@@ -107,8 +107,9 @@ public final class FetchParams {
if (fileLogProjection == null) {
fileLogProjection = new FileLogProjection();
}
+
fileLogProjection.setCurrentProjection(
- tableId, schema, compressionInfo, projectedFields);
+ tableId, schemaGetter, compressionInfo, projectedFields);
} else {
projectionEnabled = false;
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
index 130b3855d..086e23b0e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
@@ -405,6 +405,8 @@ public final class LocalLog {
(maxOffsetMetadata.getSegmentBaseOffset() == segment.getBaseOffset())
? maxOffsetMetadata.getRelativePositionInSegment()
: segment.getSizeInBytes();
+
+ // segment需要查询
fetchDataInfo =
segment.read(readOffset, maxLength, maxPosition, minOneMessage, projection);
if (fetchDataInfo == null) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
index 835a94ff6..05233e0ff 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
@@ -203,7 +203,7 @@ public class FsRemoteLogOutputStream extends FSDataOutputStream {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: may add entropy injection?
+ // todo: mayadd entropy injection?
this.outStream = fs.create(remoteLogFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
} catch (Exception e) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
index 73aee521e..86a6bbf9e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
@@ -40,7 +40,7 @@ public class LogSegmentFiles {
private final Path offsetIndex;
private final Path timeIndex;
private final @Nullable Path writerIdIndex;
- // TODO add leader epoch index after introduce leader epoch.
+ // TODOadd leader epoch index after introduce leader epoch.
public LogSegmentFiles(
Path logSegment, Path offsetIndex, Path timeIndex, @Nullable Path writerIdIndex) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
index cdde3842d..0b425b8fc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
@@ -227,7 +227,7 @@ public class LogTieringTask implements Runnable {
}
/**
- * Copy the given log segments to remote and add the successfully copied segment to the {@code
+ * Copy the given log segments to remote andadd the successfully copied segment to the {@code
* copiedSegments} parameter.
*
* @return the end offset of the last segment copied to remote.
@@ -347,7 +347,7 @@ public class LogTieringTask implements Runnable {
// the commit failed, it means the commit snapshot is invalid or register zk
// failed, we will revert this commit and delete the remote log manifest
// file.
- // TODO: add the fail reason in the future.
+ // TODO:add the fail reason in the future.
LOG.error(
"Commit remote log manifest failed for table bucket {}. We will delete the"
+ " written remote log manifest file",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
new file mode 100644
index 000000000..f56429bf1
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.metadata;
+
+import org.apache.fluss.annotation.VisibleForTesting;
+import org.apache.fluss.exception.SchemaNotExistException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.server.coordinator.MetadataManager;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.Cache;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.CacheBuilder;
+import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.UncheckedExecutionException;
+import org.apache.fluss.utils.MapUtils;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Caches and manages schema metadata for tables, providing methods to update and query schema
+ * information. Unlike other metadata cached all in {@link ServerMetadataCache}, only required
+ * schemas are cached here due to {@link Schema} being a large object. Including two kind of schema:
+ * 1. latest schema of each subscribed table, updated by UpdateMetadata request. 2. history schemas
+ * of each table, updated by lookup from tablet server.
+ */
+public class SchemaMetadataManager {
+
+ private MetadataManager metadataManager;
+ private final Map<TablePath, SchemaInfo> latestSchemaByTablePath;
+ private final Map<TablePath, Integer> subscriberCounters;
+ private final Cache<TableSchemaKey, Schema> schemaCache;
+
+ public SchemaMetadataManager(MetadataManager metadataManager) {
+ this.metadataManager = metadataManager;
+ // thread safe is guaranteed by subscriberCounters.
+ this.subscriberCounters = MapUtils.newConcurrentHashMap();
+ this.latestSchemaByTablePath = MapUtils.newConcurrentHashMap();
+ this.schemaCache = CacheBuilder.newBuilder().maximumSize(100).build();
+ }
+
+ public SchemaGetter subscribeWithInitialSchema(
+ TablePath tablePath, int initialSchemaId, Schema initialSchema) {
+ subscriberCounters.compute(
+ tablePath,
+ (key, oldValue) -> {
+ updateSchema(tablePath, initialSchemaId, initialSchema);
+ if (oldValue == null) {
+ return 1;
+ } else {
+ return oldValue + 1;
+ }
+ });
+ return new SchemaMetadataSubscriberImpl(tablePath);
+ }
+
+ public void updateLatestSchema(TablePath tablePath, short schemaId, Schema schema) {
+ // only update if tablePath is subscribed.
+ subscriberCounters.computeIfPresent(
+ tablePath,
+ (key, oldValue) -> {
+ updateSchema(tablePath, schemaId, schema);
+ return oldValue;
+ });
+ }
+
+ private void unsubscribe(TablePath tablePath) {
+ subscriberCounters.compute(
+ tablePath,
+ (key, oldValue) -> {
+ if (oldValue != null && oldValue > 1) {
+ return oldValue - 1;
+ }
+ latestSchemaByTablePath.remove(tablePath);
+ return null;
+ });
+ }
+
+ private Schema getFlussSchema(TablePath tablePath, int schemaId) throws ExecutionException {
+ return schemaCache.get(
+ new TableSchemaKey(tablePath, schemaId),
+ () -> {
+ SchemaInfo schemaInfo = latestSchemaByTablePath.get(tablePath);
+ if (schemaInfo != null && schemaInfo.getSchemaId() == schemaId) {
+ return latestSchemaByTablePath.get(tablePath).getSchema();
+ } else {
+ return metadataManager.getSchemaById(tablePath, schemaId).getSchema();
+ }
+ });
+ }
+
+ private void updateSchema(TablePath tablePath, int schemaId, Schema schema) {
+ latestSchemaByTablePath.compute(
+ tablePath,
+ (key, oldValue) -> {
+ if (oldValue == null || oldValue.getSchemaId() < schemaId) {
+ return new SchemaInfo(schema, schemaId);
+ } else {
+ return oldValue;
+ }
+ });
+
+ schemaCache.put(new TableSchemaKey(tablePath, schemaId), schema);
+ }
+
+ @VisibleForTesting
+ public Map<TablePath, SchemaInfo> getLatestSchemaByTablePath() {
+ return latestSchemaByTablePath;
+ }
+
+ @VisibleForTesting
+ Map<TablePath, Integer> getSubscriberCounters() {
+ return subscriberCounters;
+ }
+
+ private static class TableSchemaKey {
+ private final TablePath tablePath;
+ private final int schemaId;
+
+ TableSchemaKey(TablePath tablePath, int schemaId) {
+ this.tablePath = tablePath;
+ this.schemaId = schemaId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TableSchemaKey)) {
+ return false;
+ }
+ TableSchemaKey that = (TableSchemaKey) o;
+ return schemaId == that.schemaId && Objects.equals(tablePath, that.tablePath);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tablePath, schemaId);
+ }
+ }
+
+ private class SchemaMetadataSubscriberImpl implements SchemaGetter {
+ private final TablePath tablePath;
+ private volatile boolean released;
+
+ public SchemaMetadataSubscriberImpl(TablePath tablePath) {
+ this.tablePath = tablePath;
+ this.released = false;
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ return latestSchemaByTablePath.get(tablePath);
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ // todo: support with get table.
+ try {
+ return getFlussSchema(tablePath, schemaId);
+ } catch (ExecutionException | UncheckedExecutionException executionException) {
+ Throwable cause = executionException.getCause();
+ if (cause instanceof SchemaNotExistException) {
+ throw (SchemaNotExistException) cause;
+ } else {
+ throw new RuntimeException(cause);
+ }
+ }
+ }
+
+ @Override
+ public void release() {
+ if (!released) {
+ unsubscribe(tablePath);
+ released = true;
+ }
+ }
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
index 79a22621b..6a11a53ae 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
@@ -21,6 +21,9 @@ import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.cluster.ServerNode;
import org.apache.fluss.cluster.TabletServerInfo;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -65,9 +68,13 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
private final MetadataManager metadataManager;
+ private final SchemaMetadataManager schemaMetadataManager;
+
+ // todo: replace this in test with schemaMetadataManager.
public TabletServerMetadataCache(MetadataManager metadataManager) {
this.serverMetadataSnapshot = ServerMetadataSnapshot.empty();
this.metadataManager = metadataManager;
+ this.schemaMetadataManager = new SchemaMetadataManager(metadataManager);
}
@Override
@@ -133,6 +140,17 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
}
}
+ public SchemaGetter subscribeWithInitialSchema(
+ TablePath tablePath, int initialSchemaId, Schema initialSchema) {
+ return schemaMetadataManager.subscribeWithInitialSchema(
+ tablePath, initialSchemaId, initialSchema);
+ }
+
+ public void updateLatestSchema(TablePath tablePath, SchemaInfo schemaInfo) {
+ schemaMetadataManager.updateLatestSchema(
+ tablePath, (short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+
public Optional<PartitionMetadata> getPartitionMetadata(PhysicalTablePath partitionPath) {
TablePath tablePath = partitionPath.getTablePath();
String partitionName = partitionPath.getPartitionName();
@@ -185,6 +203,13 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
TableInfo tableInfo = tableMetadata.getTableInfo();
TablePath tablePath = tableInfo.getTablePath();
long tableId = tableInfo.getTableId();
+ // Update schema metadata.
+ // todo: apply schema id and schema info if needs
+ int schemaId = tableInfo.getSchemaId();
+ Schema schema = tableInfo.getSchema();
+ schemaMetadataManager.updateLatestSchema(
+ tablePath, (short) schemaId, schema);
+
if (tableId == DELETED_TABLE_ID) {
Long removedTableId = tableIdByPath.remove(tablePath);
if (removedTableId != null) {
@@ -410,4 +435,9 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
bucketMetadataMapForPartitions);
});
}
+
+ @VisibleForTesting
+ public SchemaMetadataManager getSchemaMetadataManager() {
+ return schemaMetadataManager;
+ }
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
index d3b5d63c9..251d670b6 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
@@ -77,7 +77,7 @@ public class AdjustIsrManager {
public CompletableFuture<LeaderAndIsr> submit(
TableBucket tableBucket, LeaderAndIsr leaderAndIsr) {
- // TODO add coordinatorEpoch.
+ // TODOadd coordinatorEpoch.
CompletableFuture<LeaderAndIsr> future = new CompletableFuture<>();
AdjustIsrItem adjustIsrItem = new AdjustIsrItem(tableBucket, leaderAndIsr, future);
boolean enqueued = unsentAdjustIsrMap.putIfAbsent(tableBucket, adjustIsrItem) == null;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index 3b7ac5b7f..ad2dcb24d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -33,7 +33,7 @@ import org.apache.fluss.exception.NotLeaderOrFollowerException;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
-import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -74,6 +74,7 @@ import org.apache.fluss.server.log.LogTablet;
import org.apache.fluss.server.log.checkpoint.OffsetCheckpointFile;
import org.apache.fluss.server.log.remote.RemoteLogManager;
import org.apache.fluss.server.metadata.ServerMetadataCache;
+import org.apache.fluss.server.metadata.TabletServerMetadataCache;
import org.apache.fluss.server.metrics.group.BucketMetricGroup;
import org.apache.fluss.server.metrics.group.TableMetricGroup;
import org.apache.fluss.server.metrics.group.TabletServerMetricGroup;
@@ -86,7 +87,6 @@ import org.apache.fluss.server.zk.ZkSequenceIDCounter;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
import org.apache.fluss.server.zk.data.ZkData;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.FlussPaths;
import org.apache.fluss.utils.IOUtils;
@@ -105,7 +105,6 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -167,7 +166,7 @@ public final class Replica {
/** The manger to manger the isr expand and shrink. */
private final AdjustIsrManager adjustIsrManager;
- private final Schema schema;
+ private final SchemaGetter schemaGetter;
private final TableConfig tableConfig;
// logFormat and arrowCompressionInfo are used in hot-path, so cache them here.
private final LogFormat logFormat;
@@ -216,7 +215,7 @@ public final class Replica {
DelayedOperationManager<DelayedFetchLog> delayedFetchLogManager,
AdjustIsrManager adjustIsrManager,
SnapshotContext snapshotContext,
- ServerMetadataCache metadataCache,
+ TabletServerMetadataCache metadataCache,
FatalErrorHandler fatalErrorHandler,
BucketMetricGroup bucketMetricGroup,
TableInfo tableInfo,
@@ -235,7 +234,11 @@ public final class Replica {
this.adjustIsrManager = adjustIsrManager;
this.fatalErrorHandler = fatalErrorHandler;
this.bucketMetricGroup = bucketMetricGroup;
- this.schema = tableInfo.getSchema();
+ this.schemaGetter =
+ metadataCache.subscribeWithInitialSchema(
+ physicalPath.getTablePath(),
+ tableInfo.getSchemaId(),
+ tableInfo.getSchema());
this.tableConfig = tableInfo.getTableConfig();
this.logFormat = tableConfig.getLogFormat();
this.arrowCompressionInfo = tableConfig.getArrowCompressionInfo();
@@ -286,10 +289,6 @@ public final class Replica {
return kvManager != null;
}
- public RowType getRowType() {
- return schema.getRowType();
- }
-
public ArrowCompressionInfo getArrowCompressionInfo() {
return arrowCompressionInfo;
}
@@ -374,6 +373,10 @@ public final class Replica {
return bucketMetricGroup.getTableMetricGroup();
}
+ public LogFormat getLogFormat() {
+ return logFormat;
+ }
+
public void makeLeader(NotifyLeaderAndIsrData data) throws IOException {
boolean leaderHWIncremented =
inWriteLock(
@@ -491,39 +494,11 @@ public final class Replica {
// drop log then
logManager.dropLog(tableBucket);
// close the closeable registry
+ IOUtils.closeQuietly(schemaGetter::release);
IOUtils.closeQuietly(closeableRegistry);
});
}
- public void checkProjection(@Nullable int[] projectedFields) {
- if (projectedFields != null) {
- if (logFormat != LogFormat.ARROW) {
- throw new InvalidColumnProjectionException(
- String.format(
- "Column projection is only supported for ARROW format, but the table %s is %s format.",
- physicalPath.getTablePath(), logFormat));
- }
- int fieldCount = schema.getColumns().size();
- int prev = -1;
- for (int i : projectedFields) {
- if (i <= prev) {
- throw new InvalidColumnProjectionException(
- "The projection indexes should be in field order, but is "
- + Arrays.toString(projectedFields));
- }
- if (i >= fieldCount) {
- throw new InvalidColumnProjectionException(
- "Projected fields "
- + Arrays.toString(projectedFields)
- + " is out of bound for schema with "
- + fieldCount
- + " fields.");
- }
- prev = i;
- }
- }
- }
-
public LogOffsetSnapshot fetchOffsetSnapshot(boolean fetchOnlyFromLeader) throws IOException {
return inReadLock(
leaderIsrUpdateLock,
@@ -675,7 +650,7 @@ public final class Replica {
downloadKvSnapshots(completedSnapshot, tabletDir.toPath());
// as we have downloaded kv files into the tablet dir, now, we can load it
- kvTablet = kvManager.loadKv(tabletDir);
+ kvTablet = kvManager.loadKv(tabletDir, schemaGetter);
checkNotNull(kvTablet, "kv tablet should not be null.");
restoreStartOffset = completedSnapshot.getLogOffset();
@@ -684,6 +659,7 @@ public final class Replica {
"No snapshot found for {} of {}, restore from log.",
tableBucket,
physicalPath);
+
// actually, kv manager always create a kv tablet since we will drop the kv
// if it exists before init kv tablet
kvTablet =
@@ -692,7 +668,7 @@ public final class Replica {
tableBucket,
logTablet,
tableConfig.getKvFormat(),
- schema,
+ schemaGetter,
tableConfig,
arrowCompressionInfo);
}
@@ -774,7 +750,8 @@ public final class Replica {
logTablet,
startRecoverLogOffset,
recoverContext,
- tableConfig.getKvFormat());
+ tableConfig.getKvFormat(),
+ schemaGetter);
kvRecoverHelper.recover();
} catch (Exception e) {
throw new KvStorageException(
@@ -1540,7 +1517,7 @@ public final class Replica {
List<Integer> isrToSend = new ArrayList<>(isrState.isr());
isrToSend.add(newInSyncReplicaId);
- // TODO add server epoch to isr.
+ // TODOadd server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
@@ -1562,7 +1539,7 @@ public final class Replica {
// erroneously advance the HW if the `AdjustIsr` were to fail. Hence, the "maximal ISR"
// for `PendingShrinkIsr` is the current ISR.
- // TODO add server epoch to isr.
+ // TODOadd server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
@@ -1937,4 +1914,9 @@ public final class Replica {
public List<Integer> getIsr() {
return isrState.isr();
}
+
+ @VisibleForTesting
+ public SchemaGetter getSchemaGetter() {
+ return schemaGetter;
+ }
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index 8a5f85615..9f6e4d8ad 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -21,6 +21,7 @@ import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.FencedLeaderEpochException;
+import org.apache.fluss.exception.InvalidColumnProjectionException;
import org.apache.fluss.exception.InvalidCoordinatorException;
import org.apache.fluss.exception.InvalidRequiredAcksException;
import org.apache.fluss.exception.LogOffsetOutOfRangeException;
@@ -29,6 +30,7 @@ import org.apache.fluss.exception.NotLeaderOrFollowerException;
import org.apache.fluss.exception.StorageException;
import org.apache.fluss.exception.UnknownTableOrBucketException;
import org.apache.fluss.fs.FsPath;
+import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
@@ -839,7 +841,7 @@ public class ReplicaManager {
* 2. Stop fetchers for these replicas so that no more data can be added by the replica fetcher threads.
* 3. Truncate the log and checkpoint offsets for these replicas.
* 4. Clear the delayed produce in the purgatory.
- * 5. If the server is not shutting down, add the fetcher to the new leaders.
+ * 5. If the server is not shutting down,add the fetcher to the new leaders.
* </pre>
*/
private void makeFollowers(
@@ -1066,12 +1068,20 @@ public class ReplicaManager {
"Fetching log record for replica {}, offset {}",
tb,
fetchReqInfo.getFetchOffset());
- replica.checkProjection(fetchReqInfo.getProjectFields());
+ // todo: change here to modified project fields.
+ if (fetchReqInfo.getProjectFields() != null
+ && replica.getLogFormat() != LogFormat.ARROW) {
+ throw new InvalidColumnProjectionException(
+ String.format(
+ "Column projection is only supported for ARROW format, but the table %s is %s format.",
+ replica.getTablePath(), replica.getLogFormat()));
+ }
+
fetchParams.setCurrentFetch(
tb.getTableId(),
fetchOffset,
adjustedMaxBytes,
- replica.getRowType(),
+ replica.getSchemaGetter(),
replica.getArrowCompressionInfo(),
fetchReqInfo.getProjectFields());
LogReadInfo readInfo = replica.fetchRecords(fetchParams);
@@ -1522,6 +1532,7 @@ public class ReplicaManager {
PhysicalTablePath physicalTablePath = data.getPhysicalTablePath();
TablePath tablePath = physicalTablePath.getTablePath();
TableInfo tableInfo = getTableInfo(zkClient, tablePath);
+
boolean isKvTable = tableInfo.hasPrimaryKey();
BucketMetricGroup bucketMetricGroup =
serverMetricGroup.addTableBucketMetricGroup(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
index 9e4479b3e..06e94e9ae 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
@@ -225,7 +225,7 @@ public class ReplicaFetcherManager {
fetcherThread.getLeader().leaderServerId(),
initialFetchStatusMap);
} catch (InterruptedException e) {
- LOG.error("Interrupted while add buckets to fetcher threads.", e);
+ LOG.error("Interrupted whileadd buckets to fetcher threads.", e);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
index 398bfb6ff..76195a0df 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
@@ -80,7 +80,7 @@ final class ReplicaFetcherThread extends ShutdownableThread {
private final LeaderEndpoint leader;
private final int fetchBackOffMs;
- // manually add timout logic in here, todo remove this timeout logic if
+ // manuallyadd timout logic in here, todo remove this timeout logic if
// we support global request timeout in #279
private final int timeoutSeconds = 30;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
index c89fca6f7..062eec73d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
@@ -65,7 +65,7 @@ public class RpcGatewayManager<T extends RpcGateway> implements AutoCloseable {
}
/**
- * Add a server to the manager. It'll create a new gateway for the server and add it to the
+ * Add a server to the manager. It'll create a new gateway for the server andadd it to the
* manager. If the server has already existed, it'll remove the already existing server before
* adding the new one.
*/
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
index 78c1f587b..165fd23e9 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
@@ -22,6 +22,7 @@ import org.apache.fluss.cluster.ServerNode;
import org.apache.fluss.cluster.ServerType;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.cluster.AlterConfigOpType;
+import org.apache.fluss.config.cluster.ColumnPositionType;
import org.apache.fluss.config.cluster.ConfigEntry;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.token.ObtainedSecurityToken;
@@ -52,6 +53,7 @@ import org.apache.fluss.rpc.entity.ProduceLogResultForBucket;
import org.apache.fluss.rpc.entity.PutKvResultForBucket;
import org.apache.fluss.rpc.messages.AdjustIsrRequest;
import org.apache.fluss.rpc.messages.AdjustIsrResponse;
+import org.apache.fluss.rpc.messages.AlterTableRequest;
import org.apache.fluss.rpc.messages.CommitKvSnapshotRequest;
import org.apache.fluss.rpc.messages.CommitLakeTableSnapshotRequest;
import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest;
@@ -79,6 +81,7 @@ import org.apache.fluss.rpc.messages.NotifyLeaderAndIsrRequest;
import org.apache.fluss.rpc.messages.NotifyLeaderAndIsrResponse;
import org.apache.fluss.rpc.messages.NotifyRemoteLogOffsetsRequest;
import org.apache.fluss.rpc.messages.PbAclInfo;
+import org.apache.fluss.rpc.messages.PbAddColumn;
import org.apache.fluss.rpc.messages.PbAdjustIsrReqForBucket;
import org.apache.fluss.rpc.messages.PbAdjustIsrReqForTable;
import org.apache.fluss.rpc.messages.PbAdjustIsrRespForBucket;
@@ -89,6 +92,7 @@ import org.apache.fluss.rpc.messages.PbCreateAclRespInfo;
import org.apache.fluss.rpc.messages.PbDescribeConfig;
import org.apache.fluss.rpc.messages.PbDropAclsFilterResult;
import org.apache.fluss.rpc.messages.PbDropAclsMatchingAcl;
+import org.apache.fluss.rpc.messages.PbDropColumn;
import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket;
import org.apache.fluss.rpc.messages.PbFetchLogReqForTable;
import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket;
@@ -101,6 +105,7 @@ import org.apache.fluss.rpc.messages.PbLakeTableSnapshotInfo;
import org.apache.fluss.rpc.messages.PbListOffsetsRespForBucket;
import org.apache.fluss.rpc.messages.PbLookupReqForBucket;
import org.apache.fluss.rpc.messages.PbLookupRespForBucket;
+import org.apache.fluss.rpc.messages.PbModifyColumn;
import org.apache.fluss.rpc.messages.PbNotifyLakeTableOffsetReqForBucket;
import org.apache.fluss.rpc.messages.PbNotifyLeaderAndIsrReqForBucket;
import org.apache.fluss.rpc.messages.PbNotifyLeaderAndIsrRespForBucket;
@@ -115,6 +120,7 @@ import org.apache.fluss.rpc.messages.PbPutKvReqForBucket;
import org.apache.fluss.rpc.messages.PbPutKvRespForBucket;
import org.apache.fluss.rpc.messages.PbRemoteLogSegment;
import org.apache.fluss.rpc.messages.PbRemotePathAndLocalFile;
+import org.apache.fluss.rpc.messages.PbRenameColumn;
import org.apache.fluss.rpc.messages.PbServerNode;
import org.apache.fluss.rpc.messages.PbStopReplicaReqForBucket;
import org.apache.fluss.rpc.messages.PbStopReplicaRespForBucket;
@@ -160,6 +166,8 @@ import org.apache.fluss.server.metadata.TableMetadata;
import org.apache.fluss.server.zk.data.BucketSnapshot;
import org.apache.fluss.server.zk.data.LakeTableSnapshot;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
+import org.apache.fluss.utils.json.DataTypeJsonSerde;
+import org.apache.fluss.utils.json.JsonSerdeUtils;
import javax.annotation.Nullable;
@@ -260,13 +268,87 @@ public class ServerRpcMessageUtils {
}
}
- public static List<TableChange> toTableChanges(List<PbAlterConfig> alterConfigs) {
+ public static List<TableChange> toAlterTableConfigChanges(List<PbAlterConfig> alterConfigs) {
return alterConfigs.stream()
.filter(Objects::nonNull)
.map(ServerRpcMessageUtils::toTableChange)
.collect(Collectors.toList());
}
+ public static List<TableChange> toAlterTableSchemaChanges(AlterTableRequest request) {
+ List<TableChange> alterTableSchemaChanges = new ArrayList<>();
+ alterTableSchemaChanges.addAll(toAddColumns(request.getAddColumnsList()));
+ alterTableSchemaChanges.addAll(toDropColumns(request.getDropColumnsList()));
+ alterTableSchemaChanges.addAll(toRenameColumns(request.getRenameColumnsList()));
+ alterTableSchemaChanges.addAll(toModifyColumns(request.getModifyColumnsList()));
+ return alterTableSchemaChanges;
+ }
+
+ public static List<TableChange> toAddColumns(List<PbAddColumn> addColumns) {
+ return addColumns.stream()
+ .filter(Objects::nonNull)
+ .map(
+ pbAddColumn ->
+ TableChange.addColumn(
+ pbAddColumn.getColumnName(),
+ JsonSerdeUtils.readValue(
+ pbAddColumn.getDataTypeJson(),
+ DataTypeJsonSerde.INSTANCE),
+ pbAddColumn.hasComment() ? pbAddColumn.getComment() : null,
+ toColumnPosition(pbAddColumn.getColumnPositionType())))
+ .collect(Collectors.toList());
+ }
+
+ public static List<TableChange.SchemaChange> toDropColumns(List<PbDropColumn> dropColumns) {
+ return dropColumns.stream()
+ .filter(Objects::nonNull)
+ .map(pbDropColumn -> TableChange.dropColumn(pbDropColumn.getColumnName()))
+ .collect(Collectors.toList());
+ }
+
+ public static List<TableChange.SchemaChange> toRenameColumns(
+ List<PbRenameColumn> alterColumns) {
+ return alterColumns.stream()
+ .filter(Objects::nonNull)
+ .map(
+ pbRenameColumn ->
+ TableChange.renameColumn(
+ pbRenameColumn.getOldColumnName(),
+ pbRenameColumn.getNewColumnName()))
+ .collect(Collectors.toList());
+ }
+
+ public static List<TableChange.SchemaChange> toModifyColumns(
+ List<PbModifyColumn> modifyColumns) {
+ return modifyColumns.stream()
+ .filter(Objects::nonNull)
+ .map(
+ pbModifyColumn ->
+ TableChange.modifyColumn(
+ pbModifyColumn.getColumnName(),
+ JsonSerdeUtils.readValue(
+ pbModifyColumn.getDataTypeJson(),
+ DataTypeJsonSerde.INSTANCE),
+ pbModifyColumn.hasComment()
+ ? pbModifyColumn.getComment()
+ : null,
+ pbModifyColumn.hasColumnPositionType()
+ ? toColumnPosition(
+ pbModifyColumn.getColumnPositionType())
+ : null))
+ .collect(Collectors.toList());
+ }
+
+ private static @Nullable TableChange.ColumnPosition toColumnPosition(int columnPositionType) {
+ ColumnPositionType opType = ColumnPositionType.from(columnPositionType);
+ switch (opType) {
+ case LAST:
+ return TableChange.ColumnPosition.last();
+ default:
+ throw new IllegalArgumentException("Unsupported column position type " + opType);
+ }
+ }
+
public static MetadataResponse buildMetadataResponse(
@Nullable ServerNode coordinatorServer,
Set<ServerNode> aliveTabletServers,
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
index ea6aead7b..270d334ee 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
@@ -35,7 +35,7 @@ public interface Timer {
* Add a new task to this executor. It will be executed after the task's delay (beginning from
* the time of submission).
*
- * @param timerTask the task to add
+ * @param timerTask the task toadd
*/
void add(TimerTask timerTask);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
index 1dab9f229..934777c93 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
@@ -108,8 +108,8 @@ import java.util.concurrent.atomic.AtomicInteger;
* while priority queue based timers takes O(log N) for both insert and delete where N is the number
* of items in the queue.
*
- * <p>This class is not thread-safe. There should not be any add calls while advanceClock is
- * executing. It is caller's responsibility to enforce it. Simultaneous add calls are thread-safe.
+ * <p>This class is not thread-safe. There should not be anyadd calls while advanceClock is
+ * executing. It is caller's responsibility to enforce it. Simultaneousadd calls are thread-safe.
*/
@NotThreadSafe
final class TimingWheel {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
index 1c7401b8a..e27ffd9c3 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
@@ -810,16 +810,18 @@ public class ZooKeeperClient implements AutoCloseable {
/** Register schema to ZK metadata and return the schema id. */
public int registerSchema(TablePath tablePath, Schema schema) throws Exception {
- int currentSchemaId = getCurrentSchemaId(tablePath);
+ return registerSchema(tablePath, schema, getCurrentSchemaId(tablePath) + 1);
+ }
+
+ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) throws Exception {
// increase schema id.
- currentSchemaId++;
- String path = SchemaZNode.path(tablePath, currentSchemaId);
+ String path = SchemaZNode.path(tablePath, schemaId);
zkClient.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(path, SchemaZNode.encode(schema));
- LOG.info("Registered new schema version {} for table {}.", currentSchemaId, tablePath);
- return currentSchemaId;
+ LOG.info("Registered new schema version {} for table {}.", schemaId, tablePath);
+ return schemaId;
}
/** Get the specific schema by schema id in ZK metadata. */
@@ -1130,7 +1132,7 @@ public class ZooKeeperClient implements AutoCloseable {
.forPath(
AclChangeNotificationNode.pathPrefix(),
AclChangeNotificationNode.encode(resource));
- LOG.info("add acl change notification for resource {} ", resource);
+ LOG.info("addColumn acl change notification for resource {} ", resource);
}
public Map<String, String> fetchEntityConfig() throws Exception {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java
index 024707d8f..702400184 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java
@@ -25,6 +25,7 @@ import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.security.acl.Resource;
import org.apache.fluss.security.acl.ResourceType;
import org.apache.fluss.utils.json.JsonSerdeUtils;
+import org.apache.fluss.utils.types.Tuple2;
import javax.annotation.Nullable;
@@ -130,6 +131,7 @@ public final class ZkData {
* <p>/metadata/databases/[databaseName]/tables/[tableName]/schemas
*/
public static final class SchemasZNode {
+
public static String path(TablePath tablePath) {
return TableZNode.path(tablePath) + "/schemas";
}
@@ -142,6 +144,32 @@ public final class ZkData {
* <p>/metadata/databases/[databaseName]/tables/[tableName]/schemas/[schemaId]
*/
public static final class SchemaZNode {
+
+ @Nullable
+ public static Tuple2<TablePath, Integer> parsePath(String zkPath) {
+ String splitter = "/schemas/";
+ if (!zkPath.contains(splitter)) {
+ return null;
+ }
+
+ String[] split = zkPath.split(splitter);
+ if (split.length != 2) {
+ return null;
+ }
+ TablePath tablePath = TableZNode.parsePath(split[0]);
+ if (tablePath == null) {
+ return null;
+ }
+
+ int schemaId;
+ try {
+ schemaId = Integer.parseInt(split[1]);
+ return Tuple2.of(tablePath, schemaId);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
public static String path(TablePath tablePath, int schemaId) {
return SchemasZNode.path(tablePath) + "/" + schemaId;
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
index af69caffe..abc575fd5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
@@ -487,7 +487,7 @@ public class DefaultAuthorizerTest {
Set<AccessControlEntry> expectedAcls = new HashSet<>();
for (int i = 0; i < 50; i++) {
- // each task represent that add acl to different user on same resource.
+ // each task represent that addColumn acl to different user on same resource.
final AccessControlEntry accessControlEntry =
createAclEntry(String.valueOf(i), "host-1", READ);
int finalI = i;
@@ -550,7 +550,7 @@ public class DefaultAuthorizerTest {
// generate 50 concurrent acl operation tasks.
List<Runnable> concurrentTasks = new ArrayList<>();
for (int i = 0; i < 50; i++) {
- // each task represent that add acl to same user on same resource.
+ // each task represent that addColumn acl to same user on same resource.
Runnable runnable =
() -> {
addAcls(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
index b8fec59ca..4460cfc26 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
@@ -99,13 +99,13 @@ class CompletedSnapshotStoreManagerTest {
CompletedSnapshotStoreManager completedSnapshotStoreManager =
createCompletedSnapshotStoreManager(maxNumberOfSnapshotsToRetain);
- // add snapshots for a series of buckets
+ // addColumn snapshots for a series of buckets
Set<TableBucket> tableBuckets = createTableBuckets(2, 3);
int snapshotNum = 3;
Map<TableBucket, CompletedSnapshot> tableBucketLatestCompletedSnapshots = new HashMap<>();
for (TableBucket tableBucket : tableBuckets) {
- // add some snapshots
+ // addColumn some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
@@ -137,7 +137,7 @@ class CompletedSnapshotStoreManagerTest {
assertThat(completedSnapshot)
.isEqualTo(tableBucketLatestCompletedSnapshots.get(tableBucket));
- // add a new snapshot
+ // addColumn a new snapshot
long snapshotId = completedSnapshot.getSnapshotID() + 1;
completedSnapshot = KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshotId);
addCompletedSnapshot(completedSnapshotStoreManager, completedSnapshot);
@@ -169,7 +169,7 @@ class CompletedSnapshotStoreManagerTest {
Set<TableBucket> tableBuckets = createTableBuckets(1, 2);
int snapshotNum = 3;
for (TableBucket tableBucket : tableBuckets) {
- // add some snapshots
+ // addColumn some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
index bd4467c0a..581edafcc 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
@@ -69,14 +69,14 @@ class CoordinatorChannelManagerTest {
// now, shouldn't send as we already remove the tablet server
checkSendRequest(coordinatorChannelManager, server0.id(), false);
- // test add tablet server
- // before add, shouldn't send
+ // test addColumn tablet server
+ // before addColumn, shouldn't send
ServerNode server1 = tabletServersNode.get(1);
checkSendRequest(coordinatorChannelManager, server1.id(), false);
coordinatorChannelManager.addTabletServer(server1);
- // after add the tablet server, should send
+ // after addColumn the tablet server, should send
// try to send message
checkSendRequest(coordinatorChannelManager, server1.id(), true);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
index adbb14768..2549c597e 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
@@ -28,14 +28,19 @@ import org.apache.fluss.metadata.DatabaseDescriptor;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableBucketReplica;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePartition;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.rpc.messages.AdjustIsrResponse;
... 2654 lines suppressed ...
(fluss) 02/02: [server][client] Improve the design and performance of ADD COLUMN operation
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
commit 07721fe6ecf033fb5be550d23cc82ce7fdc267e7
Author: Jark Wu <ja...@apache.org>
AuthorDate: Sun Nov 30 16:32:04 2025 +0800
[server][client] Improve the design and performance of ADD COLUMN operation
1. Make lookup requests async.
2. Introduce ServerProjectionCache to share ProjectionInfo in TabletServer level.
3. Revert FlinkAsFlussRow changes, and introduce PaddingRow for the padding null columns for schema changes.
4. Make PutKv zero-copy to persist kv records.
5. Rename SchemaMetadataManager to ServerSchemaCache and use TableId instead of TablePath to track schemas which is safer.
6. Revert source related changes to still include the projectedFieldIndexes parameter.
---
.../org/apache/fluss/client/admin/FlussAdmin.java | 22 +--
.../fluss/client/lookup/AbstractLookuper.java | 180 +++++++++++++++++++++
.../fluss/client/lookup/PrefixKeyLookuper.java | 66 ++------
.../apache/fluss/client/lookup/PrefixLookup.java | 5 +-
.../fluss/client/lookup/PrimaryKeyLookuper.java | 61 ++-----
.../fluss/client/metadata/ClientSchemaGetter.java | 69 ++++++--
.../org/apache/fluss/client/table/FlussTable.java | 6 +-
.../client/table/scanner/batch/BatchScanUtils.java | 3 +-
.../table/scanner/batch/LimitBatchScanner.java | 22 ++-
.../table/scanner/batch/SnapshotFilesReader.java | 22 ++-
.../client/table/scanner/log/BucketScanStatus.java | 3 +-
.../fluss/client/table/scanner/log/LogFetcher.java | 3 +-
.../client/table/writer/AbstractTableWriter.java | 2 -
.../fluss/client/utils/ClientRpcMessageUtils.java | 65 ++++----
.../fluss/client/write/RecordAccumulator.java | 3 +-
.../java/org/apache/fluss/client/write/Sender.java | 6 +-
.../fluss/client/admin/FlussAdminITCase.java | 37 ++++-
.../batch/KvSnapshotBatchScannerITCase.java | 4 +-
.../scanner/log/DefaultCompletedFetchTest.java | 3 +-
.../table/scanner/log/LogFetchCollectorTest.java | 3 +-
.../client/table/scanner/log/LogFetcherTest.java | 1 +
.../fluss/client/write/RecordAccumulatorTest.java | 1 -
.../org/apache/fluss/exception/ApiException.java | 16 +-
.../fluss/exception/SchemaChangeException.java | 4 +
.../java/org/apache/fluss/metadata/Schema.java | 15 +-
.../org/apache/fluss/metadata/SchemaGetter.java | 50 +++++-
.../org/apache/fluss/metadata/TableChange.java | 2 +-
.../java/org/apache/fluss/metadata/TableInfo.java | 5 +
.../java/org/apache/fluss/record/BinaryValue.java | 62 +++++++
.../apache/fluss/record/DefaultLogRecordBatch.java | 96 ++++-------
.../org/apache/fluss/record/FileLogProjection.java | 108 +++++--------
.../apache/fluss/record/LogRecordReadContext.java | 57 +++----
.../fluss/record/ProjectionPushdownCache.java | 90 +++++++++++
.../main/java/org/apache/fluss/row/PaddingRow.java | 133 +++++++++++++++
.../java/org/apache/fluss/row/ProjectedRow.java | 6 +-
.../fluss/row/decode/FixedSchemaDecoder.java | 95 +++++++++++
.../org/apache/fluss/row/decode/RowDecoder.java | 5 +-
.../org/apache/fluss/row/encode/ValueDecoder.java | 16 +-
.../org/apache/fluss/row/encode/ValueEncoder.java | 25 +--
.../java/org/apache/fluss/utils/Projection.java | 6 +-
.../java/org/apache/fluss/utils/SchemaUtil.java | 46 ++----
.../apache/fluss/record/FileLogProjectionTest.java | 55 ++-----
.../java/org/apache/fluss/record/LogTestBase.java | 1 -
.../apache/fluss/record/TestingSchemaGetter.java | 12 +-
.../java/org/apache/fluss/row/PaddingRowTest.java | 79 +++++++++
.../org/apache/fluss/row/ProjectedRowTest.java | 1 +
.../src/test/resources/log4j2-test.properties | 2 +-
.../apache/fluss/flink/row/FlinkAsFlussRow.java | 100 ++----------
.../sink/serializer/FlussSerializationSchema.java | 7 +-
.../serializer/RowDataSerializationSchema.java | 76 ++++++++-
.../sink/serializer/SerializerInitContextImpl.java | 2 +-
.../fluss/flink/sink/writer/FlinkSinkWriter.java | 21 +--
.../org/apache/fluss/flink/source/FlinkSource.java | 7 +-
.../fluss/flink/source/FlinkTableSource.java | 1 +
.../org/apache/fluss/flink/source/FlussSource.java | 4 +
.../fluss/flink/source/FlussSourceBuilder.java | 1 +
.../source/lookup/FlinkAsyncLookupFunction.java | 29 +++-
.../flink/source/lookup/FlinkLookupFunction.java | 27 +++-
.../flink/source/reader/FlinkSourceReader.java | 4 +
.../source/reader/FlinkSourceSplitReader.java | 67 ++++----
.../apache/fluss/flink/utils/FlinkConversions.java | 12 --
.../flink/utils/FlussRowToFlinkRowConverter.java | 39 ++---
.../fluss/flink/row/FlinkAsFlussRowTest.java | 3 +-
.../source/lookup/FlinkLookupFunctionTest.java | 12 +-
.../flink/source/reader/FlinkSourceReaderTest.java | 1 +
.../source/reader/FlinkSourceSplitReaderTest.java | 28 +++-
.../fluss/lake/iceberg/conf/HadoopUtils.java | 5 +-
.../iceberg/maintenance/IcebergRewriteITCase.java | 2 +-
.../fluss/rpc/netty/client/ServerConnection.java | 3 +-
fluss-rpc/src/main/proto/FlussApi.proto | 12 +-
.../org/apache/fluss/server/RpcServiceBase.java | 2 +-
.../coordinator/CompletedSnapshotStoreManager.java | 2 +-
.../coordinator/CoordinatorEventProcessor.java | 1 -
.../fluss/server/coordinator/MetadataManager.java | 16 +-
.../fluss/server/coordinator/SchemaUpdate.java | 28 ++--
.../fluss/server/coordinator/UpdateSchema.java | 28 ----
.../event/watcher/TableChangeWatcher.java | 9 --
.../server/entity/BatchRegisterLeadAndIsr.java | 2 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 99 +++++-------
.../java/org/apache/fluss/server/kv/KvTablet.java | 160 ++++++++----------
.../server/kv/partialupdate/PartialUpdater.java | 45 +++---
.../kv/partialupdate/PartialUpdaterCache.java | 9 +-
.../server/kv/rowmerger/DefaultRowMerger.java | 14 +-
.../server/kv/rowmerger/FirstRowRowMerger.java | 8 +-
.../fluss/server/kv/rowmerger/RowMerger.java | 24 +--
.../server/kv/rowmerger/VersionedRowMerger.java | 7 +-
.../server/kv/snapshot/CompletedSnapshotStore.java | 4 +-
.../fluss/server/kv/snapshot/SnapshotLocation.java | 2 +-
.../org/apache/fluss/server/log/FetchParams.java | 8 +-
.../java/org/apache/fluss/server/log/LocalLog.java | 2 -
.../server/log/remote/FsRemoteLogOutputStream.java | 2 +-
.../fluss/server/log/remote/LogSegmentFiles.java | 2 +-
.../fluss/server/log/remote/LogTieringTask.java | 4 +-
...MetadataManager.java => ServerSchemaCache.java} | 87 +++++-----
.../server/metadata/TabletServerMetadataCache.java | 24 ++-
.../fluss/server/replica/AdjustIsrManager.java | 2 +-
.../org/apache/fluss/server/replica/Replica.java | 5 +-
.../fluss/server/replica/ReplicaManager.java | 5 +-
.../replica/fetcher/ReplicaFetcherManager.java | 2 +-
.../replica/fetcher/ReplicaFetcherThread.java | 2 +-
.../fluss/server/utils/RpcGatewayManager.java | 2 +-
.../fluss/server/utils/ServerRpcMessageUtils.java | 2 +-
.../org/apache/fluss/server/utils/timer/Timer.java | 2 +-
.../fluss/server/utils/timer/TimingWheel.java | 4 +-
.../apache/fluss/server/zk/ZooKeeperClient.java | 52 ++++--
.../server/authorizer/DefaultAuthorizerTest.java | 4 +-
.../CompletedSnapshotStoreManagerTest.java | 8 +-
.../coordinator/CoordinatorChannelManagerTest.java | 6 +-
.../fluss/server/coordinator/TableManagerTest.java | 2 -
.../org/apache/fluss/server/kv/KvTabletTest.java | 11 +-
.../server/kv/prewrite/KvPreWriteBufferTest.java | 4 +-
.../server/kv/rowmerger/DefaultRowMergerTest.java | 57 ++++---
.../kv/rowmerger/VersionedRowMergerTest.java | 41 ++---
.../kv/snapshot/CompletedSnapshotStoreTest.java | 2 +-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 2 +-
.../ZooKeeperCompletedSnapshotStoreTest.java | 4 +-
.../fluss/server/log/DroppedTableRecoveryTest.java | 2 +-
.../apache/fluss/server/log/FetchParamsTest.java | 14 +-
.../apache/fluss/server/log/LogManagerTest.java | 6 +-
.../org/apache/fluss/server/log/LogTabletTest.java | 4 +-
.../server/log/remote/RemoteLogTabletTest.java | 4 +-
.../server/metadata/MetadataUpdateITCase.java | 2 +-
...ManagerTest.java => ServerSchemaCacheTest.java} | 62 ++++---
.../fluss/server/replica/AdjustIsrITCase.java | 2 +-
.../fluss/server/replica/AdjustIsrManagerTest.java | 2 +-
.../replica/HighWatermarkPersistenceTest.java | 2 +-
.../replica/RemoveOfflineReplicaFromIsrITCase.java | 2 +-
.../fluss/server/replica/ReplicaManagerTest.java | 21 +--
.../apache/fluss/server/replica/ReplicaTest.java | 26 ++-
.../fluss/server/replica/ReplicaTestBase.java | 12 +-
.../replica/fetcher/ReplicaFetcherThreadTest.java | 2 +-
.../fluss/server/tablet/TabletServiceITCase.java | 6 +-
.../server/testutils/FlussClusterExtension.java | 20 +--
.../fluss/server/zk/ZooKeeperClientTest.java | 4 +-
.../src/test/resources/log4j2-test.properties | 2 +-
fluss-test-coverage/pom.xml | 4 +-
136 files changed, 1820 insertions(+), 1224 deletions(-)
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
index 25bfcfbd8..69070e01c 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
@@ -88,9 +88,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
-import static org.apache.fluss.client.utils.ClientRpcMessageUtils.addPbAlterSchemas;
+import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeAlterTableRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeCreatePartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeDropPartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeListOffsetsRequest;
@@ -249,23 +248,8 @@ public class FlussAdmin implements Admin {
public CompletableFuture<Void> alterTable(
TablePath tablePath, List<TableChange> tableChanges, boolean ignoreIfNotExists) {
tablePath.validate();
- AlterTableRequest request = new AlterTableRequest();
- List<PbAlterConfig> pbFlussTableChanges =
- tableChanges.stream()
- .filter(tableChange -> !(tableChange instanceof TableChange.SchemaChange))
- .map(ClientRpcMessageUtils::toPbAlterConfigs)
- .collect(Collectors.toList());
-
- List<TableChange> schemaChanges =
- tableChanges.stream()
- .filter(tableChange -> tableChange instanceof TableChange.SchemaChange)
- .collect(Collectors.toList());
- addPbAlterSchemas(request, schemaChanges);
- request.addAllConfigChanges(pbFlussTableChanges)
- .setIgnoreIfNotExists(ignoreIfNotExists)
- .setTablePath()
- .setDatabaseName(tablePath.getDatabaseName())
- .setTableName(tablePath.getTableName());
+ AlterTableRequest request =
+ makeAlterTableRequest(tablePath, tableChanges, ignoreIfNotExists);
return gateway.alterTable(request).thenApply(r -> null);
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java
new file mode 100644
index 000000000..6cd93adf7
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.lookup;
+
+import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.memory.MemorySegment;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.decode.FixedSchemaDecoder;
+import org.apache.fluss.utils.CopyOnWriteMap;
+import org.apache.fluss.utils.concurrent.FutureUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+
+/** Abstract lookuper implementation for common methods. */
+abstract class AbstractLookuper implements Lookuper {
+ protected final TableInfo tableInfo;
+
+ protected final MetadataUpdater metadataUpdater;
+
+ protected final LookupClient lookupClient;
+
+ protected final short targetSchemaId;
+
+ private final SchemaGetter schemaGetter;
+
+ /**
+ * Cache for row decoders for different schema ids. Use CopyOnWriteMap for fast access, as it is
+ * not frequently updated.
+ */
+ private final CopyOnWriteMap<Short, FixedSchemaDecoder> decoders;
+
+ AbstractLookuper(
+ TableInfo tableInfo,
+ MetadataUpdater metadataUpdater,
+ LookupClient lookupClient,
+ SchemaGetter schemaGetter) {
+ this.tableInfo = tableInfo;
+ this.metadataUpdater = metadataUpdater;
+ this.lookupClient = lookupClient;
+ this.targetSchemaId = (short) tableInfo.getSchemaId();
+ this.schemaGetter = schemaGetter;
+ this.decoders = new CopyOnWriteMap<>();
+ // initialize the decoder for the same schema
+ this.decoders.put(
+ targetSchemaId,
+ new FixedSchemaDecoder(
+ tableInfo.getTableConfig().getKvFormat(), tableInfo.getSchema()));
+ }
+
+ protected void handleLookupResponse(
+ List<byte[]> result, CompletableFuture<LookupResult> lookupFuture) {
+ List<MemorySegment> valueList = new ArrayList<>(result.size());
+ Set<Short> schemaIdsToRequest = new HashSet<>();
+ boolean allTargetSchema = true;
+ for (byte[] valueBytes : result) {
+ if (valueBytes == null) {
+ continue;
+ }
+ MemorySegment memorySegment = MemorySegment.wrap(valueBytes);
+ short schemaId = memorySegment.getShort(0);
+ if (targetSchemaId != schemaId) {
+ allTargetSchema = false;
+ if (!decoders.containsKey(schemaId)) {
+ schemaIdsToRequest.add(schemaId);
+ }
+ }
+ valueList.add(memorySegment);
+ }
+ // all schema ids are the target schema id, fast path
+ if (allTargetSchema) {
+ lookupFuture.complete(processAllTargetSchemaRows(valueList));
+ return;
+ }
+
+ // all schema ids and decoders are cached in local
+ if (schemaIdsToRequest.isEmpty()) {
+ lookupFuture.complete(processSchemaMismatchedRows(valueList));
+ return;
+ }
+
+ // need to fetch schema infos for schema ids
+ List<CompletableFuture<SchemaInfo>> schemaFutures =
+ new ArrayList<>(schemaIdsToRequest.size());
+ for (short schemaId : schemaIdsToRequest) {
+ CompletableFuture<SchemaInfo> schemaFuture = schemaGetter.getSchemaInfoAsync(schemaId);
+ schemaFutures.add(schemaFuture);
+ }
+ FutureUtils.ConjunctFuture<Collection<SchemaInfo>> allFutures =
+ FutureUtils.combineAll(schemaFutures);
+ // normal async path
+ allFutures.whenComplete(
+ (schemas, error) -> {
+ if (error != null) {
+ lookupFuture.completeExceptionally(
+ new RuntimeException(
+ "Failed to get schema infos for lookup", error));
+ } else {
+ LookupResult lookupResult = processSchemaRequestedRows(schemas, valueList);
+ lookupFuture.complete(lookupResult);
+ }
+ });
+ }
+
+ protected LookupResult processAllTargetSchemaRows(List<MemorySegment> valueList) {
+ FixedSchemaDecoder decoder = decoders.get(targetSchemaId);
+ List<InternalRow> rowList = new ArrayList<>(valueList.size());
+ for (MemorySegment value : valueList) {
+ rowList.add(decoder.decode(value));
+ }
+ return new LookupResult(rowList);
+ }
+
+ protected LookupResult processSchemaMismatchedRows(List<MemorySegment> valueList) {
+ List<InternalRow> rowList = new ArrayList<>(valueList.size());
+ for (MemorySegment value : valueList) {
+ short schemaId = value.getShort(0);
+ FixedSchemaDecoder decoder = decoders.get(schemaId);
+ checkArgument(decoder != null, "Decoder for schema id %s not found", schemaId);
+ InternalRow row = decoder.decode(value);
+ rowList.add(row);
+ }
+ return new LookupResult(rowList);
+ }
+
+ protected LookupResult processSchemaRequestedRows(
+ Collection<SchemaInfo> schemaInfos, List<MemorySegment> valueList) {
+ // build a map from schema id to the target schema decoder
+ Map<Short, Schema> schemaMap = new HashMap<>();
+ for (SchemaInfo schemaInfo : schemaInfos) {
+ schemaMap.put((short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+
+ // process the value list to convert to target schema
+ List<InternalRow> rowList = new ArrayList<>(valueList.size());
+ for (MemorySegment value : valueList) {
+ short schemaId = value.getShort(0);
+ FixedSchemaDecoder decoder =
+ decoders.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema sourceSchema = schemaMap.get(id);
+ return new FixedSchemaDecoder(
+ tableInfo.getTableConfig().getKvFormat(),
+ sourceSchema,
+ tableInfo.getSchema());
+ });
+ InternalRow row = decoder.decode(value);
+ rowList.add(row);
+ }
+ return new LookupResult(rowList);
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
index c9f6308f8..cf658ce80 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
@@ -22,14 +22,11 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
-import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
-import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -49,13 +46,7 @@ import static org.apache.fluss.client.utils.ClientUtils.getPartitionId;
* of the primary key.
*/
@NotThreadSafe
-class PrefixKeyLookuper implements Lookuper {
-
- private final TableInfo tableInfo;
-
- private final MetadataUpdater metadataUpdater;
-
- private final LookupClient lookupClient;
+class PrefixKeyLookuper extends AbstractLookuper {
/** Extract bucket key from prefix lookup key row. */
private final KeyEncoder bucketKeyEncoder;
@@ -68,24 +59,16 @@ class PrefixKeyLookuper implements Lookuper {
*/
private @Nullable final PartitionGetter partitionGetter;
- /** Decode the lookup bytes to result row. */
- private final ValueDecoder kvValueDecoder;
-
- private final SchemaGetter schemaGetter;
-
public PrefixKeyLookuper(
TableInfo tableInfo,
SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient,
List<String> lookupColumnNames) {
+ super(tableInfo, metadataUpdater, lookupClient, schemaGetter);
// sanity check
validatePrefixLookup(tableInfo, lookupColumnNames);
- // initialization
- this.tableInfo = tableInfo;
this.numBuckets = tableInfo.getNumBuckets();
- this.metadataUpdater = metadataUpdater;
- this.lookupClient = lookupClient;
// the row type of the input lookup row
RowType lookupRowType = tableInfo.getRowType().project(lookupColumnNames);
DataLakeFormat lakeFormat = tableInfo.getTableConfig().getDataLakeFormat().orElse(null);
@@ -96,9 +79,6 @@ class PrefixKeyLookuper implements Lookuper {
tableInfo.isPartitioned()
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
- this.kvValueDecoder =
- new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
- this.schemaGetter = schemaGetter;
}
private void validatePrefixLookup(TableInfo tableInfo, List<String> lookupColumns) {
@@ -168,36 +148,22 @@ class PrefixKeyLookuper implements Lookuper {
}
}
+ CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- CompletableFuture<LookupResult> future = new CompletableFuture<>();
-
- CompletableFuture.runAsync(
- () -> {
- try {
- List<byte[]> result =
- lookupClient.prefixLookup(tableBucket, bucketKeyBytes).get();
- List<InternalRow> rowList = new ArrayList<>(result.size());
- for (byte[] valueBytes : result) {
- if (valueBytes == null) {
- continue;
- }
- ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
- InternalRow row;
- if (value.schemaId == tableInfo.getSchemaId()) {
- row = value.row;
+ lookupClient
+ .prefixLookup(tableBucket, bucketKeyBytes)
+ .whenComplete(
+ (result, error) -> {
+ if (error != null) {
+ lookupFuture.completeExceptionally(
+ new RuntimeException(
+ "Failed to perform prefix lookup for table: "
+ + tableInfo.getTablePath(),
+ error));
} else {
- Schema schema = schemaGetter.getSchema(value.schemaId);
- row =
- ProjectedRow.from(schema, tableInfo.getSchema())
- .replaceRow(value.row);
+ handleLookupResponse(result, lookupFuture);
}
- rowList.add(row);
- }
- future.complete(new LookupResult(rowList));
- } catch (Exception e) {
- future.complete(new LookupResult(Collections.emptyList()));
- }
- });
- return future;
+ });
+ return lookupFuture;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java
index 18ee9a61b..4eadcd8f7 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java
@@ -23,7 +23,8 @@ import org.apache.fluss.row.InternalRow;
import java.util.List;
/**
- * Used to describe the operation to prefix lookup by {@link PrefixLookuper} to a primary key table.
+ * Used to describe the operation to prefix lookup by {@link PrefixKeyLookuper} to a primary key
+ * table.
*
* @since 0.6
*/
@@ -37,7 +38,7 @@ public class PrefixLookup {
* <p>For partitioned table, the lookupColumnNames exclude partition fields should be a prefix
* of primary key exclude partition fields.
*
- * <p>See {@link PrefixLookuper#prefixLookup(InternalRow)} for more details.
+ * <p>See {@link PrefixKeyLookuper#lookup(InternalRow)} for more details.
*/
private final List<String> lookupColumnNames;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
index 091a8a38e..755e454df 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
@@ -22,14 +22,11 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
-import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
-import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -40,17 +37,10 @@ import java.util.concurrent.CompletableFuture;
import static org.apache.fluss.client.utils.ClientUtils.getPartitionId;
import static org.apache.fluss.utils.Preconditions.checkArgument;
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** An implementation of {@link Lookuper} that lookups by primary key. */
@NotThreadSafe
-class PrimaryKeyLookuper implements Lookuper {
-
- private final TableInfo tableInfo;
-
- private final MetadataUpdater metadataUpdater;
-
- private final LookupClient lookupClient;
+class PrimaryKeyLookuper extends AbstractLookuper {
private final KeyEncoder primaryKeyEncoder;
@@ -66,24 +56,17 @@ class PrimaryKeyLookuper implements Lookuper {
/** a getter to extract partition from lookup key row, null when it's not a partitioned. */
private @Nullable final PartitionGetter partitionGetter;
- /** Decode the lookup bytes to result row. */
- private final ValueDecoder kvValueDecoder;
-
- private final SchemaGetter schemaGetter;
-
public PrimaryKeyLookuper(
TableInfo tableInfo,
SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient) {
+ super(tableInfo, metadataUpdater, lookupClient, schemaGetter);
checkArgument(
tableInfo.hasPrimaryKey(),
"Log table %s doesn't support lookup",
tableInfo.getTablePath());
- this.tableInfo = tableInfo;
this.numBuckets = tableInfo.getNumBuckets();
- this.metadataUpdater = metadataUpdater;
- this.lookupClient = lookupClient;
// the row type of the input lookup row
RowType lookupRowType = tableInfo.getRowType().project(tableInfo.getPrimaryKeys());
@@ -102,9 +85,6 @@ class PrimaryKeyLookuper implements Lookuper {
tableInfo.isPartitioned()
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
- this.kvValueDecoder =
- new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
- this.schemaGetter = schemaGetter;
}
@Override
@@ -132,30 +112,21 @@ class PrimaryKeyLookuper implements Lookuper {
int bucketId = bucketingFunction.bucketing(bkBytes, numBuckets);
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- CompletableFuture<LookupResult> future = new CompletableFuture<>();
-
- CompletableFuture.runAsync(
- () -> {
- try {
- byte[] valueBytes = lookupClient.lookup(tableBucket, pkBytes).get();
- InternalRow row = null;
- if (valueBytes != null) {
- ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
- if (value.schemaId == tableInfo.getSchemaId()) {
- row = value.row;
+ CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
+ lookupClient
+ .lookup(tableBucket, pkBytes)
+ .whenComplete(
+ (result, error) -> {
+ if (error != null) {
+ lookupFuture.completeExceptionally(error);
} else {
- Schema schema = schemaGetter.getSchema(value.schemaId);
- checkNotNull(schema, "schema is null");
- row =
- ProjectedRow.from(schema, tableInfo.getSchema())
- .replaceRow(value.row);
+ handleLookupResponse(
+ result == null
+ ? Collections.emptyList()
+ : Collections.singletonList(result),
+ lookupFuture);
}
- }
- future.complete(new LookupResult(row));
- } catch (Exception e) {
- future.completeExceptionally(e);
- }
- });
- return future;
+ });
+ return lookupFuture;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
index 181767781..6b6a11d77 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
@@ -19,6 +19,8 @@ package org.apache.fluss.client.metadata;
import org.apache.fluss.annotation.Internal;
import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.exception.SchemaNotExistException;
+import org.apache.fluss.exception.TableNotExistException;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.SchemaInfo;
@@ -28,8 +30,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
+import static org.apache.fluss.utils.ExceptionUtils.stripExecutionException;
import static org.apache.fluss.utils.MapUtils.newConcurrentHashMap;
/** Schema getter for client. */
@@ -40,7 +44,7 @@ public class ClientSchemaGetter implements SchemaGetter {
private final TablePath tablePath;
private final Map<Integer, Schema> schemasById;
private final Admin admin;
- private SchemaInfo latestSchemaInfo;
+ private volatile SchemaInfo latestSchemaInfo;
public ClientSchemaGetter(TablePath tablePath, SchemaInfo latestSchemaInfo, Admin admin) {
this.tablePath = tablePath;
@@ -52,22 +56,55 @@ public class ClientSchemaGetter implements SchemaGetter {
@Override
public Schema getSchema(int schemaId) {
- return schemasById.computeIfAbsent(
- schemaId,
- (id) -> {
- try {
- SchemaInfo schemaInfo =
- admin.getTableSchema(tablePath, schemaId).get(1, TimeUnit.MINUTES);
- if (id > latestSchemaInfo.getSchemaId()) {
- latestSchemaInfo = schemaInfo;
- }
- return schemaInfo.getSchema();
+ Schema schema = schemasById.get(schemaId);
+ if (schema != null) {
+ return schema;
+ } else {
+ LOG.debug(
+ "Schema id {} not found in cache, fetching from cluster for table: {}",
+ schemaId,
+ tablePath);
+ try {
+ SchemaInfo schemaInfo =
+ admin.getTableSchema(tablePath, schemaId).get(1, TimeUnit.MINUTES);
+ if (schemaId > latestSchemaInfo.getSchemaId()) {
+ latestSchemaInfo = schemaInfo;
+ }
+ return schemaInfo.getSchema();
+ } catch (Exception e) {
+ Throwable strippedException = stripExecutionException(e);
+ if (strippedException instanceof SchemaNotExistException) {
+ throw (SchemaNotExistException) strippedException;
+ } else if (strippedException instanceof TableNotExistException) {
+ throw (TableNotExistException) strippedException;
+ } else {
+ LOG.warn("Failed to get schema for table: {}", tablePath);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
- } catch (Exception e) {
- LOG.warn("Failed to get schema for table: " + tablePath);
- throw new RuntimeException(e);
- }
- });
+ @Override
+ public CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId) {
+ Schema schema = schemasById.get(schemaId);
+ if (schema != null) {
+ return CompletableFuture.completedFuture(new SchemaInfo(schema, schemaId));
+ } else {
+ LOG.debug(
+ "Schema id {} not found in cache, fetching from cluster for table: {}",
+ schemaId,
+ tablePath);
+ return admin.getTableSchema(tablePath, schemaId)
+ .thenApply(
+ (schemaInfo) -> {
+ schemasById.put(schemaId, schemaInfo.getSchema());
+ if (schemaId > latestSchemaInfo.getSchemaId()) {
+ latestSchemaInfo = schemaInfo;
+ }
+ return schemaInfo;
+ });
+ }
}
@Override
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
index 629abe6ab..8532f2a85 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
@@ -29,7 +29,6 @@ import org.apache.fluss.client.table.writer.TableAppend;
import org.apache.fluss.client.table.writer.TableUpsert;
import org.apache.fluss.client.table.writer.Upsert;
import org.apache.fluss.metadata.SchemaGetter;
-import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -55,10 +54,7 @@ public class FlussTable implements Table {
this.tableInfo = tableInfo;
this.hasPrimaryKey = tableInfo.hasPrimaryKey();
this.schemaGetter =
- new ClientSchemaGetter(
- tableInfo.getTablePath(),
- new SchemaInfo(tableInfo.getSchema(), tableInfo.getSchemaId()),
- conn.getAdmin());
+ new ClientSchemaGetter(tablePath, tableInfo.getSchemaInfo(), conn.getAdmin());
}
@Override
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
index f4e150f30..d36bf9a7d 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
@@ -71,8 +71,7 @@ public class BatchScanUtils {
// If the scanner has more data, add it back to the queue
scannerQueue.add(scanner);
} else {
- // Close the scanner if it has no more data, and not add it back to the
- // queue
+ // Close the scanner if it has no more data, and not add it back to the queue
scanner.close();
}
} catch (Exception e) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
index e6208f368..06c79fce8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
@@ -39,6 +39,7 @@ import org.apache.fluss.rpc.messages.LimitScanRequest;
import org.apache.fluss.rpc.messages.LimitScanResponse;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
+import org.apache.fluss.utils.SchemaUtil;
import javax.annotation.Nullable;
@@ -47,7 +48,9 @@ import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -64,6 +67,12 @@ public class LimitBatchScanner implements BatchScanner {
private final KvFormat kvFormat;
private final int targetSchemaId;
+ /**
+ * A cache for schema projection mapping from source schema to target. Use HashMap here, because
+ * LimitBatchScanner is used in single thread only.
+ */
+ private final Map<Short, int[]> schemaProjectionCache = new HashMap<>();
+
private boolean endOfInput;
public LimitBatchScanner(
@@ -142,11 +151,14 @@ public class LimitBatchScanner implements BatchScanner {
for (ValueRecord record : valueRecords.records(readContext)) {
InternalRow row = record.getRow();
if (targetSchemaId != record.schemaId()) {
- row =
- ProjectedRow.from(
- schemaGetter.getSchema(record.schemaId()),
- schemaGetter.getSchema(targetSchemaId))
- .replaceRow(row);
+ int[] indexMapping =
+ schemaProjectionCache.computeIfAbsent(
+ record.schemaId(),
+ sourceSchemaId ->
+ SchemaUtil.getIndexMapping(
+ schemaGetter.getSchema(sourceSchemaId),
+ schemaGetter.getSchema(targetSchemaId)));
+ row = ProjectedRow.from(indexMapping).replaceRow(row);
}
scanRows.add(maybeProject(row));
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
index 10e773609..8a52814f3 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
@@ -22,6 +22,7 @@ import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.rocksdb.RocksDBHandle;
import org.apache.fluss.rocksdb.RocksIteratorWrapper;
import org.apache.fluss.row.InternalRow;
@@ -30,6 +31,7 @@ import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.IOUtils;
+import org.apache.fluss.utils.SchemaUtil;
import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions;
@@ -43,6 +45,8 @@ import javax.annotation.concurrent.NotThreadSafe;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
/**
* A reader to read kv snapshot files to {@link ScanRecord}s. It will return the {@link ScanRecord}s
@@ -58,6 +62,12 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
@Nullable private final int[] projectedFields;
private RocksIteratorWrapper rocksIteratorWrapper;
+ /**
+ * A cache for schema projection mapping from source schema to target. Use HashMap here, because
+ * SnapshotFilesReader is used in single thread only.
+ */
+ private final Map<Short, int[]> schemaProjectionCache = new HashMap<>();
+
private Snapshot snapshot;
private RocksDBHandle rocksDBHandle;
private boolean isClose = false;
@@ -155,12 +165,16 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
byte[] value = rocksIteratorWrapper.value();
rocksIteratorWrapper.next();
- ValueDecoder.Value originValue = valueDecoder.decodeValue(value);
+ BinaryValue originValue = valueDecoder.decodeValue(value);
InternalRow originRow = originValue.row;
if (targetSchemaId != originValue.schemaId) {
- originRow =
- ProjectedRow.from(schemaGetter.getSchema(originValue.schemaId), targetSchema)
- .replaceRow(originRow);
+ int[] indexMapping =
+ schemaProjectionCache.computeIfAbsent(
+ originValue.schemaId,
+ sourceSchemaId ->
+ SchemaUtil.getIndexMapping(
+ schemaGetter.getSchema(sourceSchemaId), targetSchema));
+ originRow = ProjectedRow.from(indexMapping).replaceRow(originRow);
}
if (projectedFields != null) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
index bee0910e1..738e04a49 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
@@ -24,8 +24,7 @@ import org.apache.fluss.annotation.Internal;
class BucketScanStatus {
private long offset; // last consumed position
private long highWatermark; // the high watermark from last fetch
- // TODO add
- // resetStrategy and nextAllowedRetryTimeMs.
+ // TODO add resetStrategy and nextAllowedRetryTimeMs.
public BucketScanStatus() {
this.offset = 0L;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
index e8067dd4c..b19347772 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
@@ -364,8 +364,7 @@ public class LogFetcher implements Closeable {
if (!MemoryLogRecords.EMPTY.equals(logRecords)
|| fetchResultForBucket.getErrorCode() != Errors.NONE.code()) {
// In oder to not signal notEmptyCondition, add completed
- // fetch to
- // buffer until log records is not empty.
+ // fetch to buffer until log records is not empty.
DefaultCompletedFetch completedFetch =
new DefaultCompletedFetch(
tb,
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
index f1811dfa2..98a93fdd2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
@@ -37,13 +37,11 @@ public abstract class AbstractTableWriter implements TableWriter {
protected final TablePath tablePath;
protected final WriterClient writerClient;
protected final int fieldCount;
- protected TableInfo tableInfo;
private final @Nullable PartitionGetter partitionFieldGetter;
protected AbstractTableWriter(
TablePath tablePath, TableInfo tableInfo, WriterClient writerClient) {
this.tablePath = tablePath;
- this.tableInfo = tableInfo;
this.writerClient = writerClient;
this.fieldCount = tableInfo.getRowType().getFieldCount();
this.partitionFieldGetter =
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
index 8b36b5cf8..58efb7f47 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
@@ -336,6 +336,44 @@ public class ClientRpcMessageUtils {
return dropPartitionRequest;
}
+ public static AlterTableRequest makeAlterTableRequest(
+ TablePath tablePath, List<TableChange> tableChanges, boolean ignoreIfNotExists) {
+ AlterTableRequest request = new AlterTableRequest();
+ request.setIgnoreIfNotExists(ignoreIfNotExists)
+ .setTablePath()
+ .setDatabaseName(tablePath.getDatabaseName())
+ .setTableName(tablePath.getTableName());
+
+ List<PbAddColumn> addColumns = new ArrayList<>();
+ List<PbDropColumn> dropColumns = new ArrayList<>();
+ List<PbRenameColumn> renameColumns = new ArrayList<>();
+ List<PbModifyColumn> modifyColumns = new ArrayList<>();
+ List<PbAlterConfig> alterConfigs = new ArrayList<>();
+ for (TableChange tableChange : tableChanges) {
+ if (tableChange instanceof TableChange.AddColumn) {
+ addColumns.add(toPbAddColumn((TableChange.AddColumn) tableChange));
+ } else if (tableChange instanceof TableChange.DropColumn) {
+ dropColumns.add(toPbDropColumn((TableChange.DropColumn) tableChange));
+ } else if (tableChange instanceof TableChange.RenameColumn) {
+ renameColumns.add(toPbRenameColumn((TableChange.RenameColumn) tableChange));
+ } else if (tableChange instanceof TableChange.ModifyColumn) {
+ modifyColumns.add(toPbModifyColumn((TableChange.ModifyColumn) tableChange));
+ } else if (tableChange instanceof TableChange.SetOption
+ || tableChange instanceof TableChange.ResetOption) {
+ alterConfigs.add(toPbAlterConfigs(tableChange));
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported table change: " + tableChange.getClass());
+ }
+ }
+ request.addAllConfigChanges(alterConfigs)
+ .addAllAddColumns(addColumns)
+ .addAllDropColumns(dropColumns)
+ .addAllRenameColumns(renameColumns)
+ .addAllModifyColumns(modifyColumns);
+ return request;
+ }
+
public static List<PartitionInfo> toPartitionInfos(ListPartitionInfosResponse response) {
return response.getPartitionsInfosList().stream()
.map(
@@ -380,33 +418,6 @@ public class ClientRpcMessageUtils {
return info;
}
- public static void addPbAlterSchemas(
- AlterTableRequest request, List<TableChange> tableChanges) {
- List<PbAddColumn> addColumns = new ArrayList<>();
- List<PbDropColumn> dropColumns = new ArrayList<>();
- List<PbRenameColumn> renameColumns = new ArrayList<>();
- List<PbModifyColumn> modifyColumns = new ArrayList<>();
- for (TableChange tableChange : tableChanges) {
- if (tableChange instanceof TableChange.AddColumn) {
- addColumns.add(toPbAddColumn((TableChange.AddColumn) tableChange));
- } else if (tableChange instanceof TableChange.DropColumn) {
- dropColumns.add(toPbDropColumn((TableChange.DropColumn) tableChange));
- } else if (tableChange instanceof TableChange.RenameColumn) {
- renameColumns.add(toPbRenameColumn((TableChange.RenameColumn) tableChange));
- } else if (tableChange instanceof TableChange.ModifyColumn) {
- modifyColumns.add(toPbModifyColumn((TableChange.ModifyColumn) tableChange));
- } else {
- throw new IllegalArgumentException(
- "Unsupported table change: " + tableChange.getClass());
- }
- }
-
- request.addAllAddColumns(addColumns)
- .addAllDropColumns(dropColumns)
- .addAllRenameColumns(renameColumns)
- .addAllModifyColumns(modifyColumns);
- }
-
public static PbAddColumn toPbAddColumn(TableChange.AddColumn addColumn) {
ColumnPositionType columnPositionType = ColumnPositionType.from(addColumn.getPosition());
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
index f7d7d3ef0..3a282ba92 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
@@ -868,8 +868,7 @@ public final class RecordAccumulator {
// Either we have reached a point where there are batches without a sequence (i.e. never
// been drained and are hence in order by default), or the batch at the front of the
// queue has a sequence greater than the incoming batch. This is the right place to
- // add
- // the incoming batch.
+ // add the incoming batch.
deque.addFirst(batch);
// Now we have to re-insert the previously queued batches in the right order.
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
index 1b9c79397..e9c285306 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
@@ -130,7 +130,7 @@ public class Sender implements Runnable {
this.idempotenceManager = idempotenceManager;
this.writerMetricGroup = writerMetricGroup;
- // TODOadd retry logic while send failed. See FLUSS-56364375
+ // TODO add retry logic while send failed. See FLUSS-56364375
}
@VisibleForTesting
@@ -167,7 +167,7 @@ public class Sender implements Runnable {
}
}
- // TODO if force close failed,add logic to abort incomplete batches.
+ // TODO if force close failed, add logic to abort incomplete batches.
LOG.debug("Shutdown of Fluss write sender I/O thread has completed.");
}
@@ -245,7 +245,7 @@ public class Sender implements Runnable {
if (!batches.isEmpty()) {
addToInflightBatches(batches);
- // TODOadd logic for batch expire.
+ // TODO add logic for batch expire.
sendWriteRequests(batches);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
index 05f196fdd..bd7954284 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
@@ -354,13 +354,43 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
.get())
.hasMessageContaining("Column c1 must be nullable");
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING().copy(false),
+ null,
+ TableChange.ColumnPosition
+ .first())),
+ false)
+ .get())
+ .hasMessageContaining("Unsupported ColumnPositionType: FIRST");
+
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING().copy(false),
+ null,
+ TableChange.ColumnPosition.after(
+ "name"))),
+ false)
+ .get())
+ .hasMessageContaining("Unsupported ColumnPositionType: AFTER(name)");
+
admin.alterTable(
tablePath,
Collections.singletonList(
TableChange.addColumn(
"c1",
DataTypes.STRING(),
- null,
+ "new column c1",
TableChange.ColumnPosition.last())),
false)
.get();
@@ -380,7 +410,10 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
new Schema.Column(
"age", DataTypes.INT(), "person age", (short) 2),
new Schema.Column(
- "c1", DataTypes.STRING(), null, (short) 3)))
+ "c1",
+ DataTypes.STRING(),
+ "new column c1",
+ (short) 3)))
.build();
SchemaInfo schemaInfo = admin.getTableSchema(tablePath).get();
assertThat(schemaInfo).isEqualTo(new SchemaInfo(expectedSchema, 2));
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
index 527994777..62b5bdc13 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
@@ -134,7 +134,7 @@ class KvSnapshotBatchScannerITCase extends ClientToServerITCaseBase {
Collections.singletonList(
TableChange.addColumn(
"new_column",
- DataTypes.BIGINT().copy(false).copy(true),
+ DataTypes.BIGINT().copy(true),
null,
TableChange.ColumnPosition.last())),
false)
@@ -148,7 +148,7 @@ class KvSnapshotBatchScannerITCase extends ClientToServerITCaseBase {
.column("name", DataTypes.STRING())
.column("new_column", DataTypes.BIGINT())
.build();
- // put into values with old schema.
+ // put into values with new schema.
List<InternalRow> rows = new ArrayList<>();
for (int i = 10; i < 20; i++) {
InternalRow row =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
index bfd686dc1..d487a2d46 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
@@ -29,6 +29,7 @@ import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
@@ -265,7 +266,7 @@ public class DefaultCompletedFetchTest {
rowType, DEFAULT_SCHEMA_ID, 0L, 1000L, magic, objects, LogFormat.ARROW));
fileLogRecords.flush();
- FileLogProjection fileLogProjection = new FileLogProjection();
+ FileLogProjection fileLogProjection = new FileLogProjection(new ProjectionPushdownCache());
fileLogProjection.setCurrentProjection(
DATA2_TABLE_ID,
testingSchemaGetter,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
index 83878a816..99a108e2f 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
@@ -93,8 +93,7 @@ public class LogFetchCollectorTest {
logFetchBuffer.add(completedFetch);
assertThat(logFetchBuffer.isEmpty()).isFalse();
- // Validate that the completed fetch isn't initialized just because we add it to the
- // buffer.
+ // Validate that the completed fetch isn't initialized just because we add it to the buffer
assertThat(completedFetch.isInitialized()).isFalse();
// Fetch the data and validate that we get all the records we want back.
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
index 03f908733..e5a8c8ff0 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
@@ -199,6 +199,7 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
scanRecords = records.get(tb0);
assertThat(scanRecords.stream().map(ScanRecord::getRow).collect(Collectors.toList()))
.isEqualTo(expectedRows);
+ newSchemaLogFetcher.close();
}
@Test
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
index bc4e53b50..47a9c2414 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
@@ -137,7 +137,6 @@ class RecordAccumulatorTest {
conf = new Configuration();
// init cluster.
cluster = updateCluster(Arrays.asList(bucket1, bucket2, bucket3));
- // todo: 从cluster中获取schema getter
schemaGetter = new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA, (short) 1));
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java b/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java
index 14b0b7f96..b577b8633 100644
--- a/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java
@@ -25,21 +25,35 @@ public class ApiException extends FlussRuntimeException {
private static final long serialVersionUID = 1L;
+ private final boolean stackTraceEnabled;
+
+ public ApiException(String message, Throwable cause, boolean stackTraceEnabled) {
+ super(message, cause);
+ this.stackTraceEnabled = stackTraceEnabled;
+ }
+
public ApiException(String message, Throwable cause) {
super(message, cause);
+ this.stackTraceEnabled = false;
}
public ApiException(String message) {
super(message);
+ this.stackTraceEnabled = false;
}
public ApiException(Throwable cause) {
super(cause);
+ this.stackTraceEnabled = false;
}
/* avoid the expensive and useless stack trace for api exceptions */
@Override
public Throwable fillInStackTrace() {
- return this;
+ if (stackTraceEnabled) {
+ return super.fillInStackTrace();
+ } else {
+ return this;
+ }
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
index 8c9446682..77983e441 100644
--- a/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
@@ -27,4 +27,8 @@ public class SchemaChangeException extends ApiException {
public SchemaChangeException(String message) {
this(message, null);
}
+
+ public SchemaChangeException(String message, boolean stackTraceEnabled) {
+ super(message, null, stackTraceEnabled);
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
index 367013363..0ae0c8b97 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
@@ -132,6 +132,7 @@ public final class Schema implements Serializable {
return columns.stream().map(Column::getName).collect(Collectors.toList());
}
+ /** Returns all column ids for top-level columns, the nested field ids are not included. */
public List<Integer> getColumnIds() {
return columns.stream().map(Column::getColumnId).collect(Collectors.toList());
}
@@ -154,6 +155,7 @@ public final class Schema implements Serializable {
return keyIndexes;
}
+ /** Returns the highest field ID in this schema. */
public int getHighestFieldId() {
return highestFieldId;
}
@@ -249,24 +251,21 @@ public final class Schema implements Serializable {
/** Adopts all columns from the given list. */
public Builder fromColumns(List<Column> inputColumns) {
- boolean nonMatchColumnId =
+ boolean nonSetColumnId =
inputColumns.stream()
.noneMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
- boolean allMatchColumnId =
+ boolean allSetColumnId =
inputColumns.stream()
.allMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
checkState(
- nonMatchColumnId || allMatchColumnId,
+ nonSetColumnId || allSetColumnId,
"All columns must have columnId or none of them must have columnId.");
- if (allMatchColumnId) {
+ if (allSetColumnId) {
columns.addAll(inputColumns);
highestFieldId =
new AtomicInteger(
- inputColumns.stream()
- .mapToInt(Column::getColumnId)
- .max()
- .orElse(-1));
+ columns.stream().mapToInt(Column::getColumnId).max().orElse(-1));
} else {
// if all columnId is not set, this maybe from old version schema. Just use its
// position as columnId.
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
index 8cdc14670..e5b8e6699 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
@@ -17,11 +17,59 @@
package org.apache.fluss.metadata;
-/** A schema getter interface. */
+import org.apache.fluss.exception.SchemaNotExistException;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Provides retrieval and lifecycle management for {@link Schema} instances identified by schema
+ * ids.
+ *
+ * <p>Implementations may expose both blocking and non-blocking retrieval paths. Clients can obtain
+ * a schema synchronously via {@link #getSchema(int)} or asynchronously via {@link
+ * #getSchemaInfoAsync(int)}.
+ */
public interface SchemaGetter {
+
+ /**
+ * Synchronously obtains the {@link Schema} associated with the given {@code schemaId}.
+ *
+ * <p>This method may block while fetching or deserializing the schema. The implementation may
+ * perform an RPC request to fetching schema metadata from the Cluster.
+ *
+ * @param schemaId numeric identifier of the requested schema
+ * @return the resolved {@code Schema}, or {@code null} if no schema is available for the id
+ * @throws SchemaNotExistException if the requested schema id is not exists.
+ */
Schema getSchema(int schemaId);
+ /**
+ * Asynchronously obtains the {@code Schema} associated with the given {@code schemaId}.
+ *
+ * <p>The returned {@link CompletableFuture} completes with the resolved {@link SchemaInfo} or
+ * completes exceptionally if an error occurs.
+ *
+ * <p>The following exceptions can be anticipated when calling {@code get()} on returned future.
+ *
+ * <ul>
+ * <li>{@link SchemaNotExistException} if the requested schema id is not exists.
+ * </ul>
+ *
+ * @param schemaId numeric identifier of the requested schema
+ * @return a {@link CompletableFuture} that will be completed with the {@link SchemaInfo} or
+ * completed exceptionally
+ */
+ CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId);
+
+ /** Returns metadata describing the latest known schema. */
SchemaInfo getLatestSchemaInfo();
+ /**
+ * Releases any resources held by this {@code SchemaGetter}.
+ *
+ * <p>After calling this method, the instance may no longer be usable. Implementations should
+ * free caches, close network connections, cancel outstanding requests, and stop background
+ * threads as appropriate.
+ */
void release();
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
index 36162f30c..e012a38f6 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
@@ -402,7 +402,7 @@ public interface TableChange {
@Override
public String toString() {
- return String.format("AFTER %s", columnName);
+ return String.format("AFTER(%s)", columnName);
}
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
index af293f120..eba41e078 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
@@ -137,6 +137,11 @@ public final class TableInfo {
return schema;
}
+ /** Returns the schema info of the table, including schema and schema id. */
+ public SchemaInfo getSchemaInfo() {
+ return new SchemaInfo(schema, schemaId);
+ }
+
/**
* Returns the row type of the table. The row type is the schema of the table, which defines the
* columns and types of the table.
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java b/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java
new file mode 100644
index 000000000..13906347d
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.record;
+
+import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.encode.ValueEncoder;
+
+import java.util.Objects;
+
+/** A value of key-value pair that contains schema id and binary row. */
+public class BinaryValue {
+
+ public final short schemaId;
+ public final BinaryRow row;
+
+ public BinaryValue(short schemaId, BinaryRow row) {
+ this.schemaId = schemaId;
+ this.row = row;
+ }
+
+ /**
+ * Encode the value (consisted of {@code row} with a {@code schemaId}) to a byte array value to
+ * be expected persisted to kv store.
+ */
+ public byte[] encodeValue() {
+ return ValueEncoder.encodeValue(schemaId, row);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ BinaryValue that = (BinaryValue) o;
+ return schemaId == that.schemaId && Objects.equals(row, that.row);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(schemaId, row);
+ }
+
+ @Override
+ public String toString() {
+ return "BinaryValue{" + "schemaId=" + schemaId + ", row=" + row + '}';
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
index b6ed40eeb..8bf27e10e 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
@@ -21,7 +21,6 @@ import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.exception.CorruptMessageException;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.LogFormat;
-import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.arrow.ArrowReader;
import org.apache.fluss.row.columnar.ColumnarRow;
@@ -34,13 +33,13 @@ import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.MurmurHashUtils;
import org.apache.fluss.utils.crc.Crc32C;
+import javax.annotation.Nullable;
+
import java.nio.ByteBuffer;
import java.util.NoSuchElementException;
-import java.util.function.Function;
import static org.apache.fluss.record.LogRecordBatchFormat.BASE_OFFSET_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.COMMIT_TIMESTAMP_OFFSET;
-import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_LENGTH;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_OVERHEAD;
@@ -213,7 +212,6 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
@Override
- @SuppressWarnings("unchecked")
public CloseableIterator<LogRecord> records(ReadContext context) {
if (getRecordCount() == 0) {
return CloseableIterator.emptyIterator();
@@ -223,26 +221,18 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
long timestamp = commitTimestamp();
LogFormat logFormat = context.getLogFormat();
RowType rowType = context.getRowType(schemaId);
- Function<InternalRow, InternalRow> outputRowProcessor =
- (row) -> {
- ProjectedRow projectedRow = context.getOutputProjectedRow(schemaId);
- if (projectedRow != null) {
- return projectedRow.replaceRow(row);
- } else {
- return row;
- }
- };
switch (logFormat) {
case ARROW:
return columnRecordIterator(
rowType,
- outputRowProcessor,
+ context.getOutputProjectedRow(schemaId),
context.getVectorSchemaRoot(schemaId),
context.getBufferAllocator(),
timestamp);
case INDEXED:
- return rowRecordIterator(rowType, outputRowProcessor, timestamp);
+ return rowRecordIterator(
+ rowType, context.getOutputProjectedRow(schemaId), timestamp);
default:
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
@@ -270,9 +260,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
private CloseableIterator<LogRecord> rowRecordIterator(
- RowType rowType,
- Function<InternalRow, InternalRow> outputRowProcessor,
- long timestamp) {
+ RowType rowType, @Nullable ProjectedRow outputProjection, long timestamp) {
DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
return new LogRecordIterator() {
int position = DefaultLogRecordBatch.this.position + recordBatchHeaderSize(magic);
@@ -281,16 +269,20 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
@Override
protected LogRecord readNext(long baseOffset) {
IndexedLogRecord logRecord =
- ProjectedIndexedLogRecord.readFrom(
- segment,
- position,
- baseOffset + rowId,
- timestamp,
- fieldTypes,
- outputRowProcessor);
+ IndexedLogRecord.readFrom(
+ segment, position, baseOffset + rowId, timestamp, fieldTypes);
rowId++;
position += logRecord.getSizeInBytes();
- return logRecord;
+ if (outputProjection == null) {
+ return logRecord;
+ } else {
+ // apply projection
+ return new GenericRecord(
+ logRecord.logOffset(),
+ logRecord.timestamp(),
+ logRecord.getChangeType(),
+ outputProjection.replaceRow(logRecord.getRow()));
+ }
}
@Override
@@ -305,7 +297,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
private CloseableIterator<LogRecord> columnRecordIterator(
RowType rowType,
- Function<InternalRow, InternalRow> outputRowProcessor,
+ @Nullable ProjectedRow outputProjection,
VectorSchemaRoot root,
BufferAllocator allocator,
long timestamp) {
@@ -319,7 +311,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputProjection) {
@Override
protected ChangeType getChangeType(int rowId) {
return ChangeType.APPEND_ONLY;
@@ -337,7 +329,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputProjection) {
@Override
protected ChangeType getChangeType(int rowId) {
return changeTypeVector.getChangeType(rowId);
@@ -351,15 +343,13 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
private final ArrowReader reader;
private final long timestamp;
private int rowId = 0;
- private final Function<InternalRow, InternalRow> outputRowProcessor;
+ @Nullable private final ProjectedRow outputProjection;
private ArrowLogRecordIterator(
- ArrowReader reader,
- long timestamp,
- Function<InternalRow, InternalRow> outputRowProcessor) {
+ ArrowReader reader, long timestamp, @Nullable ProjectedRow outputProjection) {
this.reader = reader;
this.timestamp = timestamp;
- this.outputRowProcessor = outputRowProcessor;
+ this.outputProjection = outputProjection;
}
protected abstract ChangeType getChangeType(int rowId);
@@ -377,7 +367,9 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
baseOffset + rowId,
timestamp,
getChangeType(rowId),
- outputRowProcessor.apply(originalRow));
+ outputProjection == null
+ ? originalRow
+ : outputProjection.replaceRow(originalRow));
rowId++;
return record;
}
@@ -447,38 +439,4 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
throw new UnsupportedOperationException();
}
}
-
- private static class ProjectedIndexedLogRecord extends IndexedLogRecord {
- private final Function<InternalRow, InternalRow> outputRowProcessor;
-
- ProjectedIndexedLogRecord(
- long logOffset,
- long timestamp,
- DataType[] fieldTypes,
- Function<InternalRow, InternalRow> outputRowProcessor) {
- super(logOffset, timestamp, fieldTypes);
- this.outputRowProcessor = outputRowProcessor;
- }
-
- public static IndexedLogRecord readFrom(
- MemorySegment segment,
- int position,
- long logOffset,
- long logTimestamp,
- DataType[] colTypes,
- Function<InternalRow, InternalRow> outputRowProcessor) {
- int sizeInBytes = segment.getInt(position);
- ProjectedIndexedLogRecord logRecord =
- new ProjectedIndexedLogRecord(
- logOffset, logTimestamp, colTypes, outputRowProcessor);
- logRecord.pointTo(segment, position, sizeInBytes + LENGTH_LENGTH);
- return logRecord;
- }
-
- @Override
- public InternalRow getRow() {
- InternalRow row = super.getRow();
- return outputRowProcessor.apply(row);
- }
- }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
index 11c50bb3d..9f227adb1 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
@@ -54,7 +54,6 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import static org.apache.fluss.record.DefaultLogRecordBatch.APPEND_ONLY_FLAG_MASK;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
@@ -71,7 +70,6 @@ import static org.apache.fluss.record.LogRecordBatchFormat.recordsCountOffset;
import static org.apache.fluss.record.LogRecordBatchFormat.schemaIdOffset;
import static org.apache.fluss.utils.FileUtils.readFully;
import static org.apache.fluss.utils.FileUtils.readFullyOrFail;
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
import static org.apache.fluss.utils.Preconditions.checkState;
/** Column projection util on Arrow format {@link FileLogRecords}. */
@@ -85,8 +83,8 @@ public class FileLogProjection {
private static final int ARROW_HEADER_SIZE =
ARROW_IPC_CONTINUATION_LENGTH + ARROW_IPC_METADATA_SIZE_LENGTH;
- final Map<ProjectionKey, ProjectionInfo> projectionsCache = new HashMap<>();
- ProjectionInfo currentProjection;
+ // the projection cache shared in the TabletServer
+ private final ProjectionPushdownCache projectionsCache;
// shared resources for multiple projections
private final ByteArrayOutputStream outputStream;
@@ -103,9 +101,10 @@ public class FileLogProjection {
private SchemaGetter schemaGetter;
private long tableId;
private ArrowCompressionInfo compressionInfo;
- private int[] selectedFieldPositions;
+ private int[] selectedFieldIds;
- public FileLogProjection() {
+ public FileLogProjection(ProjectionPushdownCache projectionsCache) {
+ this.projectionsCache = projectionsCache;
this.outputStream = new ByteArrayOutputStream();
this.writeChannel = new WriteChannel(Channels.newChannel(outputStream));
// fluss use little endian for encoding log records batch
@@ -122,14 +121,7 @@ public class FileLogProjection {
this.tableId = tableId;
this.schemaGetter = schemaGetter;
this.compressionInfo = compressionInfo;
-
- // Currently, only add last column is supported.Thus selectedFieldPositions is always same
- // from same selectedFieldIds.
- // TODO: if support drop column or add column in middle, this selectedFieldPositions should
- // be re-calculated for each schema.
- this.selectedFieldPositions =
- selectedFieldPositions(
- schemaGetter.getLatestSchemaInfo().getSchema(), selectedFieldIds);
+ this.selectedFieldIds = selectedFieldIds;
}
/**
@@ -144,6 +136,8 @@ public class FileLogProjection {
MultiBytesView.Builder builder = MultiBytesView.builder();
int position = start;
+ ProjectionInfo currentProjection = null;
+ short prevSchemaId = -1;
// The condition is an optimization to avoid read log header when there is no enough bytes,
// So we use V0 header size here for a conservative judgment. In the end, the condition
// of (position >= end - recordBatchHeaderSize) will ensure the final correctness.
@@ -162,8 +156,12 @@ public class FileLogProjection {
int recordBatchHeaderSize = recordBatchHeaderSize(magic);
int batchSizeInBytes = LOG_OVERHEAD + logHeaderBuffer.getInt(LENGTH_OFFSET);
short schemaId = logHeaderBuffer.getShort(schemaIdOffset(magic));
- setCurrentSchema(schemaId);
- checkNotNull(currentProjection, "There is no projection registered yet.");
+
+ // reuse projection in the current log file
+ if (currentProjection == null || prevSchemaId != schemaId) {
+ prevSchemaId = schemaId;
+ currentProjection = getOrCreateProjectionInfo(schemaId);
+ }
if (position > end - batchSizeInBytes) {
// the remaining bytes in the file are not enough to read a full batch
@@ -405,20 +403,22 @@ public class FileLogProjection {
return logHeaderBuffer;
}
- private void setCurrentSchema(short schemaId) {
+ private ProjectionInfo getOrCreateProjectionInfo(short schemaId) {
+ ProjectionInfo cachedProjection =
+ projectionsCache.getProjectionInfo(tableId, schemaId, selectedFieldIds);
+ if (cachedProjection == null) {
+ cachedProjection = createProjectionInfo(schemaId, selectedFieldIds);
+ projectionsCache.setProjectionInfo(
+ tableId, schemaId, selectedFieldIds, cachedProjection);
+ }
+ return cachedProjection;
+ }
+
+ private ProjectionInfo createProjectionInfo(short schemaId, int[] selectedFieldIds) {
org.apache.fluss.metadata.Schema schema = schemaGetter.getSchema(schemaId);
+ int[] selectedFieldPositions =
+ selectedFieldPositions(schemaGetter.getSchema(schemaId), selectedFieldIds);
RowType rowType = schema.getRowType();
- ProjectionKey projectionKey = new ProjectionKey(tableId, schemaId);
- if (projectionsCache.containsKey(projectionKey)) {
- // the schema and projection should identical for the same table id.
- currentProjection = projectionsCache.get(projectionKey);
- if (!Arrays.equals(currentProjection.selectedFieldPositions, selectedFieldPositions)
- || !currentProjection.schema.equals(rowType)) {
- throw new InvalidColumnProjectionException(
- "The schema and projection should be identical for the same table id.");
- }
- return;
- }
// initialize the projection util information
Schema arrowSchema = ArrowUtils.toArrowSchema(rowType);
@@ -451,16 +451,13 @@ public class FileLogProjection {
CompressionUtil.createBodyCompression(compressionInfo.createCompressionCodec());
int metadataLength =
ArrowUtils.estimateArrowMetadataLength(projectedArrowSchema, bodyCompression);
- currentProjection =
- new ProjectionInfo(
- nodesProjection,
- buffersProjection,
- bufferIndex,
- rowType,
- metadataLength,
- bodyCompression,
- selectedFieldPositions);
- projectionsCache.put(projectionKey, currentProjection);
+ return new ProjectionInfo(
+ nodesProjection,
+ buffersProjection,
+ bufferIndex,
+ metadataLength,
+ bodyCompression,
+ selectedFieldPositions);
}
int[] selectedFieldPositions(org.apache.fluss.metadata.Schema schema, int[] projectedFields) {
@@ -478,7 +475,8 @@ public class FileLogProjection {
if (position == null) {
throw new InvalidColumnProjectionException(
String.format(
- "Projected field id %s is not contains in %s", fieldId, columnIds));
+ "Projected field id %s is not contained in %s",
+ fieldId, columnIds));
}
selectedFieldPositions[i] = position;
@@ -493,35 +491,11 @@ public class FileLogProjection {
return selectedFieldPositions;
}
- static final class ProjectionKey {
- private final long tableId;
- private final short schemaId;
-
- ProjectionKey(long tableId, short schemaId) {
- this.tableId = tableId;
- this.schemaId = schemaId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof ProjectionKey)) {
- return false;
- }
- ProjectionKey that = (ProjectionKey) o;
- return tableId == that.tableId && schemaId == that.schemaId;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(tableId, schemaId);
- }
- }
-
- static final class ProjectionInfo {
+ /** Projection pushdown information for a specific schema and selected fields. */
+ public static final class ProjectionInfo {
final BitSet nodesProjection;
final BitSet buffersProjection;
final int bufferCount;
- final RowType schema;
final int arrowMetadataLength;
final ArrowBodyCompression bodyCompression;
final int[] selectedFieldPositions;
@@ -530,14 +504,12 @@ public class FileLogProjection {
BitSet nodesProjection,
BitSet buffersProjection,
int bufferCount,
- RowType schema,
int arrowMetadataLength,
ArrowBodyCompression bodyCompression,
int[] selectedFieldPositions) {
this.nodesProjection = nodesProjection;
this.buffersProjection = buffersProjection;
this.bufferCount = bufferCount;
- this.schema = schema;
this.arrowMetadataLength = arrowMetadataLength;
this.bodyCompression = bodyCompression;
this.selectedFieldPositions = selectedFieldPositions;
@@ -545,7 +517,7 @@ public class FileLogProjection {
}
/** Metadata of a projected arrow record batch. */
- public static final class ProjectedArrowBatch {
+ static final class ProjectedArrowBatch {
/** Number of records. */
final long numRecords;
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
index 50ec7ee66..5ec5c7c77 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
@@ -31,18 +31,20 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.ArrowUtils;
+import org.apache.fluss.utils.MapUtils;
import org.apache.fluss.utils.Projection;
import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.IntStream;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** A simple implementation for {@link LogRecordBatch.ReadContext}. */
+@ThreadSafe
public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoCloseable {
// the log format of the table
@@ -50,9 +52,7 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
// the schema of the date read form server or remote. (which is projected in the server side)
private final RowType dataRowType;
// the static schemaId of the table, should support dynamic schema evolution in the future
- private final int expectedSchemaId;
- // the Arrow vector schema root of the table, should be null if not ARROW log format
- @Nullable private final VectorSchemaRoot vectorSchemaRoot;
+ private final int targetSchemaId;
// the Arrow memory buffer allocator for the table, should be null if not ARROW log format
@Nullable private final BufferAllocator bufferAllocator;
// the final selected fields of the read data
@@ -60,7 +60,8 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
// whether the projection is push downed to the server side and the returned data is pruned.
private final boolean projectionPushDowned;
private final SchemaGetter schemaGetter;
- private final Map<Integer, VectorSchemaRoot> vectorSchemaRootMap;
+ private final ConcurrentHashMap<Integer, VectorSchemaRoot> vectorSchemaRootMap =
+ MapUtils.newConcurrentHashMap();
public static LogRecordReadContext createReadContext(
TableInfo tableInfo,
@@ -115,14 +116,11 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
SchemaGetter schemaGetter) {
// TODO: use a more reasonable memory limit
BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
- VectorSchemaRoot vectorRoot =
- VectorSchemaRoot.create(ArrowUtils.toArrowSchema(dataRowType), allocator);
FieldGetter[] fieldGetters = buildProjectedFieldGetters(dataRowType, selectedFields);
return new LogRecordReadContext(
LogFormat.ARROW,
dataRowType,
schemaId,
- vectorRoot,
allocator,
fieldGetters,
projectionPushDowned,
@@ -130,10 +128,12 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
}
/**
- * Creates a LogRecordReadContext for INDEXED log format.
+ * Creates a LogRecordReadContext for ARROW log format, that underlying Arrow resources are not
+ * reused.
*
* @param rowType the schema of the table
* @param schemaId the schemaId of the table
+ * @param schemaGetter the schema getter of to get schema by schemaId
*/
@VisibleForTesting
public static LogRecordReadContext createArrowReadContext(
@@ -142,6 +142,13 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
return createArrowReadContext(rowType, schemaId, selectedFields, false, schemaGetter);
}
+ /**
+ * Creates a LogRecordReadContext for INDEXED log format.
+ *
+ * @param rowType the schema of the table
+ * @param schemaId the schemaId of the table
+ * @param schemaGetter the schema getter of to get schema by schemaId
+ */
@VisibleForTesting
public static LogRecordReadContext createIndexedReadContext(
RowType rowType, int schemaId, SchemaGetter schemaGetter) {
@@ -155,40 +162,31 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
* @param rowType the schema of the read data
* @param schemaId the schemaId of the table
* @param selectedFields the final selected fields of the read data
+ * @param schemaGetter the schema getter of to get schema by schemaId
*/
public static LogRecordReadContext createIndexedReadContext(
RowType rowType, int schemaId, int[] selectedFields, SchemaGetter schemaGetter) {
FieldGetter[] fieldGetters = buildProjectedFieldGetters(rowType, selectedFields);
// for INDEXED log format, the projection is NEVER push downed to the server side
return new LogRecordReadContext(
- LogFormat.INDEXED,
- rowType,
- schemaId,
- null,
- null,
- fieldGetters,
- false,
- schemaGetter);
+ LogFormat.INDEXED, rowType, schemaId, null, fieldGetters, false, schemaGetter);
}
private LogRecordReadContext(
LogFormat logFormat,
RowType targetDataRowType,
- int expectedSchemaId,
- VectorSchemaRoot vectorSchemaRoot,
+ int targetSchemaId,
BufferAllocator bufferAllocator,
FieldGetter[] selectedFieldGetters,
boolean projectionPushDowned,
SchemaGetter schemaGetter) {
this.logFormat = logFormat;
this.dataRowType = targetDataRowType;
- this.expectedSchemaId = expectedSchemaId;
- this.vectorSchemaRoot = vectorSchemaRoot;
+ this.targetSchemaId = targetSchemaId;
this.bufferAllocator = bufferAllocator;
this.selectedFieldGetters = selectedFieldGetters;
this.projectionPushDowned = projectionPushDowned;
this.schemaGetter = schemaGetter;
- this.vectorSchemaRootMap = new HashMap<>();
}
@Override
@@ -223,11 +221,6 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
"Only Arrow log format provides vector schema root.");
}
- if (schemaId == this.expectedSchemaId) {
- checkNotNull(vectorSchemaRoot, "The vector schema root is not available.");
- return vectorSchemaRoot;
- }
-
RowType rowType = getRowType(schemaId);
return vectorSchemaRootMap.computeIfAbsent(
schemaId,
@@ -251,15 +244,13 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
if (isSameRowType(schemaId)) {
return null;
}
+ // TODO: should we cache the projection?
Schema originSchema = schemaGetter.getSchema(schemaId);
- Schema expectedSchema = schemaGetter.getSchema(expectedSchemaId);
+ Schema expectedSchema = schemaGetter.getSchema(targetSchemaId);
return ProjectedRow.from(originSchema, expectedSchema);
}
public void close() {
- if (vectorSchemaRoot != null) {
- vectorSchemaRoot.close();
- }
vectorSchemaRootMap.values().forEach(VectorSchemaRoot::close);
if (bufferAllocator != null) {
bufferAllocator.close();
@@ -267,7 +258,7 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
}
private boolean isSameRowType(int schemaId) {
- return expectedSchemaId == schemaId || isProjectionPushDowned();
+ return targetSchemaId == schemaId || isProjectionPushDowned();
}
private static FieldGetter[] buildProjectedFieldGetters(RowType rowType, int[] selectedFields) {
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java b/fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java
new file mode 100644
index 000000000..a2d3da1ae
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.record;
+
+import org.apache.fluss.record.FileLogProjection.ProjectionInfo;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.Cache;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.CacheBuilder;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A cache for projection pushdown information. The cache key is composed of table id, schema id,
+ * and selected column ids. The cache is designed to be shared across different tables and schemas.
+ */
+@ThreadSafe
+public class ProjectionPushdownCache {
+
+ final Cache<ProjectionKey, ProjectionInfo> projectionCache;
+
+ public ProjectionPushdownCache() {
+ // currently, the cache is shared at TabletServer level, so we give a large max size, but
+ // give a short expiration time.
+ // TODO: make the cache parameter configurable
+ this.projectionCache =
+ CacheBuilder.newBuilder()
+ .maximumSize(1000)
+ .expireAfterAccess(Duration.ofMinutes(3))
+ .build();
+ }
+
+ @Nullable
+ public ProjectionInfo getProjectionInfo(long tableId, short schemaId, int[] selectedColumnIds) {
+ ProjectionKey key = new ProjectionKey(tableId, schemaId, selectedColumnIds);
+ return projectionCache.getIfPresent(key);
+ }
+
+ public void setProjectionInfo(
+ long tableId, short schemaId, int[] selectedColumnIds, ProjectionInfo projectionInfo) {
+ ProjectionKey key = new ProjectionKey(tableId, schemaId, selectedColumnIds);
+ projectionCache.put(key, projectionInfo);
+ }
+
+ static final class ProjectionKey {
+ private final long tableId;
+ private final short schemaId;
+ private final int[] selectedColumnIds;
+
+ ProjectionKey(long tableId, short schemaId, int[] selectedColumnIds) {
+ this.tableId = tableId;
+ this.schemaId = schemaId;
+ this.selectedColumnIds = selectedColumnIds;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ProjectionKey)) {
+ return false;
+ }
+ ProjectionKey that = (ProjectionKey) o;
+ return tableId == that.tableId
+ && schemaId == that.schemaId
+ && Arrays.equals(selectedColumnIds, that.selectedColumnIds);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableId, schemaId, Arrays.hashCode(selectedColumnIds));
+ }
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java b/fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java
new file mode 100644
index 000000000..db431f2ac
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.row;
+
+/**
+ * An {@link InternalRow} that pads another {@link InternalRow} with nulls up to a target field
+ * count.
+ */
+public class PaddingRow implements InternalRow {
+
+ private final int targetFieldCount;
+ private InternalRow row;
+
+ public PaddingRow(int targetFieldCount) {
+ this.targetFieldCount = targetFieldCount;
+ }
+
+ /**
+ * Replaces the underlying {@link InternalRow} backing this {@link PaddingRow}.
+ *
+ * <p>This method replaces the row data in place and does not return a new object. This is done
+ * for performance reasons.
+ */
+ public PaddingRow replaceRow(InternalRow row) {
+ this.row = row;
+ return this;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return targetFieldCount;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ if (row.getFieldCount() > pos) {
+ return row.isNullAt(pos);
+ } else {
+ // padding null if pos exceeds the underlying row's field count
+ return true;
+ }
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ return row.getBoolean(pos);
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ return row.getByte(pos);
+ }
+
+ @Override
+ public short getShort(int pos) {
+ return row.getShort(pos);
+ }
+
+ @Override
+ public int getInt(int pos) {
+ return row.getInt(pos);
+ }
+
+ @Override
+ public long getLong(int pos) {
+ return row.getLong(pos);
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ return row.getFloat(pos);
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ return row.getDouble(pos);
+ }
+
+ @Override
+ public BinaryString getChar(int pos, int length) {
+ return row.getChar(pos, length);
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ return row.getString(pos);
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ return row.getDecimal(pos, precision, scale);
+ }
+
+ @Override
+ public TimestampNtz getTimestampNtz(int pos, int precision) {
+ return row.getTimestampNtz(pos, precision);
+ }
+
+ @Override
+ public TimestampLtz getTimestampLtz(int pos, int precision) {
+ return row.getTimestampLtz(pos, precision);
+ }
+
+ @Override
+ public byte[] getBinary(int pos, int length) {
+ return row.getBinary(pos, length);
+ }
+
+ @Override
+ public byte[] getBytes(int pos) {
+ return row.getBytes(pos);
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ return row.getArray(pos);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
index 6b4c516d1..d793f342a 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
@@ -36,11 +36,11 @@ import static org.apache.fluss.utils.SchemaUtil.getIndexMapping;
public class ProjectedRow implements InternalRow {
public static final int UNEXIST_MAPPING = -1;
- protected final int[] indexMapping;
+ private final int[] indexMapping;
- protected InternalRow row;
+ private InternalRow row;
- protected ProjectedRow(int[] indexMapping) {
+ private ProjectedRow(int[] indexMapping) {
this.indexMapping = indexMapping;
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java
new file mode 100644
index 000000000..d809a095f
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.row.decode;
+
+import org.apache.fluss.memory.MemorySegment;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.record.ValueRecord;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.types.DataType;
+import org.apache.fluss.utils.SchemaUtil;
+
+import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH;
+
+/**
+ * A decoder that deserializes raw byte arrays of {@link ValueRecord} with dynamic or
+ * self-describing schemas into structured {@link InternalRow} objects conforming to a predefined,
+ * fixed output schema.
+ */
+public class FixedSchemaDecoder {
+
+ /** The underlying row decoder used to decode the source schema rows. */
+ private final RowDecoder rowDecoder;
+
+ /**
+ * Schema projection mapping from source schema id to target schema field index mapping. The
+ * projection mapping is an array where the index is the target schema field index and the value
+ * is the source schema field index.
+ */
+ private final int[] fieldIdMapping;
+
+ /** Indicates whether there is no projection between source schema and target schema. */
+ private final boolean noProjection;
+
+ public FixedSchemaDecoder(KvFormat kvFormat, Schema sourceSchema, Schema targetSchema) {
+ this.rowDecoder =
+ RowDecoder.create(
+ kvFormat, sourceSchema.getRowType().getChildren().toArray(new DataType[0]));
+ this.fieldIdMapping = SchemaUtil.getIndexMapping(sourceSchema, targetSchema);
+ this.noProjection = false;
+ }
+
+ /**
+ * Creates a FixedSchemaDecoder without projection, i.e., the source schema is the same as the
+ * target schema.
+ */
+ public FixedSchemaDecoder(KvFormat kvFormat, Schema schema) {
+ this.rowDecoder =
+ RowDecoder.create(
+ kvFormat, schema.getRowType().getChildren().toArray(new DataType[0]));
+ this.fieldIdMapping = null;
+ this.noProjection = true;
+ }
+
+ /**
+ * Decode the bytes in the memory segment to {@link InternalRow} that adheres to the fixed
+ * {@code targetSchema}.
+ *
+ * @param segment the memory segment to read.
+ * @param offset the offset in the memory segment to read from.
+ * @param sizeInBytes the total size in bytes to read.
+ */
+ public InternalRow decode(MemorySegment segment, int offset, int sizeInBytes) {
+ if (noProjection) {
+ return rowDecoder.decode(segment, offset, sizeInBytes);
+ } else {
+ InternalRow sourceRow = rowDecoder.decode(segment, offset, sizeInBytes);
+ return ProjectedRow.from(fieldIdMapping).replaceRow(sourceRow);
+ }
+ }
+
+ /**
+ * Decode the value memory segment (in {@link ValueRecord} format) to {@link InternalRow} that
+ * adheres to the fixed {@code targetSchema}.
+ */
+ public InternalRow decode(MemorySegment valueSegment) {
+ return decode(valueSegment, SCHEMA_ID_LENGTH, valueSegment.size() - SCHEMA_ID_LENGTH);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java
index cb3fafa6a..e257d1998 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java
@@ -20,13 +20,14 @@ package org.apache.fluss.row.decode;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.record.ValueRecord;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.DataType;
/**
- * A decoder to read {@link BinaryRow binary format InternalRow} from a byte array or memory
- * segment.
+ * A decoder to read {@link BinaryRow binary format InternalRow} from a byte array or memory segment
+ * of a value record {@link ValueRecord}.
*
* @since 0.2
*/
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
index e0252ae11..794e6ad6b 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
@@ -21,6 +21,7 @@ import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.types.DataType;
@@ -47,7 +48,7 @@ public class ValueDecoder {
}
/** Decode the value bytes and return the schema id and the row encoded in the value bytes. */
- public Value decodeValue(byte[] valueBytes) {
+ public BinaryValue decodeValue(byte[] valueBytes) {
MemorySegment memorySegment = MemorySegment.wrap(valueBytes);
short schemaId = memorySegment.getShort(0);
@@ -64,17 +65,6 @@ public class ValueDecoder {
BinaryRow row =
rowDecoder.decode(
memorySegment, SCHEMA_ID_LENGTH, valueBytes.length - SCHEMA_ID_LENGTH);
- return new Value(schemaId, row);
- }
-
- /** The schema id and {@link BinaryRow} stored as the value of kv store. */
- public static class Value {
- public final short schemaId;
- public final BinaryRow row;
-
- private Value(short schemaId, BinaryRow row) {
- this.schemaId = schemaId;
- this.row = row;
- }
+ return new BinaryValue(schemaId, row);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
index bbfc6a7f5..2d842ab17 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
@@ -17,16 +17,13 @@
package org.apache.fluss.row.encode;
-import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.row.BinaryRow;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.UnsafeUtils;
/** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */
public class ValueEncoder {
- static final int SCHEMA_ID_LENGTH = 2;
+ public static final int SCHEMA_ID_LENGTH = 2;
/**
* Encode the {@code row} with a {@code schemaId} to a byte array value to be expected persisted
@@ -41,24 +38,4 @@ public class ValueEncoder {
row.copyTo(values, SCHEMA_ID_LENGTH);
return values;
}
-
- public static byte[] encodeRow(
- short schemaId, KvFormat kvFormat, RowType currentRowType, InternalRow row)
- throws Exception {
- if (row instanceof BinaryRow) {
- return encodeValue(schemaId, (BinaryRow) row);
- }
- // todo: reuse the encoder here
- try (RowEncoder rowEncoder = RowEncoder.create(kvFormat, currentRowType)) {
- rowEncoder.startNewRow();
- for (int i = 0; i < currentRowType.getFieldCount(); i++) {
- rowEncoder.encodeField(
- i,
- InternalRow.createFieldGetter(currentRowType.getTypeAt(i), i)
- .getFieldOrNull(row));
- }
- BinaryRow binaryRow = rowEncoder.finishRow();
- return encodeValue(schemaId, binaryRow);
- }
- }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
index c597caf81..7c392b055 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
@@ -112,11 +112,7 @@ public class Projection {
return projectionPositions;
}
- /**
- * The id of the projection, which is used for fluss server scan.
- *
- * @return
- */
+ /** The id of the projection, which is used for fluss server scan. */
public int[] getProjectionIdInOrder() {
return projectionIdsInOrder;
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
index 6b66c0cd6..f10782ea1 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
@@ -20,7 +20,6 @@ package org.apache.fluss.utils;
import org.apache.fluss.exception.SchemaChangeException;
import org.apache.fluss.metadata.Schema;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -32,37 +31,17 @@ import static org.apache.fluss.row.ProjectedRow.UNEXIST_MAPPING;
public class SchemaUtil {
/**
- * get the target columns in expected schema.
+ * Get the index mapping from origin schema to expected schema. This mapping can be used to
+ * convert the row with origin schema to the row with expected schema via {@code
+ * ProjectedRow.from(getIndexMapping(inputSchema, targetSchema)).replace(inputRow)}.
*
- * @param targetColumns
- * @param originSchema
- * @param expectedSchema
- * @return
- */
- public static int[] getTargetColumns(
- int[] targetColumns, Schema originSchema, Schema expectedSchema) {
- if (targetColumns == null) {
- return null;
- }
-
- // get the mapping from expected schema to origin schema
- int[] indexMapping = getIndexMapping(expectedSchema, originSchema);
- ArrayList<Integer> targetColumnsInExpectedSchema = new ArrayList<>();
- for (int i = 0; i < targetColumns.length; i++) {
- if (indexMapping[targetColumns[i]] != UNEXIST_MAPPING) {
- targetColumnsInExpectedSchema.add(indexMapping[targetColumns[i]]);
- }
- }
- return targetColumnsInExpectedSchema.stream().mapToInt(value -> value).toArray();
- }
-
- /**
- * Get the index of each value of expectedSchema from originSchema.
- *
- * @param originSchema
- * @param expectedSchema
- * @return indexMapping the value of i-th means the expectedSchema's i-th column is the
- * originSchema's indexMapping[i]-th column.
+ * @param originSchema The origin schema.
+ * @param expectedSchema The expected schema.
+ * @return The index mapping array. The length of the array is the number of columns in the
+ * expected schema. Each element in the array is the index of the corresponding column in
+ * the origin schema. If a column in the expected schema does not exist in the origin
+ * schema, the corresponding element is UNEXIST_MAPPING (-1).
+ * @throws SchemaChangeException if there is a datatype mismatch between the two schemas.
*/
public static int[] getIndexMapping(Schema originSchema, Schema expectedSchema) {
List<Schema.Column> originColumns = originSchema.getColumns();
@@ -84,7 +63,7 @@ public class SchemaUtil {
if (originColumn != null
&& !Objects.equals(
expectedColumn.getDataType().copy(true),
- originColumns.get(indexMapping[i]).getDataType().copy(true))) {
+ originColumn.getDataType().copy(true))) {
throw new SchemaChangeException(
String.format(
@@ -92,7 +71,8 @@ public class SchemaUtil {
expectedColumn.getColumnId(),
expectedColumn.getName(),
expectedColumn.getDataType(),
- originColumn.getDataType()));
+ originColumn.getDataType()),
+ true);
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
index 2c3ff36ef..9a27089ac 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
@@ -78,7 +78,8 @@ class FileLogProjectionTest {
TestData.DATA2,
TestData.DATA2);
- FileLogProjection projection = new FileLogProjection();
+ ProjectionPushdownCache cache = new ProjectionPushdownCache();
+ FileLogProjection projection = new FileLogProjection(cache);
// get schema during running.
doProjection(
1L,
@@ -87,16 +88,13 @@ class FileLogProjectionTest {
recordsOfData2RowType,
new int[] {0, 2},
recordsOfData2RowType.sizeInBytes());
- FileLogProjection.ProjectionInfo info1 = projection.currentProjection;
+ assertThat(cache.projectionCache.size()).isEqualTo(1);
+ FileLogProjection.ProjectionInfo info1 =
+ cache.getProjectionInfo(1L, schemaId, new int[] {0, 2});
assertThat(info1).isNotNull();
assertThat(info1.nodesProjection.stream().toArray()).isEqualTo(new int[] {0, 2});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info1.buffersProjection.stream().toArray()).isEqualTo(new int[] {0, 1, 5, 6, 7});
- assertThat(projection.projectionsCache).hasSize(1);
- assertThat(
- projection.projectionsCache.get(
- new FileLogProjection.ProjectionKey(1L, schemaId)))
- .isSameAs(info1);
doProjection(
2L,
@@ -105,37 +103,13 @@ class FileLogProjectionTest {
recordsOfData2RowType,
new int[] {1},
recordsOfData2RowType.sizeInBytes());
- FileLogProjection.ProjectionInfo info2 = projection.currentProjection;
+ assertThat(cache.projectionCache.size()).isEqualTo(2);
+ FileLogProjection.ProjectionInfo info2 =
+ cache.getProjectionInfo(2L, schemaId, new int[] {1});
assertThat(info2).isNotNull();
assertThat(info2.nodesProjection.stream().toArray()).isEqualTo(new int[] {1});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info2.buffersProjection.stream().toArray()).isEqualTo(new int[] {2, 3, 4});
- assertThat(projection.projectionsCache).hasSize(2);
- assertThat(
- projection.projectionsCache.get(
- new FileLogProjection.ProjectionKey(2L, schemaId)))
- .isSameAs(info2);
-
- doProjection(
- 1L,
- schemaId,
- projection,
- recordsOfData2RowType,
- new int[] {0, 2},
- recordsOfData2RowType.sizeInBytes());
- assertThat(projection.currentProjection).isNotNull().isSameAs(info1);
-
- assertThatThrownBy(
- () ->
- doProjection(
- 1L,
- schemaId,
- projection,
- recordsOfData2RowType,
- new int[] {1},
- recordsOfData2RowType.sizeInBytes()))
- .isInstanceOf(InvalidColumnProjectionException.class)
- .hasMessage("The schema and projection should be identical for the same table id.");
}
@Test
@@ -149,7 +123,7 @@ class FileLogProjectionTest {
TestData.DATA2_ROW_TYPE,
TestData.DATA2,
TestData.DATA2);
- FileLogProjection projection = new FileLogProjection();
+ FileLogProjection projection = new FileLogProjection(new ProjectionPushdownCache());
assertThatThrownBy(
() ->
doProjection(
@@ -160,7 +134,7 @@ class FileLogProjectionTest {
new int[] {3},
recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
- .hasMessage("Projected field id 3 is not contains in [0, 1, 2]");
+ .hasMessage("Projected field id 3 is not contained in [0, 1, 2]");
assertThatThrownBy(
() ->
@@ -220,7 +194,7 @@ class FileLogProjectionTest {
doProjection(
1L,
schemaId,
- new FileLogProjection(),
+ new FileLogProjection(new ProjectionPushdownCache()),
fileLogRecords,
projectedFields,
Integer.MAX_VALUE);
@@ -248,7 +222,7 @@ class FileLogProjectionTest {
TestData.DATA1_ROW_TYPE,
TestData.DATA1,
TestData.ANOTHER_DATA1);
- FileLogProjection projection = new FileLogProjection();
+ FileLogProjection projection = new FileLogProjection(new ProjectionPushdownCache());
// overwrite the wrong decoding byte order endian
projection.getLogHeaderBuffer().order(ByteOrder.BIG_ENDIAN);
assertThatThrownBy(
@@ -305,7 +279,10 @@ class FileLogProjectionTest {
int maxBytes = totalSize / i;
List<Object[]> results =
doProjection(
- new FileLogProjection(), fileLogRecords, new int[] {0, 1}, maxBytes);
+ new FileLogProjection(new ProjectionPushdownCache()),
+ fileLogRecords,
+ new int[] {0, 1},
+ maxBytes);
if (results.isEmpty()) {
hasEmpty = true;
} else if (results.size() == TestData.DATA1.size()) {
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
index 108ab2acf..36aef5b9d 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
@@ -72,7 +72,6 @@ public abstract class LogTestBase {
assertLogRecordsListEquals(expected, actual, baseRowType);
}
- // todo: 在这个基础上可以加更多最简单的schema evolution测试，非常好
protected void assertIndexedLogRecordBatchAndRowEquals(
LogRecordBatch actual,
LogRecordBatch expected,
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
index f74f379eb..9a9581cab 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
@@ -24,11 +24,12 @@ import org.apache.fluss.metadata.SchemaInfo;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
/** A testing schema metadata subscriber. */
public class TestingSchemaGetter implements SchemaGetter {
private SchemaInfo schemaInfo;
- private Map<Integer, Schema> schemaCaches;
+ private final Map<Integer, Schema> schemaCaches;
public TestingSchemaGetter() {
this.schemaInfo = null;
@@ -56,6 +57,15 @@ public class TestingSchemaGetter implements SchemaGetter {
return schemaCaches.get(schemaId);
}
+ @Override
+ public CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId) {
+ if (!schemaCaches.containsKey(schemaId)) {
+ throw new SchemaNotExistException("Schema not exist");
+ }
+ return CompletableFuture.completedFuture(
+ new SchemaInfo(schemaCaches.get(schemaId), schemaId));
+ }
+
@Override
public SchemaInfo getLatestSchemaInfo() {
return schemaInfo;
diff --git a/fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java b/fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java
new file mode 100644
index 000000000..453f02be3
--- /dev/null
+++ b/fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.row;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link PaddingRow}. */
+class PaddingRowTest {
+
+ @Test
+ void testPaddingRowIsNullAt() {
+ // mock an InternalRow with only 2 columns
+ InternalRow row = GenericRow.of(1, null);
+ PaddingRow paddingRow = new PaddingRow(4).replaceRow(row);
+
+ assertThat(paddingRow.getFieldCount()).isEqualTo(4);
+ assertThat(paddingRow.isNullAt(0)).isFalse();
+ assertThat(paddingRow.isNullAt(1)).isTrue();
+ assertThat(paddingRow.isNullAt(2)).isTrue(); // exceed original row, should be null
+ assertThat(paddingRow.isNullAt(3)).isTrue();
+ }
+
+ @Test
+ void testPaddingRowDelegatesGetters() {
+ InternalRow row =
+ GenericRow.of(
+ true,
+ (byte) 2,
+ (short) 3,
+ 4,
+ 5L,
+ 6.5f,
+ 7.5,
+ new byte[] {1, 2, 3},
+ new byte[] {4, 5},
+ BinaryString.fromString("abc"),
+ TimestampLtz.fromEpochMillis(1000));
+ PaddingRow paddingRow = new PaddingRow(9).replaceRow(row);
+
+ assertThat(paddingRow.getBoolean(0)).isTrue();
+ assertThat(paddingRow.getByte(1)).isEqualTo((byte) 2);
+ assertThat(paddingRow.getShort(2)).isEqualTo((short) 3);
+ assertThat(paddingRow.getInt(3)).isEqualTo(4);
+ assertThat(paddingRow.getLong(4)).isEqualTo(5L);
+ assertThat(paddingRow.getFloat(5)).isEqualTo(6.5f);
+ assertThat(paddingRow.getDouble(6)).isEqualTo(7.5);
+ assertThat(paddingRow.getBinary(7, 3)).isEqualTo(new byte[] {1, 2, 3});
+ assertThat(paddingRow.getBytes(8)).isEqualTo(new byte[] {4, 5});
+ assertThat(paddingRow.getString(9)).isEqualTo(BinaryString.fromString("abc"));
+ assertThat(paddingRow.getTimestampLtz(10, 3)).isEqualTo(TimestampLtz.fromEpochMillis(1000));
+ }
+
+ @Test
+ void testReplaceRowInPlace() {
+ InternalRow row1 = GenericRow.of(1);
+ InternalRow row2 = GenericRow.of(2);
+ PaddingRow paddingRow = new PaddingRow(1).replaceRow(row1);
+ assertThat(paddingRow.getInt(0)).isEqualTo(1);
+ paddingRow.replaceRow(row2);
+ assertThat(paddingRow.getInt(0)).isEqualTo(2);
+ }
+}
diff --git a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
index 09467d31a..80b21a719 100644
--- a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
@@ -183,6 +183,7 @@ class ProjectedRowTest {
.isExactlyInstanceOf(SchemaChangeException.class)
.hasMessage(
"Expected datatype of column(id=0,name=a) is [INT], while the actual datatype is [BIGINT]");
+
assertThatThrownBy(
() ->
ProjectedRow.from(
diff --git a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
index bd1b6af81..12b05f186 100644
--- a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
+++ b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level = INFO
+rootLogger.level = OFF
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
index 314e4b40f..003e910ea 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
@@ -23,34 +23,17 @@ import org.apache.fluss.row.InternalArray;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
-import org.apache.fluss.types.RowType;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.TimestampData;
-import javax.annotation.Nullable;
-
-import java.util.List;
-
/** Wraps a Flink {@link RowData} as a Fluss {@link InternalRow}. */
public class FlinkAsFlussRow implements InternalRow {
private RowData flinkRow;
- /**
- * Creates a FlinkAsFlussRow with a mapping from Fluss field index to Flink field index. If the
- * indexMapping is null, flink and flink share same field name.
- */
- @Nullable private final int[] indexMapping;
-
- public FlinkAsFlussRow() {
- this(null);
- }
-
- public FlinkAsFlussRow(int[] indexMapping) {
- this.indexMapping = indexMapping;
- }
+ public FlinkAsFlussRow() {}
public FlinkAsFlussRow replace(RowData flinkRow) {
this.flinkRow = flinkRow;
@@ -59,94 +42,62 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public int getFieldCount() {
- if (indexMapping == null) {
- return flinkRow.getArity();
- } else {
- return indexMapping.length;
- }
+ return flinkRow.getArity();
}
@Override
public boolean isNullAt(int pos) {
- // if no mapping, means that the field is not found in Flink, just ignore it.
- if ((indexMapping != null && indexMapping[pos] == -1)) {
- return true;
- }
-
- // If pos is larger than flinkRow.getArity(), it indicates that the Fluss table's data type
- // is wider than the data provided by Flink.
- // This often occurs when a schema change happens to the catalog table before job restart.
- // Only appending columns at the end is compatible, just need to ignore the trailing
- // columns.
- // Other types of schema changes (e.g., dropping columns) would lead to incompatibility.
- // Therefore, Fluss currently only supports appending columns, and here we simply ignore any
- // extra columns.
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- if (flussPos >= flinkRow.getArity()) {
- return true;
- }
-
- return flinkRow.isNullAt(flussPos);
+ return flinkRow.isNullAt(pos);
}
@Override
public boolean getBoolean(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getBoolean(flussPos);
+ return flinkRow.getBoolean(pos);
}
@Override
public byte getByte(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getByte(flussPos);
+ return flinkRow.getByte(pos);
}
@Override
public short getShort(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getShort(flussPos);
+ return flinkRow.getShort(pos);
}
@Override
public int getInt(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getInt(flussPos);
+ return flinkRow.getInt(pos);
}
@Override
public long getLong(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getLong(flussPos);
+ return flinkRow.getLong(pos);
}
@Override
public float getFloat(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getFloat(flussPos);
+ return flinkRow.getFloat(pos);
}
@Override
public double getDouble(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getDouble(flussPos);
+ return flinkRow.getDouble(pos);
}
@Override
public BinaryString getChar(int pos, int length) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
+ return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
}
@Override
public BinaryString getString(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
+ return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
}
@Override
public Decimal getDecimal(int pos, int precision, int scale) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return fromFlinkDecimal(flinkRow.getDecimal(flussPos, precision, scale));
+ return fromFlinkDecimal(flinkRow.getDecimal(pos, precision, scale));
}
public static Decimal fromFlinkDecimal(DecimalData decimal) {
@@ -161,43 +112,26 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public TimestampNtz getTimestampNtz(int pos, int precision) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
+ TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
return TimestampNtz.fromMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public TimestampLtz getTimestampLtz(int pos, int precision) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
+ TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
return TimestampLtz.fromEpochMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public byte[] getBinary(int pos, int length) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getBinary(flussPos);
+ return flinkRow.getBinary(pos);
}
@Override
public byte[] getBytes(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getBinary(flussPos);
- }
-
- /** Converts a Flink {@link RowType} to a Fluss {@link RowType}, based on the field name. */
- public static FlinkAsFlussRow from(
- org.apache.flink.table.types.logical.RowType flinkRowType, RowType flussRowType) {
- int[] indexMapping = new int[flussRowType.getFieldCount()];
- List<String> fieldNames = flussRowType.getFieldNames();
- for (int i = 0; i < flussRowType.getFieldCount(); i++) {
- String fieldName = fieldNames.get(i);
- int fieldIndex = flinkRowType.getFieldIndex(fieldName);
- indexMapping[i] = fieldIndex;
- }
- return new FlinkAsFlussRow(indexMapping);
+ return flinkRow.getBinary(pos);
}
@Override
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
index 0f3fff210..2096b1095 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
@@ -69,10 +69,11 @@ public interface FlussSerializationSchema<T> extends Serializable {
RowType getRowSchema();
/**
- * Returns the Flink table row schema.
+ * Returns the input row schema which is the Flink {@link
+ * org.apache.flink.table.data.RowData}.
*
- * @return The schema of the Flink table row.
+ * @return The schema of the input Flink row.
*/
- org.apache.flink.table.types.logical.RowType getTableRowType();
+ org.apache.flink.table.types.logical.RowType getInputRowSchema();
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
index 05e2921fc..1a46ec0bc 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
@@ -21,11 +21,49 @@ import org.apache.fluss.flink.row.FlinkAsFlussRow;
import org.apache.fluss.flink.row.OperationType;
import org.apache.fluss.flink.row.RowWithOp;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.PaddingRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.flink.table.data.RowData;
import org.apache.flink.types.RowKind;
-/** Default implementation of RowDataConverter for RowData. */
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/**
+ * Default implementation of RowDataConverter for RowData.
+ *
+ * <p>Schema Evolution Handling:
+ *
+ * <p>There are three schema in sink case.
+ *
+ * <ul>
+ * <li>(1) Consumed Row Schema (RowData): the schema of RowData consumed, currently, Flink doesn't
+ * provide API to fetch it, but we can get the row arity from RowData.getArity().
+ * <li>(2) Plan Row Schema ({@link InitializationContext#getInputRowSchema()}): the compiled input
+ * row schema of the sink, but the schema is fetched from catalog, so it is not the consumed
+ * row schema. This is updated when job is re-compiled.
+ * <li>(3) Table Latest Row Schema ({@link InitializationContext#getRowSchema()}): the latest
+ * schema of the sink table, which is fetched at open() during runtime, so it will be updated
+ * when Flink job restarts.
+ * </ul>
+ *
+ * <p>We always want to use latest schema to write data, so we need to add conversions from consumed
+ * Flink RowData.
+ *
+ * <ul>
+ * <li>The {@link #converter} is used to wrap Flink {@link RowData} into Fluss {@link InternalRow}
+ * without any schema transformation.
+ * <li>The {@link #outputPadding} is used to pad nulls for new columns when new columns are added.
+ * This may happen when table is added new columns after Flink SQL job restored from
+ * CompiledPlan that the Consumed Row Schema is old but the Plan Row Schema is new, so the
+ * Consumed Row Schema has less fields than Plan Row Schema.
+ * <li>The {@link #outputProjection} is used to re-arrange the fields according to latest schema
+ * if Plan Row Schema is not match Table Latest Row Schema. This may happen when table is
+ * added new columns after the Flink job compiled and before job restarts.
+ * </ul>
+ */
public class RowDataSerializationSchema implements FlussSerializationSchema<RowData> {
private static final long serialVersionUID = 1L;
@@ -44,6 +82,19 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
*/
private transient FlinkAsFlussRow converter;
+ /**
+ * The padding row for output, used when the input row has fewer fields than the target schema.
+ * This may happen when table is added new columns between Flink job restarts.
+ */
+ private transient PaddingRow outputPadding;
+
+ /**
+ * The projected row for output, used when schema evolution occurs, because we want to write
+ * rows using new schema (e.g., fill null for new columns). This may happen when table is *
+ * added new columns after the Flink job compiled and before job restarts.
+ */
+ @Nullable private transient ProjectedRow outputProjection;
+
/**
* Constructs a new {@code RowSerializationSchema}.
*
@@ -63,7 +114,20 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
*/
@Override
public void open(InitializationContext context) throws Exception {
- this.converter = FlinkAsFlussRow.from(context.getTableRowType(), context.getRowSchema());
+ this.converter = new FlinkAsFlussRow();
+ List<String> targetFieldNames = context.getRowSchema().getFieldNames();
+ List<String> inputFieldNames = context.getInputRowSchema().getFieldNames();
+ this.outputPadding = new PaddingRow(inputFieldNames.size());
+ if (targetFieldNames.size() != inputFieldNames.size()) {
+ // there is a schema evolution happens (e.g., ADD COLUMN), need to build index mapping
+ int[] indexMapping = new int[targetFieldNames.size()];
+ for (int i = 0; i < targetFieldNames.size(); i++) {
+ String fieldName = targetFieldNames.get(i);
+ int fieldIndex = inputFieldNames.indexOf(fieldName);
+ indexMapping[i] = fieldIndex;
+ }
+ outputProjection = ProjectedRow.from(indexMapping);
+ }
}
/**
@@ -82,6 +146,14 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
"Converter not initialized. The open() method must be called before serializing records.");
}
InternalRow row = converter.replace(value);
+ // handling schema evolution for changes before job compilation
+ if (row.getFieldCount() < outputPadding.getFieldCount()) {
+ row = outputPadding.replaceRow(row);
+ }
+ // handling schema evolution for changes after job compilation
+ if (outputProjection != null) {
+ row = outputProjection.replaceRow(row);
+ }
OperationType opType = toOperationType(value.getRowKind());
return new RowWithOp(row, opType);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
index 93b849e58..cc1d168cd 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
@@ -47,7 +47,7 @@ public class SerializerInitContextImpl implements FlussSerializationSchema.Initi
}
@Override
- public org.apache.flink.table.types.logical.RowType getTableRowType() {
+ public org.apache.flink.table.types.logical.RowType getInputRowSchema() {
return flinkRowType;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
index a4469fd32..fe0eb28ea 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
@@ -29,7 +29,6 @@ import org.apache.fluss.flink.row.OperationType;
import org.apache.fluss.flink.row.RowWithOp;
import org.apache.fluss.flink.sink.serializer.FlussSerializationSchema;
import org.apache.fluss.flink.sink.serializer.SerializerInitContextImpl;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.metrics.Gauge;
@@ -111,10 +110,10 @@ public abstract class FlinkSinkWriter<InputT> implements SinkWriter<InputT> {
connection = ConnectionFactory.createConnection(flussConfig, flinkMetricRegistry);
table = connection.getTable(tablePath);
LOG.info(
- "Current Fluss Schema is {}, Table RowType is {}",
+ "Current Fluss Schema is {}, input Flink RowType is {}",
table.getTableInfo().getSchema(),
tableRowType);
- // sanityCheck(table.getTableInfo());
+ sanityCheck(table.getTableInfo());
try {
this.serializationSchema.open(
@@ -209,22 +208,6 @@ public abstract class FlinkSinkWriter<InputT> implements SinkWriter<InputT> {
"Primary key constraint is not matched between metadata in Fluss (%s) and Flink (%s).",
flussTableInfo.hasPrimaryKey(), hasPrimaryKey));
}
- RowType currentTableRowType = FlinkConversions.toFlinkRowType(flussTableInfo.getRowType());
- if (!this.tableRowType.copy(false).equals(currentTableRowType.copy(false))) {
- // The default nullability of Flink row type and Fluss row type might be not the same,
- // thus we need to compare the row type without nullability here.
-
- // Throw exception if the schema is the not same, this should rarely happen because we
- // only allow fluss tables derived from fluss catalog. But this can happen if an ALTER
- // TABLE command executed on the fluss table, after the job is submitted but before the
- // SinkFunction is opened.
- throw new ValidationException(
- "The Flink query schema is not matched to current Fluss table schema. "
- + "\nFlink query schema: "
- + this.tableRowType
- + "\nFluss table schema: "
- + currentTableRowType);
- }
}
private long computeSendTime() {
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
index 0bf9f187c..bc15390fe 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
@@ -60,6 +60,7 @@ public class FlinkSource<OUT>
private final boolean hasPrimaryKey;
private final boolean isPartitioned;
private final RowType sourceOutputType;
+ @Nullable private final int[] projectedFields;
protected final OffsetsInitializer offsetsInitializer;
protected final long scanPartitionDiscoveryIntervalMs;
private final boolean streaming;
@@ -73,6 +74,7 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -84,6 +86,7 @@ public class FlinkSource<OUT>
hasPrimaryKey,
isPartitioned,
sourceOutputType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
@@ -98,6 +101,7 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -109,6 +113,7 @@ public class FlinkSource<OUT>
this.hasPrimaryKey = hasPrimaryKey;
this.isPartitioned = isPartitioned;
this.sourceOutputType = sourceOutputType;
+ this.projectedFields = projectedFields;
this.offsetsInitializer = offsetsInitializer;
this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs;
this.deserializationSchema = deserializationSchema;
@@ -182,7 +187,6 @@ public class FlinkSource<OUT>
context.getUserCodeClassLoader(),
sourceOutputType));
FlinkRecordEmitter<OUT> recordEmitter = new FlinkRecordEmitter<>(deserializationSchema);
- // recall to projectedFields
return new FlinkSourceReader<>(
elementsQueue,
@@ -190,6 +194,7 @@ public class FlinkSource<OUT>
tablePath,
sourceOutputType,
context,
+ projectedFields,
flinkSourceReaderMetrics,
recordEmitter,
lakeSource);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
index b110e9f88..ef4b63812 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
@@ -325,6 +325,7 @@ public class FlinkTableSource
hasPrimaryKey(),
isPartitioned(),
flussRowType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
new RowDataDeserializationSchema(),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
index 5f0b2ff8c..427741834 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
@@ -24,6 +24,8 @@ import org.apache.fluss.flink.source.enumerator.initializer.OffsetsInitializer;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.types.RowType;
+import javax.annotation.Nullable;
+
/**
* A Flink DataStream source implementation for reading data from Fluss tables.
*
@@ -61,6 +63,7 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -72,6 +75,7 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
index 4891ab256..f16639753 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
@@ -296,6 +296,7 @@ public class FlussSourceBuilder<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
index 6e4647f80..82fd1bbc3 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
@@ -27,10 +27,12 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.TableNotExistException;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
import org.apache.fluss.flink.source.lookup.LookupNormalizer.RemainingFilter;
+import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.utils.ExceptionUtils;
import org.apache.flink.table.data.RowData;
@@ -47,6 +49,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.stream.IntStream;
/** A flink async lookup function for fluss. */
public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
@@ -59,7 +62,7 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
private final TablePath tablePath;
private final RowType flinkRowType;
private final LookupNormalizer lookupNormalizer;
- @Nullable private final int[] projection;
+ @Nullable private int[] projection;
private transient FlussRowToFlinkRowConverter flussRowToFlinkRowConverter;
private transient Connection connection;
@@ -85,20 +88,22 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
- org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
- LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
- // Currently, only primary key and prefix lookup are supported. These keys must be primary
- // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
if (projection == null) {
outputRowType = flinkRowType;
+ // we force to do projection if no projection pushdown, in order to handle schema
+ // changes (ADD COLUMN LAST), this guarantees the input row of
+ // flussRowToFlinkRowConverter is in expected schema even new columns are added.
+ projection = IntStream.range(0, flinkRowType.getFieldCount()).toArray();
} else {
outputRowType = FlinkUtils.projectRowType(flinkRowType, projection);
}
-
- flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
+ // TODO: currently, we assume only ADD COLUMN LAST schema changes, so the projection
+ // positions can still work even after such changes.
+ flussRowToFlinkRowConverter =
+ new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -164,7 +169,7 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
List<RowData> projectedRow = new ArrayList<>();
for (InternalRow row : lookupResult) {
if (row != null) {
- RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
+ RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRow.add(flinkRow);
}
@@ -173,6 +178,14 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
resultFuture.complete(projectedRow);
}
+ private InternalRow maybeProject(InternalRow row) {
+ if (projection == null) {
+ return row;
+ }
+ // should not reuse objects for async operations
+ return ProjectedRow.from(projection).replaceRow(row);
+ }
+
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
index e9a5ad915..477666737 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
@@ -25,6 +25,7 @@ import org.apache.fluss.client.lookup.Lookuper;
import org.apache.fluss.client.table.Table;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
+import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
@@ -44,6 +45,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.stream.IntStream;
/** A flink lookup function for fluss. */
public class FlinkLookupFunction extends LookupFunction {
@@ -82,22 +84,25 @@ public class FlinkLookupFunction extends LookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
- org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
- LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
- // Currently, only primary key and prefix lookup are supported. These keys must be primary
- // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
if (projection == null) {
outputRowType = flinkRowType;
- projectedRow = null;
+ // we force to do projection if no projection pushdown, in order to handle schema
+ // changes (ADD COLUMN LAST), this guarantees the input row of
+ // flussRowToFlinkRowConverter is in expected schema even new columns are added.
+ projectedRow =
+ ProjectedRow.from(IntStream.range(0, flinkRowType.getFieldCount()).toArray());
} else {
outputRowType = FlinkUtils.projectRowType(flinkRowType, projection);
// reuse the projected row
projectedRow = ProjectedRow.from(projection);
}
- flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
+ // TODO: currently, we assume only ADD COLUMN LAST schema changes, so the projection
+ // positions can still work even after such changes.
+ flussRowToFlinkRowConverter =
+ new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -133,7 +138,8 @@ public class FlinkLookupFunction extends LookupFunction {
List<RowData> projectedRows = new ArrayList<>();
for (InternalRow row : lookupRows) {
if (row != null) {
- RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
+ RowData flinkRow =
+ flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRows.add(flinkRow);
}
@@ -146,6 +152,13 @@ public class FlinkLookupFunction extends LookupFunction {
}
}
+ private InternalRow maybeProject(InternalRow row) {
+ if (projectedRow == null) {
+ return row;
+ }
+ return projectedRow.replaceRow(row);
+ }
+
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
index 639d52513..6364532bb 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
@@ -40,6 +40,8 @@ import org.apache.flink.api.connector.source.SourceReaderContext;
import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
+import javax.annotation.Nullable;
+
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
@@ -55,6 +57,7 @@ public class FlinkSourceReader<OUT>
TablePath tablePath,
RowType sourceOutputType,
SourceReaderContext context,
+ @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
FlinkRecordEmitter<OUT> recordEmitter,
LakeSource<LakeSplit> lakeSource) {
@@ -67,6 +70,7 @@ public class FlinkSourceReader<OUT>
flussConfig,
tablePath,
sourceOutputType,
+ projectedFields,
flinkSourceReaderMetrics,
lakeSource),
(ignore) -> {}),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
index 10997e91e..ce50fe406 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
@@ -58,6 +58,7 @@ import java.io.IOException;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -119,6 +120,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
Configuration flussConf,
TablePath tablePath,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
@Nullable LakeSource<LakeSplit> lakeSource) {
this.flinkMetricRegistry =
@@ -128,17 +130,15 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
this.sourceOutputType = sourceOutputType;
this.boundedSplits = new ArrayDeque<>();
this.subscribedBuckets = new HashMap<>();
+ this.projectedFields = projectedFields;
+ if (projectedFields == null) {}
+
this.flinkSourceReaderMetrics = flinkSourceReaderMetrics;
- this.projectedFields =
- reCalculateProjectedFields(sourceOutputType, table.getTableInfo().getRowType());
+ sanityCheck(table.getTableInfo().getRowType(), projectedFields);
this.logScanner = table.newScan().project(projectedFields).createLogScanner();
this.stoppingOffsets = new HashMap<>();
this.emptyLogSplits = new HashSet<>();
this.lakeSource = lakeSource;
- LOG.info(
- "fluss table schema: {}, flink table output type:{}",
- table.getTableInfo().getSchema(),
- sourceOutputType);
}
@Override
@@ -216,8 +216,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
LakeSnapshotAndFlussLogSplit lakeSnapshotAndFlussLogSplit =
(LakeSnapshotAndFlussLogSplit) sourceSplitBase;
if (lakeSnapshotAndFlussLogSplit.isStreaming()) {
- // is streaming split which has no stopping offset, we need also
- // subscribe
+ // is streaming split which has no stopping offset, we need also subscribe
// change log
subscribeLog(
lakeSnapshotAndFlussLogSplit,
@@ -564,33 +563,41 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
flinkMetricRegistry.close();
}
- /**
- * The projected fields for the fluss table from the source output types. Mapping based on
- * column name rather thn column id.
- */
- private static int[] reCalculateProjectedFields(
- RowType sourceOutputType, RowType flussRowType) {
- if (sourceOutputType.copy(false).equals(flussRowType.copy(false))) {
- return null;
- }
-
- List<String> fieldNames = sourceOutputType.getFieldNames();
- int[] projectedFlussFields = new int[fieldNames.size()];
- for (int i = 0; i < fieldNames.size(); i++) {
- int fieldIndex = flussRowType.getFieldIndex(fieldNames.get(i));
- if (fieldIndex == -1) {
- throw new ValidationException(
- String.format(
- "The field %s is not found in the fluss table.",
- fieldNames.get(i)));
+ private void sanityCheck(RowType flussTableRowType, @Nullable int[] projectedFields) {
+ RowType tableRowType =
+ projectedFields != null
+ ? flussTableRowType.project(projectedFields)
+ // only read the output fields from source
+ : flussTableRowType.project(sourceOutputType.getFieldNames());
+ if (!sourceOutputType.copy(false).equals(tableRowType.copy(false))) {
+ // The default nullability of Flink row type and Fluss row type might be not the same,
+ // thus we need to compare the row type without nullability here.
+
+ final String flussSchemaMsg;
+ if (projectedFields == null) {
+ flussSchemaMsg = "\nFluss table schema: " + tableRowType;
+ } else {
+ flussSchemaMsg =
+ "\nFluss table schema: "
+ + tableRowType
+ + " (projection "
+ + Arrays.toString(projectedFields)
+ + ")";
}
- projectedFlussFields[i] = fieldIndex;
+ // Throw exception if the schema is the not same, this should rarely happen because we
+ // only allow fluss tables derived from fluss catalog. But this can happen if an ALTER
+ // TABLE command executed on the fluss table, after the job is submitted but before the
+ // SinkFunction is opened.
+ throw new ValidationException(
+ "The Flink query schema is not matched to Fluss table schema. "
+ + "\nFlink query schema: "
+ + sourceOutputType
+ + flussSchemaMsg);
}
- return projectedFlussFields;
}
@VisibleForTesting
- public int[] getProjectedFields() {
+ int[] getProjectedFields() {
return projectedFields;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
index 74375e24c..2553b0c18 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
@@ -86,18 +86,6 @@ public class FlinkConversions {
private FlinkConversions() {}
- public static int[] toFlinkRowTypeIndexMapping(
- RowType sourceFlussRowType,
- org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
- int[] indexMapping = new int[targetFlinkRowType.getFieldCount()];
- for (int i = 0; i < targetFlinkRowType.getFieldCount(); i++) {
- String fieldName = targetFlinkRowType.getFieldNames().get(i);
- int index = sourceFlussRowType.getFieldIndex(fieldName);
- indexMapping[i] = index;
- }
- return indexMapping;
- }
-
/** Convert Fluss's type to Flink's type. */
@VisibleForTesting
public static org.apache.flink.table.types.DataType toFlinkType(DataType flussDataType) {
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
index 4dc5287b0..e52a83a82 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
@@ -39,32 +39,18 @@ import org.apache.flink.types.RowKind;
import java.io.Serializable;
import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowKind;
-import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
/** A converter to convert Fluss's {@link InternalRow} to Flink's {@link RowData}. */
public class FlussRowToFlinkRowConverter {
private final FlussDeserializationConverter[] toFlinkFieldConverters;
private final InternalRow.FieldGetter[] flussFieldGetters;
- private final int[] indexMapping;
- // todo: remove
public FlussRowToFlinkRowConverter(RowType rowType) {
- this(rowType, toFlinkRowType(rowType));
- }
-
- public FlussRowToFlinkRowConverter(
- RowType sourceFlussRowType,
- org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
- this.indexMapping =
- FlinkConversions.toFlinkRowTypeIndexMapping(sourceFlussRowType, targetFlinkRowType);
- this.toFlinkFieldConverters =
- new FlussDeserializationConverter[sourceFlussRowType.getFieldCount()];
- this.flussFieldGetters = new InternalRow.FieldGetter[sourceFlussRowType.getFieldCount()];
- for (int i = 0; i < sourceFlussRowType.getFieldCount(); i++) {
- toFlinkFieldConverters[i] =
- createNullableInternalConverter(sourceFlussRowType.getTypeAt(i));
- flussFieldGetters[i] =
- InternalRow.createFieldGetter(sourceFlussRowType.getTypeAt(i), i);
+ this.toFlinkFieldConverters = new FlussDeserializationConverter[rowType.getFieldCount()];
+ this.flussFieldGetters = new InternalRow.FieldGetter[rowType.getFieldCount()];
+ for (int i = 0; i < rowType.getFieldCount(); i++) {
+ toFlinkFieldConverters[i] = createNullableInternalConverter(rowType.getTypeAt(i));
+ flussFieldGetters[i] = InternalRow.createFieldGetter(rowType.getTypeAt(i), i);
}
}
@@ -77,18 +63,11 @@ public class FlussRowToFlinkRowConverter {
}
private RowData toFlinkRowData(InternalRow flussRow, RowKind rowKind) {
- int targetLength = indexMapping.length;
- GenericRowData genericRowData = new GenericRowData(targetLength);
+ GenericRowData genericRowData = new GenericRowData(toFlinkFieldConverters.length);
genericRowData.setRowKind(rowKind);
- for (int i = 0; i < targetLength; i++) {
- int flussIndex = indexMapping[i];
- if (flussIndex == -1) {
- genericRowData.setField(i, null);
- } else {
- Object flussField = flussFieldGetters[flussIndex].getFieldOrNull(flussRow);
- genericRowData.setField(
- i, toFlinkFieldConverters[flussIndex].deserialize(flussField));
- }
+ for (int i = 0; i < toFlinkFieldConverters.length; i++) {
+ Object flussField = flussFieldGetters[i].getFieldOrNull(flussRow);
+ genericRowData.setField(i, toFlinkFieldConverters[i].deserialize(flussField));
}
return genericRowData;
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
index 1f147751a..73ba044d8 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
@@ -31,7 +31,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
-import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.offset;
@@ -60,7 +59,7 @@ class FlinkAsFlussRowTest {
TimestampData.fromEpochMillis(1672531200000L, 3),
new byte[] {1, 2, 3},
null);
- row = new FlinkAsFlussRow(IntStream.range(0, 14).toArray()).replace(flinkRow);
+ row = new FlinkAsFlussRow().replace(flinkRow);
}
@Test
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
index 596780302..9903e7c45 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
@@ -64,6 +64,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
tablePath,
flinkRowType,
createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ // no projection when job compiling, new column added after that.
null);
ListOutputCollector collector = new ListOutputCollector();
@@ -102,6 +103,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
tablePath,
flinkRowType,
createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ // no projection when job compiling, new column added after that.
null);
asyncLookupFunction.open(null);
@@ -154,7 +156,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
tablePath,
flinkRowType,
createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
- null);
+ new int[] {1, 0});
ListOutputCollector collector = new ListOutputCollector();
lookupFunction.setCollector(collector);
@@ -191,7 +193,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
.sorted()
.collect(Collectors.toList());
assertThat(result)
- .containsExactly("+I(0,name0)", "+I(1,name1)", "+I(2,name2)", "+I(3,name3)");
+ .containsExactly("+I(name0,0)", "+I(name1,1)", "+I(name2,2)", "+I(name3,3)");
lookupFunction.close();
// start lookup job after schema change.
@@ -227,8 +229,9 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
.schema(
Schema.newBuilder()
.primaryKey("id")
- .column("name", DataTypes.STRING())
.column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ // added an extra column 'age'
.column("age", DataTypes.INT())
.build())
.distributedBy(DEFAULT_BUCKET_NUM, "id")
@@ -241,7 +244,8 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
try (Table table = conn.getTable(tablePath)) {
UpsertWriter upsertWriter = table.newUpsert().createWriter();
for (int i = 0; i < rows; i++) {
- upsertWriter.upsert(schemaNotMatch ? row("name" + i, i, i) : row(i, "name" + i));
+ upsertWriter.upsert(
+ schemaNotMatch ? row(i, "name" + i, i * 2) : row(i, "name" + i));
}
upsertWriter.flush();
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
index 88e2cb539..6634236aa 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
@@ -178,6 +178,7 @@ class FlinkSourceReaderTest extends FlinkTestBase {
tablePath,
sourceOutputType,
context,
+ null,
new FlinkSourceReaderMetrics(context.metricGroup()),
recordEmitter,
null);
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
index c30c9cf72..44bd1e98d 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
@@ -79,6 +79,24 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
TableDescriptor descriptor1 = TableDescriptor.builder().schema(schema1).build();
createTable(tablePath1, descriptor1);
+ assertThatThrownBy(
+ () ->
+ new FlinkSourceSplitReader(
+ clientConf,
+ tablePath1,
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ "id", DataTypes.BIGINT().copy(false)),
+ DataTypes.FIELD("name", DataTypes.STRING())),
+ new int[] {1, 0},
+ createMockSourceReaderMetrics(),
+ null))
+ .isInstanceOf(ValidationException.class)
+ .hasMessage(
+ "The Flink query schema is not matched to Fluss table schema. \n"
+ + "Flink query schema: ROW<`id` BIGINT NOT NULL, `name` STRING>\n"
+ + "Fluss table schema: ROW<`name` STRING, `id` BIGINT NOT NULL> (projection [1, 0])");
+
assertThatThrownBy(
() ->
new FlinkSourceSplitReader(
@@ -87,10 +105,11 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
DataTypes.ROW(
DataTypes.FIELD("name2", DataTypes.STRING()),
DataTypes.FIELD("id", DataTypes.BIGINT())),
+ null,
createMockSourceReaderMetrics(),
null))
- .isInstanceOf(ValidationException.class)
- .hasMessage("The field name2 is not found in the fluss table.");
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Field name2 does not exist in the row type.");
FlinkSourceSplitReader flinkSourceSplitReader =
new FlinkSourceSplitReader(
@@ -99,9 +118,10 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
DataTypes.ROW(
DataTypes.FIELD("name", DataTypes.STRING()),
DataTypes.FIELD("id", DataTypes.BIGINT().copy(false))),
+ null,
createMockSourceReaderMetrics(),
null);
- assertThat(flinkSourceSplitReader.getProjectedFields()).isEqualTo(new int[] {1, 0});
+ assertThat(flinkSourceSplitReader.getProjectedFields()).isNull();
}
@Test
@@ -384,7 +404,7 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
private FlinkSourceSplitReader createSplitReader(TablePath tablePath, RowType rowType) {
return new FlinkSourceSplitReader(
- clientConf, tablePath, rowType, createMockSourceReaderMetrics(), null);
+ clientConf, tablePath, rowType, null, createMockSourceReaderMetrics(), null);
}
private FlinkSourceReaderMetrics createMockSourceReaderMetrics() {
diff --git a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
index 9c86a92c1..5aad1d5bf 100644
--- a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
+++ b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
@@ -79,8 +79,7 @@ public class HadoopUtils {
}
// Approach 3: Fluss configuration
- // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop
- // conf
+ // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop conf
for (String key : flussConfiguration.keySet()) {
for (String prefix : FLUSS_CONFIG_PREFIXES) {
if (key.startsWith(prefix)) {
@@ -109,7 +108,7 @@ public class HadoopUtils {
}
/**
- * Search Hadoop configuration files in the given path, andadd them to the configuration if
+ * Search Hadoop configuration files in the given path, and add them to the configuration if
* found.
*/
private static boolean addHadoopConfIfFound(
diff --git a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
index 133c23322..4b6d9de71 100644
--- a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
+++ b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
@@ -147,7 +147,7 @@ class IcebergRewriteITCase extends FlinkIcebergTieringTestBase {
// add pos-delete and trigger compaction
rows = Arrays.asList(row(4, "v1"), row(4, "v2"));
flussRows.add(writeIcebergTableRecords(t1, t1Bucket, 6, false, rows).get(1));
- // rewritten files should fail to commit due to conflict,add check here
+ // rewritten files should fail to commit due to conflict, add check here
checkRecords(getIcebergRecords(t1), flussRows);
// 4 data file and 1 delete file
checkFileStatusInIcebergTable(t1, 4, true);
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
index 7f4d6e84e..a09a50c09 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
@@ -69,8 +69,7 @@ final class ServerConnection {
private final ServerNode node;
- // TODO: add max inflight requests limit like Kafka's
- // "max.in.flight.requests.per.connection"
+ // TODO: add max inflight requests limit like Kafka's "max.in.flight.requests.per.connection"
private final Map<Integer, InflightRequest> inflightRequests = MapUtils.newConcurrentHashMap();
private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
private final ConnectionMetrics connectionMetrics;
diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto
index 9feb38dc1..a08eeaba4 100644
--- a/fluss-rpc/src/main/proto/FlussApi.proto
+++ b/fluss-rpc/src/main/proto/FlussApi.proto
@@ -918,33 +918,33 @@ message PbAlterConfig {
required int32 op_type = 3; // SET=0, DELETE=1, APPEND=2, SUBTRACT=3
}
-message PbAddColumn{
+message PbAddColumn {
required string column_name = 1;
required bytes data_type_json = 2;
optional string comment = 3;
required int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
}
-message PbDropColumn{
+message PbDropColumn {
required string column_name = 1;
}
-message PbRenameColumn{
+message PbRenameColumn {
required string old_column_name = 1;
required string new_column_name = 2;
}
-message PbModifyColumn{
+message PbModifyColumn {
required string column_name = 1;
- required bytes data_type_json = 2;
+ optional bytes data_type_json = 2;
optional string comment = 3;
optional int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
}
-message PbDescribeConfig{
+message PbDescribeConfig {
required string config_key = 1;
optional string config_value = 2;
required string config_source = 3;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
index ededf510f..82f3d5ec2 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
@@ -384,7 +384,7 @@ public abstract class RpcServiceBase extends RpcGatewayService implements AdminR
@Override
public CompletableFuture<GetFileSystemSecurityTokenResponse> getFileSystemSecurityToken(
GetFileSystemSecurityTokenRequest request) {
- // TODO:add ACL for per-table in https://github.com/apache/fluss/issues/752
+ // TODO: add ACL for per-table in https://github.com/apache/fluss/issues/752
try {
// In order to avoid repeatedly obtaining security token, cache it for a while.
long currentTimeMs = System.currentTimeMillis();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
index def57231d..b7c92289b 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
@@ -148,7 +148,7 @@ public class CompletedSnapshotStoreManager {
() -> getAllSnapshotSize(bucket));
} else {
LOG.warn(
- "Failed toadd bucketMetricGroup for tableBucket {} when creating completed snapshot.",
+ "Failed to add bucketMetricGroup for tableBucket {} when creating completed snapshot.",
bucket);
}
return snapshotStore;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index fbe0986b1..d1d551121 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -556,7 +556,6 @@ public class CoordinatorEventProcessor implements EventProcessor {
private void processCreateTable(CreateTableEvent createTableEvent) {
long tableId = createTableEvent.getTableInfo().getTableId();
- int schemaId = createTableEvent.getTableInfo().getSchemaId();
// skip the table if it already exists
if (coordinatorContext.containsTableId(tableId)) {
return;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
index 19b13aaa9..a074827d8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
@@ -297,7 +297,7 @@ public class MetadataManager {
// first register a schema to the zk, if then register the table
// to zk fails, there's no harm to register a new schema to zk again
try {
- zookeeperClient.registerSchema(tablePath, tableToCreate.getSchema());
+ zookeeperClient.registerFirstSchema(tablePath, tableToCreate.getSchema());
} catch (Exception e) {
throw new FlussRuntimeException(
"Fail to register schema when creating table " + tablePath, e);
@@ -330,13 +330,9 @@ public class MetadataManager {
// validate the table column changes
if (!schemaChanges.isEmpty()) {
- UpdateSchema schemaUpdate = new SchemaUpdate(table);
- for (TableChange schemaChange : schemaChanges) {
- schemaUpdate = schemaUpdate.applySchemaChange(schemaChange);
- }
- Schema newSchema = schemaUpdate.getSchema();
+ Schema newSchema = SchemaUpdate.applySchemaChanges(table, schemaChanges);
// update the schema
- zookeeperClient.registerSchema(tablePath, newSchema);
+ zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1);
}
} catch (Exception e) {
if (e instanceof TableNotExistException) {
@@ -485,7 +481,7 @@ public class MetadataManager {
if (toEnableDataLake) {
TableInfo newTableInfo = newTableRegistration.toTableInfo(tablePath, schemaInfo);
- // if the table is lake table, we need toadd it to lake table tiering manager
+ // if the table is lake table, we need to add it to lake table tiering manager
lakeTableTieringManager.addNewLakeTable(newTableInfo);
} else if (toDisableDataLake) {
lakeTableTieringManager.removeLakeTable(newTableRegistration.tableId);
@@ -580,14 +576,14 @@ public class MetadataManager {
zookeeperClient.getTables(tablePaths);
// currently, we don't support schema evolution, so all schemas are version 1
Map<TablePath, SchemaInfo> tablePath2SchemaInfos =
- zookeeperClient.getV1Schemas(tablePaths);
+ zookeeperClient.getLatestSchemas(tablePaths);
for (TablePath tablePath : tablePaths) {
if (!tablePath2TableRegistrations.containsKey(tablePath)) {
throw new TableNotExistException("Table '" + tablePath + "' does not exist.");
}
if (!tablePath2SchemaInfos.containsKey(tablePath)) {
throw new SchemaNotExistException(
- "Schema for '" + tablePath + "' with schema_id=1 does not exist.");
+ "Schema for '" + tablePath + "' does not exist.");
}
TableRegistration tableReg = tablePath2TableRegistrations.get(tablePath);
SchemaInfo schemaInfo = tablePath2SchemaInfos.get(tablePath);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
index 1cdc702d7..9efab2a51 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
@@ -29,12 +29,20 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
/** Schema update. */
-public class SchemaUpdate implements UpdateSchema {
+public class SchemaUpdate {
+
+ /** Apply schema changes to the given table info and return the updated schema. */
+ public static Schema applySchemaChanges(TableInfo tableInfo, List<TableChange> changes) {
+ SchemaUpdate schemaUpdate = new SchemaUpdate(tableInfo);
+ for (TableChange change : changes) {
+ schemaUpdate = schemaUpdate.applySchemaChange(change);
+ }
+ return schemaUpdate.getSchema();
+ }
+
private final List<Schema.Column> columns;
private final AtomicInteger highestFieldId;
private final List<String> primaryKeys;
- private final List<String> bucketKeys;
- private final List<String> partitionKeys;
private final Map<String, Schema.Column> existedColumns;
public SchemaUpdate(TableInfo tableInfo) {
@@ -42,15 +50,12 @@ public class SchemaUpdate implements UpdateSchema {
this.existedColumns = new HashMap<>();
this.highestFieldId = new AtomicInteger(tableInfo.getSchema().getHighestFieldId());
this.primaryKeys = tableInfo.getPrimaryKeys();
- this.bucketKeys = tableInfo.getBucketKeys();
- this.partitionKeys = tableInfo.getPartitionKeys();
this.columns.addAll(tableInfo.getSchema().getColumns());
for (Schema.Column column : columns) {
existedColumns.put(column.getName(), column);
}
}
- @Override
public Schema getSchema() {
Schema.Builder builder =
Schema.newBuilder()
@@ -63,8 +68,7 @@ public class SchemaUpdate implements UpdateSchema {
return builder.build();
}
- @Override
- public UpdateSchema applySchemaChange(TableChange columnChange) {
+ public SchemaUpdate applySchemaChange(TableChange columnChange) {
if (columnChange instanceof TableChange.AddColumn) {
return addColumn((TableChange.AddColumn) columnChange);
} else if (columnChange instanceof TableChange.ModifyColumn) {
@@ -78,7 +82,7 @@ public class SchemaUpdate implements UpdateSchema {
"Unknown column change type " + columnChange.getClass().getName());
}
- private UpdateSchema addColumn(TableChange.AddColumn addColumn) {
+ private SchemaUpdate addColumn(TableChange.AddColumn addColumn) {
if (existedColumns.containsKey(addColumn.getName())) {
throw new IllegalArgumentException(
"Column " + addColumn.getName() + " already exists.");
@@ -105,15 +109,15 @@ public class SchemaUpdate implements UpdateSchema {
return this;
}
- private UpdateSchema dropColumn(TableChange.DropColumn dropColumn) {
+ private SchemaUpdate dropColumn(TableChange.DropColumn dropColumn) {
throw new SchemaChangeException("Not support drop column now.");
}
- private UpdateSchema modifiedColumn(TableChange.ModifyColumn modifyColumn) {
+ private SchemaUpdate modifiedColumn(TableChange.ModifyColumn modifyColumn) {
throw new SchemaChangeException("Not support modify column now.");
}
- private UpdateSchema renameColumn(TableChange.RenameColumn renameColumn) {
+ private SchemaUpdate renameColumn(TableChange.RenameColumn renameColumn) {
throw new SchemaChangeException("Not support rename column now.");
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
deleted file mode 100644
index 1d51704ac..000000000
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.fluss.server.coordinator;
-
-import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.metadata.TableChange;
-
-/** Schema update. */
-public interface UpdateSchema {
- Schema getSchema();
-
- UpdateSchema applySchemaChange(TableChange columnChange);
-}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
index acfb7e689..09c273e7d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
@@ -246,16 +246,7 @@ public class TableChangeWatcher {
private void processSchemaChange(TablePath tablePath, int schemaId) {
try {
- int currentSchemaId = zooKeeperClient.getCurrentSchemaId(tablePath);
SchemaInfo schemaInfo;
- if (schemaId != currentSchemaId) {
- LOG.warn(
- "Schema id {} is not equal to current schema id {}. Skipping schema change processing.",
- schemaId,
- currentSchemaId);
- return;
- }
-
Optional<SchemaInfo> optSchema = zooKeeperClient.getSchemaById(tablePath, schemaId);
if (!optSchema.isPresent()) {
LOG.error("No schema for table {} in zookeeper.", tablePath);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
index fad96a596..460562bc1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
@@ -55,7 +55,7 @@ public class BatchRegisterLeadAndIsr {
tableBucket, leaderAndIsr, partitionName, liveReplicas));
} else {
throw new IllegalArgumentException(
- "Try toadd a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "Try to add a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "batch tableId="
+ tableId
+ " batch partitionId:"
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
index 1f1c4192d..96bb0eba8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
@@ -66,15 +66,6 @@ public class KvRecoverHelper {
private InternalRow.FieldGetter[] currentFieldGetters;
- public KvRecoverHelper(
- KvTablet kvTablet,
- LogTablet logTablet,
- long recoverPointOffset,
- KvRecoverContext recoverContext,
- KvFormat kvFormat) {
- throw new UnsupportedOperationException();
- }
-
public KvRecoverHelper(
KvTablet kvTablet,
LogTablet logTablet,
@@ -136,64 +127,52 @@ public class KvRecoverHelper {
FetchIsolation fetchIsolation,
ThrowingConsumer<KeyValueAndLogOffset, Exception> resumeRecordConsumer)
throws Exception {
- long nextFetchOffset = startFetchOffset;
- while (true) {
- LogRecords logRecords =
- logTablet
- .read(
- nextFetchOffset,
- recoverContext.maxFetchLogSizeInRecoverKv,
- fetchIsolation,
- true,
- null)
- .getRecords();
- if (logRecords == MemoryLogRecords.EMPTY) {
- break;
- }
-
- for (LogRecordBatch logRecordBatch : logRecords.batches()) {
- // short schemaId = logRecordBatch.schemaId();
- // if (currentSchemaId == null) {
- // initSchema(schemaId);
- // } else if (currentSchemaId != schemaId) {
- // throw new KvStorageException(
- // String.format(
- // "Can't recover kv tablet for table bucket from
- // log %s since the schema changes from schema id %d to schema id %d. "
- // + "Currently, schema change is not
- // supported.",
- // recoverContext.tableBucket, currentSchemaId,
- // schemaId));
- // }
+ try (LogRecordReadContext readContext =
+ LogRecordReadContext.createArrowReadContext(
+ currentRowType, currentSchemaId, schemaGetter)) {
+ long nextFetchOffset = startFetchOffset;
+ while (true) {
+ LogRecords logRecords =
+ logTablet
+ .read(
+ nextFetchOffset,
+ recoverContext.maxFetchLogSizeInRecoverKv,
+ fetchIsolation,
+ true,
+ null)
+ .getRecords();
+ if (logRecords == MemoryLogRecords.EMPTY) {
+ break;
+ }
- // todo: currentRowType和currentSchemaId无法对齐
- try (LogRecordReadContext readContext =
- LogRecordReadContext.createArrowReadContext(
- currentRowType, currentSchemaId, schemaGetter);
- CloseableIterator<LogRecord> logRecordIter =
- logRecordBatch.records(readContext)) {
- while (logRecordIter.hasNext()) {
- LogRecord logRecord = logRecordIter.next();
- if (logRecord.getChangeType() != ChangeType.UPDATE_BEFORE) {
- InternalRow logRow = logRecord.getRow();
- byte[] key = keyEncoder.encodeKey(logRow);
- byte[] value = null;
- if (logRecord.getChangeType() != ChangeType.DELETE) {
- // the log row format may not compatible with kv row format,
- // e.g, arrow vs. compacted, thus needs a conversion here.
- BinaryRow row = toKvRow(logRecord.getRow());
- // todo: short value是否会有问题，感觉可以先check一下
- value = ValueEncoder.encodeValue(currentSchemaId.shortValue(), row);
+ for (LogRecordBatch logRecordBatch : logRecords.batches()) {
+ try (CloseableIterator<LogRecord> logRecordIter =
+ logRecordBatch.records(readContext)) {
+ while (logRecordIter.hasNext()) {
+ LogRecord logRecord = logRecordIter.next();
+ if (logRecord.getChangeType() != ChangeType.UPDATE_BEFORE) {
+ InternalRow logRow = logRecord.getRow();
+ byte[] key = keyEncoder.encodeKey(logRow);
+ byte[] value = null;
+ if (logRecord.getChangeType() != ChangeType.DELETE) {
+ // the log row format may not compatible with kv row format,
+ // e.g, arrow vs. compacted, thus needs a conversion here.
+ BinaryRow row = toKvRow(logRecord.getRow());
+ value =
+ ValueEncoder.encodeValue(
+ currentSchemaId.shortValue(), row);
+ }
+ resumeRecordConsumer.accept(
+ new KeyValueAndLogOffset(
+ key, value, logRecord.logOffset()));
}
- resumeRecordConsumer.accept(
- new KeyValueAndLogOffset(key, value, logRecord.logOffset()));
}
}
+ nextFetchOffset = logRecordBatch.nextLogOffset();
}
- nextFetchOffset = logRecordBatch.nextLogOffset();
}
+ return nextFetchOffset;
}
- return nextFetchOffset;
}
// TODO: this is very in-efficient, because the conversion is CPU heavy. Should be optimized in
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
index 176e749f5..b6957c5dc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
@@ -34,16 +34,16 @@ import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.KvRecord;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.record.KvRecordReadContext;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.PaddingRow;
import org.apache.fluss.row.arrow.ArrowWriterPool;
import org.apache.fluss.row.arrow.ArrowWriterProvider;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.row.encode.ValueEncoder;
import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer;
import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.TruncateReason;
import org.apache.fluss.server.kv.rocksdb.RocksDBKv;
@@ -83,7 +83,6 @@ import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import static org.apache.fluss.utils.SchemaUtil.getTargetColumns;
import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock;
import static org.apache.fluss.utils.concurrent.LockUtils.inWriteLock;
@@ -255,6 +254,20 @@ public final class KvTablet {
/**
* Put the KvRecordBatch into the kv storage, and return the appended wal log info.
*
+ * <p>Schema Evolution Handling:
+ *
+ * <p>We don't allow shema of input kv records to be larger than the latest schema id known by
+ * the tablet. Besides, we currently only support ADD COLUMN LAST operation, so the input row or
+ * old row must have same or fewer columns than latest schema. This helps to simplify the schema
+ * change handling.
+ *
+ * <p>1. We write the kv records into KvStore without converting it into latest schema for
+ * performance consideration. We have mechanisms that writer client dynamically use latest
+ * schema for writing records.
+ *
+ * <p>2. We always use the latest schema for writing WAL logs, because it anyway happens
+ * deserialization&serialization to convert the compacted format into Arrow format.
+ *
* @param kvRecords the kv records to put into
* @param targetColumns the target columns to put, null if put all columns
*/
@@ -267,32 +280,28 @@ public final class KvTablet {
SchemaInfo schemaInfo = schemaGetter.getLatestSchemaInfo();
Schema latestSchema = schemaInfo.getSchema();
- // In the following scenario, we need to support schema migration:
- // schema1: old data's schema -> valueDecoder
- // schema2: new data's schema
- // schema3: current schema (also the latest schema)
- // schema1 <= schema2 <= current schema id
short latestSchemaId = (short) schemaInfo.getSchemaId();
short schemaIdOfNewData = kvRecords.schemaId();
if (schemaIdOfNewData > latestSchemaId || schemaIdOfNewData < 0) {
+ // TODO: we may need to support retriable exception here
throw new SchemaNotExistException(
"Invalid schema id: "
+ schemaIdOfNewData
+ ", latest schema id: "
+ latestSchemaId);
}
- Schema schemaOfNewData = schemaGetter.getSchema(schemaIdOfNewData);
+
+ // we only support ADD COLUMN, so targetColumns is fine to be used directly
RowMerger currentMerger =
rowMerger.configureTargetColumns(
- schemaIdOfNewData == latestSchemaId
- ? targetColumns
- : getTargetColumns(
- targetColumns, schemaOfNewData, latestSchema),
- latestSchemaId,
- latestSchema);
- RowType currentRowType = latestSchema.getRowType();
- WalBuilder walBuilder = createWalBuilder(latestSchemaId, currentRowType);
+ targetColumns, latestSchemaId, latestSchema);
+ RowType latestRowType = latestSchema.getRowType();
+ WalBuilder walBuilder = createWalBuilder(latestSchemaId, latestRowType);
walBuilder.setWriterState(kvRecords.writerId(), kvRecords.batchSequence());
+ // we only support ADD COLUMN LAST, so the BinaryRow after RowMerger is
+ // only has fewer ending columns than latest schema, so we pad nulls to
+ // the end of the BinaryRow to get the latest schema row.
+ PaddingRow latestSchemaRow = new PaddingRow(latestRowType.getFieldCount());
// get offset to track the offset corresponded to the kv record
long logEndOffsetOfPrevBatch = logTablet.localLogEndOffset();
try {
@@ -301,22 +310,15 @@ public final class KvTablet {
// TODO: reuse the read context and decoder
KvRecordBatch.ReadContext readContext =
KvRecordReadContext.createReadContext(kvFormat, schemaGetter);
-
- // replace new row with current schema if the schema id is older than now
- ProjectedRow projectedRow =
- ProjectedRow.from(schemaOfNewData, latestSchema);
ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat);
for (KvRecord kvRecord : kvRecords.records(readContext)) {
-
byte[] keyBytes = BytesUtils.toArray(kvRecord.getKey());
KvPreWriteBuffer.Key key = KvPreWriteBuffer.Key.of(keyBytes);
- InternalRow row = kvRecord.getRow();
- if (row != null && schemaIdOfNewData < latestSchemaId) {
- row = projectedRow.replaceRow(row);
- }
-
- if (row == null) {
- DeleteBehavior deleteBehavior = rowMerger.deleteBehavior();
+ BinaryRow row = kvRecord.getRow();
+ BinaryValue currentValue =
+ row == null ? null : new BinaryValue(schemaIdOfNewData, row);
+ if (currentValue == null) {
+ DeleteBehavior deleteBehavior = currentMerger.deleteBehavior();
if (deleteBehavior == DeleteBehavior.IGNORE) {
// skip delete rows if the merger doesn't support yet
continue;
@@ -326,99 +328,69 @@ public final class KvTablet {
+ "The table.delete.behavior is set to 'disable'.");
}
// it's for deletion
- byte[] oldValue = getFromBufferOrKv(key);
+ byte[] oldValueBytes = getFromBufferOrKv(key);
- if (oldValue == null) {
+ if (oldValueBytes == null) {
// there might be large amount of such deletion, so we don't log
LOG.debug(
"The specific key can't be found in kv tablet although the kv record is for deletion, "
+ "ignore it directly as it doesn't exist in the kv tablet yet.");
} else {
- ValueDecoder.Value oldRowAndSchemaId =
- valueDecoder.decodeValue(oldValue);
- InternalRow oldRow = oldRowAndSchemaId.row;
- if (oldRow != null
- && oldRowAndSchemaId.schemaId != latestSchemaId) {
- // todo: 后续想办法复用对应的projected row.
- Schema schemaOfOldData =
- schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
- oldRow =
- ProjectedRow.from(schemaOfOldData, latestSchema)
- .replaceRow(oldRow);
- }
- InternalRow newRow = currentMerger.delete(oldRow);
+ BinaryValue oldValue = valueDecoder.decodeValue(oldValueBytes);
+ BinaryValue newValue = currentMerger.delete(oldValue);
// if newRow is null, it means the row should be deleted
- if (newRow == null) {
- walBuilder.append(ChangeType.DELETE, oldRow);
+ if (newValue == null) {
+ walBuilder.append(
+ ChangeType.DELETE,
+ latestSchemaRow.replaceRow(oldValue.row));
kvPreWriteBuffer.delete(key, logOffset++);
} else {
// otherwise, it's a partial update, should produce -U,+U
- walBuilder.append(ChangeType.UPDATE_BEFORE, oldRow);
- walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
+ walBuilder.append(
+ ChangeType.UPDATE_BEFORE,
+ latestSchemaRow.replaceRow(oldValue.row));
+ walBuilder.append(
+ ChangeType.UPDATE_AFTER,
+ latestSchemaRow.replaceRow(newValue.row));
kvPreWriteBuffer.put(
- key,
- ValueEncoder.encodeRow(
- latestSchemaId,
- kvFormat,
- currentRowType,
- newRow),
- logOffset + 1);
+ key, newValue.encodeValue(), logOffset + 1);
logOffset += 2;
}
}
} else {
// upsert operation
- byte[] oldValue = getFromBufferOrKv(key);
+ byte[] oldValueBytes = getFromBufferOrKv(key);
// it's update
- if (oldValue != null) {
- ValueDecoder.Value oldRowAndSchemaId =
- valueDecoder.decodeValue(oldValue);
- InternalRow oldRow = oldRowAndSchemaId.row;
- if (oldRow != null
- && oldRowAndSchemaId.schemaId != latestSchemaId) {
- // todo: 后续想办法复用对应的projected row.
- Schema schemaOfOldData =
- schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
- oldRow =
- ProjectedRow.from(schemaOfOldData, latestSchema)
- .replaceRow(oldRow);
- }
-
- InternalRow newRow = currentMerger.merge(oldRow, row);
- if (newRow == oldRow) {
- // newRow is the same to oldRow, means nothing
+ if (oldValueBytes != null) {
+ BinaryValue oldValue = valueDecoder.decodeValue(oldValueBytes);
+ BinaryValue newValue =
+ currentMerger.merge(oldValue, currentValue);
+ if (newValue == oldValue) {
+ // newValue is the same to oldValue, means nothing
// happens (no update/delete), and input should be ignored
continue;
}
- walBuilder.append(ChangeType.UPDATE_BEFORE, oldRow);
- walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
+ walBuilder.append(
+ ChangeType.UPDATE_BEFORE,
+ latestSchemaRow.replaceRow(oldValue.row));
+ walBuilder.append(
+ ChangeType.UPDATE_AFTER,
+ latestSchemaRow.replaceRow(newValue.row));
// logOffset is for -U, logOffset + 1 is for +U, we need to use
// the log offset for +U
kvPreWriteBuffer.put(
- key,
- ValueEncoder.encodeRow(
- latestSchemaId,
- kvFormat,
- currentRowType,
- newRow),
- logOffset + 1);
+ key, newValue.encodeValue(), logOffset + 1);
logOffset += 2;
} else {
// it's insert
- // TODO: we shouldadd guarantees that all non-specified
- // columns
+ // TODO: we should add guarantees that all non-specified columns
// of the input row are set to null.
- InternalRow newRow = row;
- walBuilder.append(ChangeType.INSERT, newRow);
+ walBuilder.append(
+ ChangeType.INSERT,
+ latestSchemaRow.replaceRow(currentValue.row));
kvPreWriteBuffer.put(
- key,
- ValueEncoder.encodeRow(
- latestSchemaId,
- kvFormat,
- currentRowType,
- newRow),
- logOffset++);
+ key, currentValue.encodeValue(), logOffset++);
}
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java
index 688ac8dab..d8741e010 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.partialupdate;
import org.apache.fluss.exception.InvalidTargetColumnException;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.encode.RowEncoder;
import org.apache.fluss.types.DataType;
@@ -34,6 +34,7 @@ import java.util.BitSet;
@NotThreadSafe
public class PartialUpdater {
+ private final short targetSchemaId;
private final InternalRow.FieldGetter[] flussFieldGetters;
private final RowEncoder rowEncoder;
@@ -42,7 +43,8 @@ public class PartialUpdater {
private final BitSet primaryKeyCols = new BitSet();
private final DataType[] fieldDataTypes;
- public PartialUpdater(KvFormat kvFormat, Schema schema, int[] targetColumns) {
+ public PartialUpdater(KvFormat kvFormat, short schemaId, Schema schema, int[] targetColumns) {
+ this.targetSchemaId = schemaId;
for (int targetColumn : targetColumns) {
partialUpdateCols.set(targetColumn);
}
@@ -89,43 +91,44 @@ public class PartialUpdater {
}
/**
- * Partial update the {@code oldRow} with the given new row {@code partialRow}. The {@code
- * oldRow} may be null, in this case, the field don't exist in the {@code partialRow} will be
+ * Partial update the {@code oldValue} with the given new row {@code partialValue}. The {@code
+ * oldValue} may be null, in this case, the field don't exist in the {@code partialRow} will be
* set to null.
*
- * @param oldRow the old row to be updated
- * @param partialRow the new row to be updated.
- * @return the updated row
+ * @param oldValue the old value to be updated
+ * @param partialValue the new value to be updated.
+ * @return the updated value (schema id + row bytes)
*/
- public BinaryRow updateRow(@Nullable InternalRow oldRow, InternalRow partialRow) {
+ public BinaryValue updateRow(@Nullable BinaryValue oldValue, BinaryValue partialValue) {
rowEncoder.startNewRow();
// write each field
for (int i = 0; i < fieldDataTypes.length; i++) {
// use the partial row value
if (partialUpdateCols.get(i)) {
- rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(partialRow));
+ rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(partialValue.row));
} else {
// use the old row value
- if (oldRow == null) {
+ if (oldValue == null) {
rowEncoder.encodeField(i, null);
} else {
- rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(oldRow));
+ rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(oldValue.row));
}
}
}
- return rowEncoder.finishRow();
+ return new BinaryValue(targetSchemaId, rowEncoder.finishRow());
}
/**
- * Partial delete the given {@code row}. If all the fields except for {@link #partialUpdateCols}
- * in {@code row} are null, return null. Otherwise, update all the {@link #partialUpdateCols} in
- * the {@code row} except for the primary key columns to null values, return the updated row.
+ * Partial delete the given {@code value}. If all the fields except for {@link
+ * #partialUpdateCols} in {@code value.row} are null, return null. Otherwise, update all the
+ * {@link #partialUpdateCols} in the {@code value.row} except for the primary key columns to
+ * null values, return the updated value.
*
- * @param row the row to be deleted
- * @return the row after partial deleted
+ * @param value the value to be deleted
+ * @return the value after partial deleted
*/
- public @Nullable BinaryRow deleteRow(InternalRow row) {
- if (isFieldsNull(row, partialUpdateCols)) {
+ public @Nullable BinaryValue deleteRow(BinaryValue value) {
+ if (isFieldsNull(value.row, partialUpdateCols)) {
return null;
} else {
rowEncoder.startNewRow();
@@ -137,10 +140,10 @@ public class PartialUpdater {
rowEncoder.encodeField(i, null);
} else {
// use the old row value
- rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(row));
+ rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(value.row));
}
}
- return rowEncoder.finishRow();
+ return new BinaryValue(targetSchemaId, rowEncoder.finishRow());
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
index 3ce22854c..0c227aaf2 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
@@ -25,6 +25,7 @@ import com.github.benmanes.caffeine.cache.Caffeine;
import javax.annotation.concurrent.ThreadSafe;
+import java.time.Duration;
import java.util.Arrays;
/** The cache for {@link PartialUpdater}. */
@@ -37,7 +38,11 @@ public class PartialUpdaterCache {
// currently, the cache is used per-bucket, so we limit the cache size to 5 to have a
// maximal 5 parallel partial updaters. This is a temporary solution and should be
// shared across all buckets in the future.
- this.rowPartialUpdaters = Caffeine.newBuilder().maximumSize(5).build();
+ this.rowPartialUpdaters =
+ Caffeine.newBuilder()
+ .maximumSize(5)
+ .expireAfterAccess(Duration.ofMinutes(5))
+ .build();
}
// TODO: extend to tableId and schemaId when the cache is shared across all tables
@@ -45,7 +50,7 @@ public class PartialUpdaterCache {
KvFormat kvFormat, short schemaId, Schema schema, int[] targetColumns) {
return rowPartialUpdaters.get(
getPartialUpdaterKey(targetColumns, schemaId),
- k -> new PartialUpdater(kvFormat, schema, targetColumns));
+ k -> new PartialUpdater(kvFormat, schemaId, schema, targetColumns));
}
private String getPartialUpdaterKey(int[] targetColumns, int schemaId) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
index af3500c59..2db73c819 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.server.kv.partialupdate.PartialUpdater;
import org.apache.fluss.server.kv.partialupdate.PartialUpdaterCache;
@@ -46,14 +46,14 @@ public class DefaultRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
// always retain the new row (latest row)
- return newRow;
+ return newValue;
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
// returns null to indicate the row is deleted
return null;
}
@@ -98,13 +98,13 @@ public class DefaultRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
- return partialUpdater.updateRow(oldRow, newRow);
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
+ return partialUpdater.updateRow(oldValue, newValue);
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
return partialUpdater.deleteRow(oldRow);
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
index 79592e177..33bd6ee1f 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.record.BinaryValue;
import javax.annotation.Nullable;
@@ -44,14 +44,14 @@ public class FirstRowRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
// always retain the old row (first row)
- return oldRow;
+ return oldValue;
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the first_row merge engine.");
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
index 5f3414574..89bff8c0a 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
@@ -23,7 +23,7 @@ import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.record.BinaryValue;
import javax.annotation.Nullable;
@@ -33,14 +33,14 @@ import java.util.Optional;
public interface RowMerger {
/**
- * Merge the old row with the new row.
+ * Merge the old value with the new value.
*
- * @param oldRow the old row
- * @param newRow the new row
- * @return the merged row, if the returned row is the same to the old row, then nothing happens
- * to the row (no update, no delete).
+ * @param oldValue the old value
+ * @param newValue the new row
+ * @return the merged value, if the returned row is the same to the old row, then nothing
+ * happens to the row (no update, no delete).
*/
- InternalRow merge(InternalRow oldRow, InternalRow newRow);
+ BinaryValue merge(BinaryValue oldValue, BinaryValue newValue);
/**
* Merge the old row with a delete row.
@@ -52,7 +52,7 @@ public interface RowMerger {
* @return the merged row, or null if the row is deleted.
*/
@Nullable
- InternalRow delete(InternalRow oldRow);
+ BinaryValue delete(BinaryValue oldRow);
/**
* The behavior of delete operations on primary key tables.
@@ -61,7 +61,13 @@ public interface RowMerger {
*/
DeleteBehavior deleteBehavior();
- /** Dynamically configure the target columns to merge and return the effective merger. */
+ /**
+ * Dynamically configure the target columns to merge and return the effective merger.
+ *
+ * @param targetColumns the partial update target column positions, null means full update
+ * @param schemaId the schema id used to generate new rows
+ * @param schema the schema used to generate new rows
+ */
RowMerger configureTargetColumns(@Nullable int[] targetColumns, short schemaId, Schema schema);
/** Create a row merger based on the given configuration. */
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
index 9706230b5..4b555602e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
@@ -20,6 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
@@ -64,14 +65,14 @@ public class VersionedRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
// return newRow if newRow's version is larger or equal than oldRow's version
- return versionComparator.compare(oldRow, newRow) <= 0 ? newRow : oldRow;
+ return versionComparator.compare(oldValue.row, newValue.row) <= 0 ? newValue : oldValue;
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the versioned merge engine.");
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
index b8d941221..3e93cc46a 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
@@ -99,7 +99,7 @@ public class CompletedSnapshotStore {
* Synchronously writes the new snapshots to snapshot handle store and asynchronously removes
* older ones.
*
- * @param snapshot Completed snapshot toadd.
+ * @param snapshot Completed snapshot to add.
*/
@VisibleForTesting
CompletedSnapshot addSnapshotAndSubsumeOldestOne(
@@ -116,7 +116,7 @@ public class CompletedSnapshotStore {
completedSnapshotHandleStore.add(
snapshot.getTableBucket(), snapshot.getSnapshotID(), completedSnapshotHandle);
- // Nowadd the new one. If it fails, we don't want to lose existing data.
+ // Now add the new one. If it fails, we don't want to lose existing data.
completedSnapshots.addLast(snapshot);
// Remove completed snapshot from queue and snapshotStateHandleStore, not discard.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
index e09c93b22..fe517f268 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
@@ -328,7 +328,7 @@ public class SnapshotLocation {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: mayadd entropy injection?
+ // todo: may add entropy injection?
this.kvFilePath = createFilePath();
this.outStream = fs.create(kvFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
index 847bfa883..0ec42a090 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
@@ -21,6 +21,7 @@ import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.compression.ArrowCompressionInfo;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.FileLogProjection;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.rpc.messages.FetchLogRequest;
import javax.annotation.Nullable;
@@ -66,7 +67,7 @@ public final class FetchParams {
private final int minFetchBytes;
private final long maxWaitMs;
- // TODO:add more params like epoch etc.
+ // TODO: add more params like epoch etc.
public FetchParams(int replicaId, int maxFetchBytes) {
this(replicaId, true, maxFetchBytes, DEFAULT_MIN_FETCH_BYTES, DEFAULT_MAX_WAIT_MS);
@@ -99,13 +100,14 @@ public final class FetchParams {
int maxFetchBytes,
SchemaGetter schemaGetter,
ArrowCompressionInfo compressionInfo,
- @Nullable int[] projectedFields) {
+ @Nullable int[] projectedFields,
+ ProjectionPushdownCache projectionCache) {
this.fetchOffset = fetchOffset;
this.maxFetchBytes = maxFetchBytes;
if (projectedFields != null) {
projectionEnabled = true;
if (fileLogProjection == null) {
- fileLogProjection = new FileLogProjection();
+ fileLogProjection = new FileLogProjection(projectionCache);
}
fileLogProjection.setCurrentProjection(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
index 086e23b0e..130b3855d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
@@ -405,8 +405,6 @@ public final class LocalLog {
(maxOffsetMetadata.getSegmentBaseOffset() == segment.getBaseOffset())
? maxOffsetMetadata.getRelativePositionInSegment()
: segment.getSizeInBytes();
-
- // segment需要查询
fetchDataInfo =
segment.read(readOffset, maxLength, maxPosition, minOneMessage, projection);
if (fetchDataInfo == null) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
index 05233e0ff..835a94ff6 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
@@ -203,7 +203,7 @@ public class FsRemoteLogOutputStream extends FSDataOutputStream {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: mayadd entropy injection?
+ // todo: may add entropy injection?
this.outStream = fs.create(remoteLogFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
} catch (Exception e) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
index 86a6bbf9e..73aee521e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
@@ -40,7 +40,7 @@ public class LogSegmentFiles {
private final Path offsetIndex;
private final Path timeIndex;
private final @Nullable Path writerIdIndex;
- // TODOadd leader epoch index after introduce leader epoch.
+ // TODO add leader epoch index after introduce leader epoch.
public LogSegmentFiles(
Path logSegment, Path offsetIndex, Path timeIndex, @Nullable Path writerIdIndex) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
index 0b425b8fc..cdde3842d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
@@ -227,7 +227,7 @@ public class LogTieringTask implements Runnable {
}
/**
- * Copy the given log segments to remote andadd the successfully copied segment to the {@code
+ * Copy the given log segments to remote and add the successfully copied segment to the {@code
* copiedSegments} parameter.
*
* @return the end offset of the last segment copied to remote.
@@ -347,7 +347,7 @@ public class LogTieringTask implements Runnable {
// the commit failed, it means the commit snapshot is invalid or register zk
// failed, we will revert this commit and delete the remote log manifest
// file.
- // TODO:add the fail reason in the future.
+ // TODO: add the fail reason in the future.
LOG.error(
"Commit remote log manifest failed for table bucket {}. We will delete the"
+ " written remote log manifest file",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
similarity index 68%
rename from fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
rename to fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
index f56429bf1..374e20501 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
@@ -29,8 +29,11 @@ import org.apache.fluss.shaded.guava32.com.google.common.cache.CacheBuilder;
import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.UncheckedExecutionException;
import org.apache.fluss.utils.MapUtils;
+import javax.annotation.concurrent.ThreadSafe;
+
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
/**
@@ -40,74 +43,77 @@ import java.util.concurrent.ExecutionException;
* 1. latest schema of each subscribed table, updated by UpdateMetadata request. 2. history schemas
* of each table, updated by lookup from tablet server.
*/
-public class SchemaMetadataManager {
+@ThreadSafe
+public class ServerSchemaCache {
- private MetadataManager metadataManager;
- private final Map<TablePath, SchemaInfo> latestSchemaByTablePath;
- private final Map<TablePath, Integer> subscriberCounters;
+ private final MetadataManager metadataManager;
+ private final Map<Long, SchemaInfo> latestSchemaByTableId;
+ // table_id -> subscriber count
+ private final Map<Long, Integer> subscriberCounters;
private final Cache<TableSchemaKey, Schema> schemaCache;
- public SchemaMetadataManager(MetadataManager metadataManager) {
+ public ServerSchemaCache(MetadataManager metadataManager) {
this.metadataManager = metadataManager;
// thread safe is guaranteed by subscriberCounters.
this.subscriberCounters = MapUtils.newConcurrentHashMap();
- this.latestSchemaByTablePath = MapUtils.newConcurrentHashMap();
+ this.latestSchemaByTableId = MapUtils.newConcurrentHashMap();
this.schemaCache = CacheBuilder.newBuilder().maximumSize(100).build();
}
public SchemaGetter subscribeWithInitialSchema(
- TablePath tablePath, int initialSchemaId, Schema initialSchema) {
+ long tableId, TablePath tablePath, int initialSchemaId, Schema initialSchema) {
subscriberCounters.compute(
- tablePath,
+ tableId,
(key, oldValue) -> {
- updateSchema(tablePath, initialSchemaId, initialSchema);
+ updateSchema(tableId, initialSchemaId, initialSchema);
if (oldValue == null) {
return 1;
} else {
return oldValue + 1;
}
});
- return new SchemaMetadataSubscriberImpl(tablePath);
+ return new SchemaMetadataSubscriberImpl(tableId, tablePath);
}
- public void updateLatestSchema(TablePath tablePath, short schemaId, Schema schema) {
+ public void updateLatestSchema(long tableId, short schemaId, Schema schema) {
// only update if tablePath is subscribed.
subscriberCounters.computeIfPresent(
- tablePath,
+ tableId,
(key, oldValue) -> {
- updateSchema(tablePath, schemaId, schema);
+ updateSchema(tableId, schemaId, schema);
return oldValue;
});
}
- private void unsubscribe(TablePath tablePath) {
+ private void unsubscribe(long tableId) {
subscriberCounters.compute(
- tablePath,
+ tableId,
(key, oldValue) -> {
if (oldValue != null && oldValue > 1) {
return oldValue - 1;
}
- latestSchemaByTablePath.remove(tablePath);
+ latestSchemaByTableId.remove(tableId);
return null;
});
}
- private Schema getFlussSchema(TablePath tablePath, int schemaId) throws ExecutionException {
+ private Schema getFlussSchema(long tableId, TablePath tablePath, int schemaId)
+ throws ExecutionException {
return schemaCache.get(
- new TableSchemaKey(tablePath, schemaId),
+ new TableSchemaKey(tableId, schemaId),
() -> {
- SchemaInfo schemaInfo = latestSchemaByTablePath.get(tablePath);
+ SchemaInfo schemaInfo = latestSchemaByTableId.get(tableId);
if (schemaInfo != null && schemaInfo.getSchemaId() == schemaId) {
- return latestSchemaByTablePath.get(tablePath).getSchema();
+ return latestSchemaByTableId.get(tableId).getSchema();
} else {
return metadataManager.getSchemaById(tablePath, schemaId).getSchema();
}
});
}
- private void updateSchema(TablePath tablePath, int schemaId, Schema schema) {
- latestSchemaByTablePath.compute(
- tablePath,
+ private void updateSchema(long tableId, int schemaId, Schema schema) {
+ latestSchemaByTableId.compute(
+ tableId,
(key, oldValue) -> {
if (oldValue == null || oldValue.getSchemaId() < schemaId) {
return new SchemaInfo(schema, schemaId);
@@ -116,25 +122,25 @@ public class SchemaMetadataManager {
}
});
- schemaCache.put(new TableSchemaKey(tablePath, schemaId), schema);
+ schemaCache.put(new TableSchemaKey(tableId, schemaId), schema);
}
@VisibleForTesting
- public Map<TablePath, SchemaInfo> getLatestSchemaByTablePath() {
- return latestSchemaByTablePath;
+ public Map<Long, SchemaInfo> getLatestSchemaByTableId() {
+ return latestSchemaByTableId;
}
@VisibleForTesting
- Map<TablePath, Integer> getSubscriberCounters() {
+ Map<Long, Integer> getSubscriberCounters() {
return subscriberCounters;
}
private static class TableSchemaKey {
- private final TablePath tablePath;
+ private final long tableId;
private final int schemaId;
- TableSchemaKey(TablePath tablePath, int schemaId) {
- this.tablePath = tablePath;
+ TableSchemaKey(long tableId, int schemaId) {
+ this.tableId = tableId;
this.schemaId = schemaId;
}
@@ -144,34 +150,35 @@ public class SchemaMetadataManager {
return false;
}
TableSchemaKey that = (TableSchemaKey) o;
- return schemaId == that.schemaId && Objects.equals(tablePath, that.tablePath);
+ return schemaId == that.schemaId && Objects.equals(tableId, that.tableId);
}
@Override
public int hashCode() {
- return Objects.hash(tablePath, schemaId);
+ return Objects.hash(tableId, schemaId);
}
}
private class SchemaMetadataSubscriberImpl implements SchemaGetter {
+ private final long tableId;
private final TablePath tablePath;
private volatile boolean released;
- public SchemaMetadataSubscriberImpl(TablePath tablePath) {
+ public SchemaMetadataSubscriberImpl(long tableId, TablePath tablePath) {
+ this.tableId = tableId;
this.tablePath = tablePath;
this.released = false;
}
@Override
public SchemaInfo getLatestSchemaInfo() {
- return latestSchemaByTablePath.get(tablePath);
+ return latestSchemaByTableId.get(tableId);
}
@Override
public Schema getSchema(int schemaId) {
- // todo: support with get table.
try {
- return getFlussSchema(tablePath, schemaId);
+ return getFlussSchema(tableId, tablePath, schemaId);
} catch (ExecutionException | UncheckedExecutionException executionException) {
Throwable cause = executionException.getCause();
if (cause instanceof SchemaNotExistException) {
@@ -182,10 +189,16 @@ public class SchemaMetadataManager {
}
}
+ @Override
+ public CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId) {
+ // TODO: make it async using async zookeeper API.
+ return CompletableFuture.completedFuture(new SchemaInfo(getSchema(schemaId), schemaId));
+ }
+
@Override
public void release() {
if (!released) {
- unsubscribe(tablePath);
+ unsubscribe(tableId);
released = true;
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
index 6a11a53ae..f4ecff6f0 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
@@ -68,13 +68,12 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
private final MetadataManager metadataManager;
- private final SchemaMetadataManager schemaMetadataManager;
+ private final ServerSchemaCache serverSchemaCache;
- // todo: replace this in test with schemaMetadataManager.
public TabletServerMetadataCache(MetadataManager metadataManager) {
this.serverMetadataSnapshot = ServerMetadataSnapshot.empty();
this.metadataManager = metadataManager;
- this.schemaMetadataManager = new SchemaMetadataManager(metadataManager);
+ this.serverSchemaCache = new ServerSchemaCache(metadataManager);
}
@Override
@@ -141,14 +140,14 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
}
public SchemaGetter subscribeWithInitialSchema(
- TablePath tablePath, int initialSchemaId, Schema initialSchema) {
- return schemaMetadataManager.subscribeWithInitialSchema(
- tablePath, initialSchemaId, initialSchema);
+ TablePath tablePath, long tableId, int initialSchemaId, Schema initialSchema) {
+ return serverSchemaCache.subscribeWithInitialSchema(
+ tableId, tablePath, initialSchemaId, initialSchema);
}
- public void updateLatestSchema(TablePath tablePath, SchemaInfo schemaInfo) {
- schemaMetadataManager.updateLatestSchema(
- tablePath, (short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ public void updateLatestSchema(long tableId, SchemaInfo schemaInfo) {
+ serverSchemaCache.updateLatestSchema(
+ tableId, (short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
}
public Optional<PartitionMetadata> getPartitionMetadata(PhysicalTablePath partitionPath) {
@@ -207,8 +206,7 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
// todo: apply schema id and schema info if needs
int schemaId = tableInfo.getSchemaId();
Schema schema = tableInfo.getSchema();
- schemaMetadataManager.updateLatestSchema(
- tablePath, (short) schemaId, schema);
+ serverSchemaCache.updateLatestSchema(tableId, (short) schemaId, schema);
if (tableId == DELETED_TABLE_ID) {
Long removedTableId = tableIdByPath.remove(tablePath);
@@ -437,7 +435,7 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
}
@VisibleForTesting
- public SchemaMetadataManager getSchemaMetadataManager() {
- return schemaMetadataManager;
+ public ServerSchemaCache getServerSchemaCache() {
+ return serverSchemaCache;
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
index 251d670b6..d3b5d63c9 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
@@ -77,7 +77,7 @@ public class AdjustIsrManager {
public CompletableFuture<LeaderAndIsr> submit(
TableBucket tableBucket, LeaderAndIsr leaderAndIsr) {
- // TODOadd coordinatorEpoch.
+ // TODO add coordinatorEpoch.
CompletableFuture<LeaderAndIsr> future = new CompletableFuture<>();
AdjustIsrItem adjustIsrItem = new AdjustIsrItem(tableBucket, leaderAndIsr, future);
boolean enqueued = unsentAdjustIsrMap.putIfAbsent(tableBucket, adjustIsrItem) == null;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index ad2dcb24d..d8c88cbd7 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -237,6 +237,7 @@ public final class Replica {
this.schemaGetter =
metadataCache.subscribeWithInitialSchema(
physicalPath.getTablePath(),
+ tableInfo.getTableId(),
tableInfo.getSchemaId(),
tableInfo.getSchema());
this.tableConfig = tableInfo.getTableConfig();
@@ -1517,7 +1518,7 @@ public final class Replica {
List<Integer> isrToSend = new ArrayList<>(isrState.isr());
isrToSend.add(newInSyncReplicaId);
- // TODOadd server epoch to isr.
+ // TODO add server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
@@ -1539,7 +1540,7 @@ public final class Replica {
// erroneously advance the HW if the `AdjustIsr` were to fail. Hence, the "maximal ISR"
// for `PendingShrinkIsr` is the current ISR.
- // TODOadd server epoch to isr.
+ // TODO add server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index 9f6e4d8ad..206586bf1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -39,6 +39,7 @@ import org.apache.fluss.metrics.MetricNames;
import org.apache.fluss.metrics.groups.MetricGroup;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.remote.RemoteLogFetchInfo;
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.rpc.RpcClient;
@@ -149,6 +150,7 @@ public class ReplicaManager {
private final Map<TableBucket, HostedReplica> allReplicas = MapUtils.newConcurrentHashMap();
private final TabletServerMetadataCache metadataCache;
+ private final ProjectionPushdownCache projectionsCache = new ProjectionPushdownCache();
private final Lock replicaStateChangeLock = new ReentrantLock();
/**
@@ -1083,7 +1085,8 @@ public class ReplicaManager {
adjustedMaxBytes,
replica.getSchemaGetter(),
replica.getArrowCompressionInfo(),
- fetchReqInfo.getProjectFields());
+ fetchReqInfo.getProjectFields(),
+ projectionsCache);
LogReadInfo readInfo = replica.fetchRecords(fetchParams);
// Once we read from a non-empty bucket, we stop ignoring request and bucket
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
index 06e94e9ae..9e4479b3e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
@@ -225,7 +225,7 @@ public class ReplicaFetcherManager {
fetcherThread.getLeader().leaderServerId(),
initialFetchStatusMap);
} catch (InterruptedException e) {
- LOG.error("Interrupted whileadd buckets to fetcher threads.", e);
+ LOG.error("Interrupted while add buckets to fetcher threads.", e);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
index 76195a0df..398bfb6ff 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
@@ -80,7 +80,7 @@ final class ReplicaFetcherThread extends ShutdownableThread {
private final LeaderEndpoint leader;
private final int fetchBackOffMs;
- // manuallyadd timout logic in here, todo remove this timeout logic if
+ // manually add timout logic in here, todo remove this timeout logic if
// we support global request timeout in #279
private final int timeoutSeconds = 30;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
index 062eec73d..c89fca6f7 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
@@ -65,7 +65,7 @@ public class RpcGatewayManager<T extends RpcGateway> implements AutoCloseable {
}
/**
- * Add a server to the manager. It'll create a new gateway for the server andadd it to the
+ * Add a server to the manager. It'll create a new gateway for the server and add it to the
* manager. If the server has already existed, it'll remove the already existing server before
* adding the new one.
*/
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
index 165fd23e9..815771166 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
@@ -339,7 +339,7 @@ public class ServerRpcMessageUtils {
.collect(Collectors.toList());
}
- private static @Nullable TableChange.ColumnPosition toColumnPosition(int columnPositionType) {
+ private static TableChange.ColumnPosition toColumnPosition(int columnPositionType) {
ColumnPositionType opType = ColumnPositionType.from(columnPositionType);
switch (opType) {
case LAST:
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
index 270d334ee..ea6aead7b 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
@@ -35,7 +35,7 @@ public interface Timer {
* Add a new task to this executor. It will be executed after the task's delay (beginning from
* the time of submission).
*
- * @param timerTask the task toadd
+ * @param timerTask the task to add
*/
void add(TimerTask timerTask);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
index 934777c93..1dab9f229 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
@@ -108,8 +108,8 @@ import java.util.concurrent.atomic.AtomicInteger;
* while priority queue based timers takes O(log N) for both insert and delete where N is the number
* of items in the queue.
*
- * <p>This class is not thread-safe. There should not be anyadd calls while advanceClock is
- * executing. It is caller's responsibility to enforce it. Simultaneousadd calls are thread-safe.
+ * <p>This class is not thread-safe. There should not be any add calls while advanceClock is
+ * executing. It is caller's responsibility to enforce it. Simultaneous add calls are thread-safe.
*/
@NotThreadSafe
final class TimingWheel {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
index e27ffd9c3..75ca2b7b4 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
@@ -536,18 +536,46 @@ public class ZooKeeperClient implements AutoCloseable {
"tables registration");
}
- /** Get the v1 schema for given tables in ZK. */
- public Map<TablePath, SchemaInfo> getV1Schemas(Collection<TablePath> tablePaths)
+ /** Get the latest schema for given tables in ZK. */
+ public Map<TablePath, SchemaInfo> getLatestSchemas(Collection<TablePath> tablePaths)
throws Exception {
- Map<String, TablePath> path2TablePathMap =
- tablePaths.stream()
- .collect(toMap(p -> SchemaZNode.path(p, DEFAULT_SCHEMA_ID), path -> path));
+ Map<String, TablePath> schemaChildren2TablePathMap =
+ tablePaths.stream().collect(toMap(SchemasZNode::path, path -> path));
+ List<ZkGetChildrenResponse> childrenResponses =
+ getChildrenInBackground(schemaChildren2TablePathMap.keySet());
+ // get the schema ids for each table
+ Map<TablePath, List<String>> schemaIdsForTables =
+ processGetChildrenResponses(
+ childrenResponses,
+ response -> schemaChildren2TablePathMap.get(response.getPath()),
+ "schema children for tables");
+
+ // get the schema info for each latest schema id
+ Map<TablePath, Integer> latestSchemaIdMap = new HashMap<>();
+ Map<String, TablePath> path2TablePathMap = new HashMap<>();
+ schemaIdsForTables.forEach(
+ (tp, schemaIds) -> {
+ int latestSchemaId =
+ schemaIds.stream().map(Integer::parseInt).reduce(Math::max).orElse(0);
+ latestSchemaIdMap.put(tp, latestSchemaId);
+ path2TablePathMap.put(SchemaZNode.path(tp, latestSchemaId), tp);
+ });
+
List<ZkGetDataResponse> responses = getDataInBackground(path2TablePathMap.keySet());
- return processGetDataResponses(
- responses,
- resp -> path2TablePathMap.get(resp.getPath()),
- b -> new SchemaInfo(SchemaZNode.decode(b), DEFAULT_SCHEMA_ID),
- "schema");
+ Map<TablePath, Schema> schemasForTables =
+ processGetDataResponses(
+ responses,
+ resp -> path2TablePathMap.get(resp.getPath()),
+ SchemaZNode::decode,
+ "schema");
+
+ Map<TablePath, SchemaInfo> result = new HashMap<>();
+ schemasForTables.forEach(
+ (tp, schema) -> {
+ int schemaId = latestSchemaIdMap.get(tp);
+ result.put(tp, new SchemaInfo(schema, schemaId));
+ });
+ return result;
}
/** Update the table in ZK. */
@@ -809,8 +837,8 @@ public class ZooKeeperClient implements AutoCloseable {
// --------------------------------------------------------------------------------------------
/** Register schema to ZK metadata and return the schema id. */
- public int registerSchema(TablePath tablePath, Schema schema) throws Exception {
- return registerSchema(tablePath, schema, getCurrentSchemaId(tablePath) + 1);
+ public int registerFirstSchema(TablePath tablePath, Schema schema) throws Exception {
+ return registerSchema(tablePath, schema, DEFAULT_SCHEMA_ID);
}
public int registerSchema(TablePath tablePath, Schema schema, int schemaId) throws Exception {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
index abc575fd5..af69caffe 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
@@ -487,7 +487,7 @@ public class DefaultAuthorizerTest {
Set<AccessControlEntry> expectedAcls = new HashSet<>();
for (int i = 0; i < 50; i++) {
- // each task represent that addColumn acl to different user on same resource.
+ // each task represent that add acl to different user on same resource.
final AccessControlEntry accessControlEntry =
createAclEntry(String.valueOf(i), "host-1", READ);
int finalI = i;
@@ -550,7 +550,7 @@ public class DefaultAuthorizerTest {
// generate 50 concurrent acl operation tasks.
List<Runnable> concurrentTasks = new ArrayList<>();
for (int i = 0; i < 50; i++) {
- // each task represent that addColumn acl to same user on same resource.
+ // each task represent that add acl to same user on same resource.
Runnable runnable =
() -> {
addAcls(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
index 4460cfc26..b8fec59ca 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
@@ -99,13 +99,13 @@ class CompletedSnapshotStoreManagerTest {
CompletedSnapshotStoreManager completedSnapshotStoreManager =
createCompletedSnapshotStoreManager(maxNumberOfSnapshotsToRetain);
- // addColumn snapshots for a series of buckets
+ // add snapshots for a series of buckets
Set<TableBucket> tableBuckets = createTableBuckets(2, 3);
int snapshotNum = 3;
Map<TableBucket, CompletedSnapshot> tableBucketLatestCompletedSnapshots = new HashMap<>();
for (TableBucket tableBucket : tableBuckets) {
- // addColumn some snapshots
+ // add some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
@@ -137,7 +137,7 @@ class CompletedSnapshotStoreManagerTest {
assertThat(completedSnapshot)
.isEqualTo(tableBucketLatestCompletedSnapshots.get(tableBucket));
- // addColumn a new snapshot
+ // add a new snapshot
long snapshotId = completedSnapshot.getSnapshotID() + 1;
completedSnapshot = KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshotId);
addCompletedSnapshot(completedSnapshotStoreManager, completedSnapshot);
@@ -169,7 +169,7 @@ class CompletedSnapshotStoreManagerTest {
Set<TableBucket> tableBuckets = createTableBuckets(1, 2);
int snapshotNum = 3;
for (TableBucket tableBucket : tableBuckets) {
- // addColumn some snapshots
+ // add some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
index 581edafcc..bd4467c0a 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
@@ -69,14 +69,14 @@ class CoordinatorChannelManagerTest {
// now, shouldn't send as we already remove the tablet server
checkSendRequest(coordinatorChannelManager, server0.id(), false);
- // test addColumn tablet server
- // before addColumn, shouldn't send
+ // test add tablet server
+ // before add, shouldn't send
ServerNode server1 = tabletServersNode.get(1);
checkSendRequest(coordinatorChannelManager, server1.id(), false);
coordinatorChannelManager.addTabletServer(server1);
- // after addColumn the tablet server, should send
+ // after add the tablet server, should send
// try to send message
checkSendRequest(coordinatorChannelManager, server1.id(), true);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
index b8b348bdb..3c417efe8 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
@@ -211,8 +211,6 @@ class TableManagerTest {
assertThat(coordinatorContext.getAllReplicasForTable(tableId)).isEmpty();
}
- void testSchemaChange() throws Exception {}
-
@Test
void testResumeDeletionAfterRestart() throws Exception {
// first, create a table
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
index 87b768608..f2013a91a 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
@@ -38,6 +38,7 @@ import org.apache.fluss.record.KvRecordTestUtils;
import org.apache.fluss.record.LogRecords;
import org.apache.fluss.record.LogTestBase;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.record.TestData;
import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.record.bytesview.MultiBytesView;
@@ -651,9 +652,10 @@ class KvTabletTest {
TablePath tablePath = TablePath.of("testDb", tableName);
initLogTabletAndKvTablet(tablePath, DATA1_SCHEMA_PK, config);
RowType rowType = DATA1_SCHEMA_PK.getRowType();
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
FileLogProjection logProjection = null;
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
@@ -749,7 +751,7 @@ class KvTabletTest {
schemaGetter.updateLatestSchemaInfo(new SchemaInfo(DATA2_SCHEMA, 2));
readLogRowType = doProjection ? DATA2_ROW_TYPE.project(new int[] {0}) : DATA2_ROW_TYPE;
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
@@ -834,9 +836,10 @@ class KvTabletTest {
KvRecordTestUtils.KvRecordFactory kvRecordFactory =
KvRecordTestUtils.KvRecordFactory.of(rowType);
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
FileLogProjection logProjection = null;
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
@@ -956,7 +959,7 @@ class KvTabletTest {
? newSchema.getRowType().project(new int[] {0})
: newSchema.getRowType();
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java
index 74d145bda..b39ec39ac 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java
@@ -163,13 +163,13 @@ class KvPreWriteBufferTest {
}
assertThat(getValue(buffer, "key6")).isNull();
- // addColumn delete records.
+ // add delete records.
elementCount = 5;
bufferDelete(buffer, "key4", elementCount++);
bufferDelete(buffer, "key3", elementCount++);
assertThat(getValue(buffer, "key3")).isNull();
- // addColumn update records
+ // add update records
bufferPut(buffer, "key2", "value2-1", elementCount++);
bufferPut(buffer, "key1", "value1-1", elementCount++);
assertThat(getValue(buffer, "key1")).isEqualTo("value1-1");
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java
index 95afb907d..3b5fc88c4 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java
@@ -20,9 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.types.DataTypes;
import org.junit.jupiter.params.ParameterizedTest;
@@ -47,18 +45,21 @@ class DefaultRowMergerTest {
Schema.newBuilder()
.fromColumns(
Arrays.asList(
- new Schema.Column("age", DataTypes.STRING(), null, (short) 2),
new Schema.Column("id", DataTypes.INT(), null, (short) 0),
- new Schema.Column("name", DataTypes.STRING(), null, (short) 1)))
+ new Schema.Column("name", DataTypes.STRING(), null, (short) 1),
+ // add new column at end
+ new Schema.Column("age", DataTypes.STRING(), null, (short) 2)))
.primaryKey("id")
.build();
- private BinaryRow createBinaryRow(int id, String name) {
- return compactedRow(SCHEMA.getRowType(), new Object[] {id, name});
+ private BinaryValue createBinaryValue(int id, String name) {
+ return new BinaryValue(
+ (short) 1, compactedRow(SCHEMA.getRowType(), new Object[] {id, name}));
}
- private BinaryRow createBinaryRow(String age, int id, String name) {
- return compactedRow(SCHEMA_2.getRowType(), new Object[] {age, id, name});
+ private BinaryValue createBinaryValue(int id, String name, String age) {
+ return new BinaryValue(
+ (short) 2, compactedRow(SCHEMA_2.getRowType(), new Object[] {id, name, age}));
}
@ParameterizedTest
@@ -67,26 +68,25 @@ class DefaultRowMergerTest {
DefaultRowMerger merger = new DefaultRowMerger(KvFormat.COMPACTED, deleteBehavior);
merger.configureTargetColumns(null, (byte) 1, SCHEMA);
- InternalRow oldRow = createBinaryRow(1, "old");
- BinaryRow newRow = createBinaryRow(1, "new");
+ BinaryValue oldValue = createBinaryValue(1, "old");
+ BinaryValue newValue = createBinaryValue(1, "new");
// Test merge operation - should return new row
- InternalRow mergedRow = merger.merge(oldRow, newRow);
- assertThat(mergedRow).isSameAs(newRow);
+ BinaryValue mergedValue = merger.merge(oldValue, newValue);
+ assertThat(mergedValue).isSameAs(newValue);
// Test delete operation - should return null (deleted)
- InternalRow deletedRow = merger.delete(oldRow);
- assertThat(deletedRow).isNull();
+ BinaryValue deletedValue = merger.delete(oldValue);
+ assertThat(deletedValue).isNull();
// Test supportsDelete - should return true
assertThat(merger.deleteBehavior()).isEqualTo(deleteBehavior);
// Test schema change.
merger.configureTargetColumns(null, (byte) 2, SCHEMA_2);
- oldRow = ProjectedRow.from(SCHEMA, SCHEMA_2).replaceRow(oldRow);
- newRow = createBinaryRow("20", 1, "new2");
- assertThat(merger.merge(oldRow, newRow)).isSameAs(newRow);
- assertThat(merger.delete(newRow)).isNull();
+ newValue = createBinaryValue(1, "new2", "20");
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(newValue);
+ assertThat(merger.delete(newValue)).isNull();
}
@ParameterizedTest
@@ -98,20 +98,19 @@ class DefaultRowMergerTest {
RowMerger partialMerger =
merger.configureTargetColumns(new int[] {0, 1}, (byte) 1, SCHEMA); // id + name
- InternalRow oldRow = createBinaryRow(1, "old");
+ BinaryValue oldValue = createBinaryValue(1, "old");
- InternalRow ignoredRow = partialMerger.delete(oldRow);
- assertThat(ignoredRow).isNull();
+ BinaryValue ignoredValue = partialMerger.delete(oldValue);
+ assertThat(ignoredValue).isNull();
assertThat(partialMerger.deleteBehavior()).isEqualTo(deleteBehavior);
- assertThat(partialMerger.merge(null, oldRow)).isEqualTo(oldRow);
+ assertThat(partialMerger.merge(null, oldValue)).isEqualTo(oldValue);
// schema change then partial update (except name column).
- partialMerger = merger.configureTargetColumns(new int[] {0, 1}, (byte) 2, SCHEMA_2);
- oldRow = ProjectedRow.from(SCHEMA, SCHEMA_2).replaceRow(oldRow);
- BinaryRow newRow = createBinaryRow("20", 1, null);
- BinaryRow mergeRow = createBinaryRow("20", 1, "old");
- assertThat(partialMerger.merge(oldRow, newRow)).isEqualTo(mergeRow);
- assertThat(partialMerger.delete(mergeRow)).isEqualTo(createBinaryRow(null, 1, "old"));
+ partialMerger = merger.configureTargetColumns(new int[] {0, 2}, (byte) 2, SCHEMA_2);
+ BinaryValue newValue = createBinaryValue(1, null, "20");
+ BinaryValue mergeValue = createBinaryValue(1, "old", "20");
+ assertThat(partialMerger.merge(oldValue, newValue)).isEqualTo(mergeValue);
+ assertThat(partialMerger.delete(mergeValue)).isEqualTo(createBinaryValue(1, "old", null));
}
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java
index 008c2fafa..9163b23b0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java
@@ -19,9 +19,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
import org.apache.fluss.types.DataType;
@@ -132,13 +130,13 @@ class VersionedRowMergerTest {
merger.configureTargetColumns(null, (short) 1, schema);
for (TestSpec testSpec : testSpecs) {
- BinaryRow oldRow = compactedRow(rowType, new Object[] {testSpec.oldValue, "dummy"});
- BinaryRow newRow = compactedRow(rowType, new Object[] {testSpec.newValue, "dummy"});
- InternalRow mergedRow = merger.merge(oldRow, newRow);
+ BinaryValue oldValue = binaryValue(rowType, new Object[] {testSpec.oldValue, "dummy"});
+ BinaryValue newValue = binaryValue(rowType, new Object[] {testSpec.newValue, "dummy"});
+ BinaryValue mergedValue = merger.merge(oldValue, newValue);
if (testSpec.expected.equals("old")) {
- assertThat(mergedRow).isSameAs(oldRow);
+ assertThat(mergedValue).isSameAs(oldValue);
} else if (testSpec.expected.equals("new")) {
- assertThat(mergedRow).isSameAs(newRow);
+ assertThat(mergedValue).isSameAs(newValue);
} else {
throw new IllegalArgumentException("Unknown expected value: " + testSpec.expected);
}
@@ -169,27 +167,29 @@ class VersionedRowMergerTest {
RowType rowType = schema.getRowType();
VersionedRowMerger merger = new VersionedRowMerger("a", DeleteBehavior.DISABLE);
merger.configureTargetColumns(null, (short) 1, schema);
- InternalRow oldRow = compactedRow(rowType, new Object[] {11, "dummy"});
- BinaryRow newRow = compactedRow(rowType, new Object[] {2, "dummy"});
- assertThat(merger.merge(oldRow, newRow)).isSameAs(oldRow);
+ BinaryValue oldValue = binaryValue(rowType, new Object[] {11, "dummy"});
+ BinaryValue newValue = binaryValue(rowType, new Object[] {2, "dummy"});
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(oldValue);
Schema schema2 =
Schema.newBuilder()
.fromColumns(
Arrays.asList(
- new Schema.Column("c", DataTypes.STRING(), null, (short) 2),
new Schema.Column("a", DataTypes.INT(), null, (short) 0),
+ new Schema.Column("b", DataTypes.STRING(), null, (short) 1),
+ // add new column at end
new Schema.Column(
- "b", DataTypes.STRING(), null, (short) 1)))
+ "c", DataTypes.STRING(), null, (short) 2)))
.build();
rowType = schema2.getRowType();
merger.configureTargetColumns(null, (short) 2, schema2);
- oldRow = ProjectedRow.from(schema, schema2).replaceRow(oldRow);
- newRow = compactedRow(rowType, new Object[] {"a", 2, "dummy"});
- assertThat(merger.merge(oldRow, newRow)).isSameAs(oldRow);
- newRow = compactedRow(rowType, new Object[] {"b", 20, "dummy"});
- assertThat(merger.merge(oldRow, newRow)).isSameAs(newRow);
+ newValue = binaryValue(rowType, new Object[] {2, "dummy", "a"});
+ // version is smaller than old value
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(oldValue);
+ newValue = binaryValue(rowType, new Object[] {20, "dummy", "b"});
+ // version is greater than old value
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(newValue);
}
private static TimestampNtz timestampNtz(String timestamp) {
@@ -201,6 +201,11 @@ class VersionedRowMergerTest {
return TimestampLtz.fromInstant(instant);
}
+ private static BinaryValue binaryValue(RowType rowType, Object[] objects) {
+ int schemaId = rowType.getFieldCount() == 2 ? 1 : 2;
+ return new BinaryValue((short) schemaId, compactedRow(rowType, objects));
+ }
+
/** Test specification for {@link VersionedRowMerger}. */
private static class TestSpec {
private final Object oldValue;
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java
index 87fb23259..8d45628d8 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java
@@ -161,7 +161,7 @@ class CompletedSnapshotStoreTest {
final CompletedSnapshot ckp = getSnapshot(ckpId);
assertThatThrownBy(() -> completedSnapshotStore.add(ckp))
- .as("We should get an exception when addColumn snapshot to failed..")
+ .as("We should get an exception when add snapshot to failed..")
.hasMessageContaining(errMsg)
.isInstanceOf(FlussException.class);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
index 2451e7ecc..9f9ea078d 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
@@ -247,7 +247,7 @@ class KvTabletSnapshotTargetTest {
final String errMsg = "Add to snapshot handle failed.";
AtomicBoolean shouldFail = new AtomicBoolean(true);
// we use a store will fail when the variable shouldFail is true
- // addColumn the snapshot to store will fail
+ // add the snapshot to store will fail
CompletedSnapshotHandleStore completedSnapshotHandleStore =
TestCompletedSnapshotHandleStore.newBuilder()
.setAddFunction(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java
index 5b890acd6..c6e90ff63 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java
@@ -94,8 +94,8 @@ class ZooKeeperCompletedSnapshotStoreTest {
}
/**
- * Tests that the snapshot does not exist in the store when we fail to addColumn it into the
- * store (i.e., there exists an exception thrown by the method).
+ * Tests that the snapshot does not exist in the store when we fail to add it into the store
+ * (i.e., there exists an exception thrown by the method).
*/
@Test
void testAddSnapshotWithFailedRemove(@TempDir Path tmpDir) throws Exception {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java
index b217d8234..cea1513c5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java
@@ -109,7 +109,7 @@ final class DroppedTableRecoveryTest extends LogTestBase {
ZOO_KEEPER_EXTENSION_WRAPPER.getCustomExtension().cleanupRoot();
zkClient.registerTable(
tablePath, TableRegistration.newTable(DATA1_TABLE_ID, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(tablePath, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(tablePath, DATA1_SCHEMA);
}
@AfterEach
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java
index 132de4482..d4c2be57b 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java
@@ -19,6 +19,7 @@ package org.apache.fluss.server.log;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.record.FileLogProjection;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.record.TestData;
import org.apache.fluss.record.TestingSchemaGetter;
@@ -33,13 +34,15 @@ class FetchParamsTest {
@Test
void testSetCurrentFetch() {
FetchParams fetchParams = new FetchParams(1, 100);
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
fetchParams.setCurrentFetch(
1L,
20L,
1024,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA1_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- null);
+ null,
+ projectionCache);
assertThat(fetchParams.fetchOffset()).isEqualTo(20L);
assertThat(fetchParams.maxFetchBytes()).isEqualTo(1024);
assertThat(fetchParams.projection()).isNull();
@@ -50,7 +53,8 @@ class FetchParamsTest {
512,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA2_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- new int[] {0, 2});
+ new int[] {0, 2},
+ projectionCache);
assertThat(fetchParams.fetchOffset()).isEqualTo(30L);
assertThat(fetchParams.maxFetchBytes()).isEqualTo(512);
assertThat(fetchParams.projection()).isNotNull();
@@ -63,7 +67,8 @@ class FetchParamsTest {
256,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA1_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- null);
+ null,
+ projectionCache);
assertThat(fetchParams.projection()).isNull();
fetchParams.setCurrentFetch(
@@ -72,7 +77,8 @@ class FetchParamsTest {
512,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA2_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- new int[] {0, 2});
+ new int[] {0, 2},
+ projectionCache);
// the FileLogProjection should be cached
assertThat(fetchParams.projection()).isNotNull().isSameAs(prevProjection);
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java
index f4e536b86..80e99c2f5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java
@@ -79,7 +79,7 @@ final class LogManagerTest extends LogTestBase {
private TableBucket tableBucket2;
private LogManager logManager;
- // TODOadd more tests refer to kafka's LogManagerTest.
+ // TODO add more tests refer to kafka's LogManagerTest.
@BeforeAll
static void baseBeforeAll() {
@@ -114,10 +114,10 @@ final class LogManagerTest extends LogTestBase {
ZOO_KEEPER_EXTENSION_WRAPPER.getCustomExtension().cleanupRoot();
zkClient.registerTable(
tablePath1, TableRegistration.newTable(DATA1_TABLE_ID, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(tablePath1, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(tablePath1, DATA1_SCHEMA);
zkClient.registerTable(
tablePath2, TableRegistration.newTable(DATA2_TABLE_ID, DATA2_TABLE_DESCRIPTOR));
- zkClient.registerSchema(tablePath2, DATA2_SCHEMA);
+ zkClient.registerFirstSchema(tablePath2, DATA2_SCHEMA);
}
static List<String> partitionProvider() {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java
index 7af086823..7b88a53a5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java
@@ -73,7 +73,7 @@ final class LogTabletTest extends LogTestBase {
private FlussScheduler scheduler;
private File logDir;
- // TODOadd more tests refer to kafka's UnifiedLogTest.
+ // TODO add more tests refer to kafka's UnifiedLogTest.
@BeforeEach
public void setup() throws Exception {
@@ -374,7 +374,7 @@ final class LogTabletTest extends LogTestBase {
log.updateHighWatermark(log.localLogEndOffset());
- // TODOadd delete log segment logic.
+ // TODO add delete log segment logic.
}
@Test
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java
index 5bedd921e..0d571e1ea 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java
@@ -94,7 +94,7 @@ class RemoteLogTabletTest extends RemoteLogTestBase {
assertThat(remoteLogTablet.getRemoteLogStartOffset()).isEqualTo(0);
assertThat(remoteLogTablet.getRemoteLogEndOffset()).isEqualTo(OptionalLong.of(30));
- // delete first remote log segment and addColumn another one remote log segments.
+ // delete first remote log segment and add another one remote log segments.
remoteLogTablet.addAndDeleteLogSegments(
Collections.singletonList(remoteLogSegmentList.get(3)),
Collections.singletonList(remoteLogSegmentList.get(0)));
@@ -102,7 +102,7 @@ class RemoteLogTabletTest extends RemoteLogTestBase {
assertThat(remoteLogTablet.getRemoteLogStartOffset()).isEqualTo(10);
assertThat(remoteLogTablet.getRemoteLogEndOffset()).isEqualTo(OptionalLong.of(40));
- // delete all exist and append one. we will first addColumn then delete.
+ // delete all exist and append one. we will first add then delete.
remoteLogTablet.addAndDeleteLogSegments(
Collections.singletonList(remoteLogSegmentList.get(4)), remoteLogSegmentList);
assertThat(remoteLogTablet.getIdToRemoteLogSegmentMap()).hasSize(0);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java
index 35c36d30c..b6fdd9e93 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java
@@ -157,7 +157,7 @@ class MetadataUpdateITCase {
expectedTablePathById,
Collections.emptyMap()));
- // addColumn back tablet server2
+ // add back tablet server2
FLUSS_CLUSTER_EXTENSION.startTabletServer(2);
retry(
Duration.ofMinutes(1),
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/metadata/SchemaMetadataManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
similarity index 73%
rename from fluss-server/src/test/java/org/apache/fluss/server/metadata/SchemaMetadataManagerTest.java
rename to fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
index 672910b04..c42c7b296 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/metadata/SchemaMetadataManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
@@ -44,40 +44,37 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_DESCRIPTOR;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-/** Test for {@link SchemaMetadataManager}. */
-public class SchemaMetadataManagerTest {
+/** Test for {@link ServerSchemaCache}. */
+public class ServerSchemaCacheTest {
@Test
- void testPublishAndsubscribeSchemaChange() {
- SchemaMetadataManager manager =
- new SchemaMetadataManager(new TestingMetadataManager(Collections.emptyList()));
+ void testPublishAndSubscribeSchemaChange() {
+ ServerSchemaCache manager =
+ new ServerSchemaCache(new TestingMetadataManager(Collections.emptyList()));
SchemaGetter subscriber1 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
+ 1L, new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
SchemaGetter subscriber2 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
+ 2L, new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA1_SCHEMA, 1));
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA1_SCHEMA, 1));
- manager.updateLatestSchema(
- new TablePath("test_tb", "test_table_2"), (short) 2, DATA2_SCHEMA);
+ manager.updateLatestSchema(2L, (short) 2, DATA2_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA1_SCHEMA, 1));
assertThat(subscriber2.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber2.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
- manager.updateLatestSchema(
- new TablePath("test_tb", "test_table_1"), (short) 2, DATA2_SCHEMA);
+ manager.updateLatestSchema(1L, (short) 2, DATA2_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
assertThat(subscriber2.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber2.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
// one more redundant schema change.
- manager.updateLatestSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA2_SCHEMA);
+ manager.updateLatestSchema(1L, (short) 1, DATA2_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
assertThat(subscriber2.getLatestSchemaInfo()).isNotNull();
@@ -94,11 +91,12 @@ public class SchemaMetadataManagerTest {
DATA2_TABLE_DESCRIPTOR,
System.currentTimeMillis(),
System.currentTimeMillis());
- SchemaMetadataManager manager =
- new SchemaMetadataManager(
+ ServerSchemaCache manager =
+ new ServerSchemaCache(
new TestingMetadataManager(Arrays.asList(DATA1_TABLE_INFO, tableInfo2)));
SchemaGetter schemaGetter =
- manager.subscribeWithInitialSchema(DATA1_TABLE_PATH, (short) 2, DATA2_SCHEMA);
+ manager.subscribeWithInitialSchema(
+ DATA1_TABLE_ID, DATA1_TABLE_PATH, (short) 2, DATA2_SCHEMA);
assertThat(schemaGetter.getSchema(2)).isEqualTo(DATA2_SCHEMA);
assertThat(schemaGetter.getSchema(1)).isEqualTo(DATA1_SCHEMA);
assertThatThrownBy(() -> schemaGetter.getSchema(3))
@@ -107,43 +105,39 @@ public class SchemaMetadataManagerTest {
@Test
void testUnsubscribeSchemaChange() {
- SchemaMetadataManager manager =
- new SchemaMetadataManager(new TestingMetadataManager(Collections.emptyList()));
+ ServerSchemaCache manager =
+ new ServerSchemaCache(new TestingMetadataManager(Collections.emptyList()));
SchemaGetter subscriber1 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
+ 1L, new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
SchemaGetter subscriber2 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
+ 1L, new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
SchemaGetter subscriber3 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(2);
+ 2L, new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(2);
assertThat(manager.getSubscriberCounters()).hasSize(2);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isEqualTo(2);
+ assertThat(manager.getSubscriberCounters().get(1L)).isEqualTo(2);
subscriber1.release();
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(2);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(2);
assertThat(manager.getSubscriberCounters()).hasSize(2);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isEqualTo(1);
+ assertThat(manager.getSubscriberCounters().get(1L)).isEqualTo(1);
// one more redundant unsubscribe.
subscriber1.release();
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(2);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(2);
assertThat(manager.getSubscriberCounters()).hasSize(2);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isEqualTo(1);
+ assertThat(manager.getSubscriberCounters().get(1L)).isEqualTo(1);
subscriber2.release();
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(1);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(1);
assertThat(manager.getSubscriberCounters()).hasSize(1);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isNull();
+ assertThat(manager.getSubscriberCounters().get(1L)).isNull();
subscriber3.release();
- assertThat(manager.getLatestSchemaByTablePath()).isEmpty();
+ assertThat(manager.getLatestSchemaByTableId()).isEmpty();
assertThat(manager.getSubscriberCounters()).isEmpty();
subscriber3.release();
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java
index bb2138881..933abd231 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java
@@ -140,7 +140,7 @@ public class AdjustIsrITCase {
isr.add(stopFollower);
FLUSS_CLUSTER_EXTENSION.notifyLeaderAndIsr(
stopFollower, DATA1_TABLE_PATH, tb, newLeaderAndIsr, isr);
- // retry until the stop follower addColumn back to ISR.
+ // retry until the stop follower add back to ISR.
retry(
Duration.ofMinutes(1),
() ->
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java
index 75fb9e248..3f8623569 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java
@@ -32,7 +32,7 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link AdjustIsrManager}. */
class AdjustIsrManagerTest {
- // TODO addColumn more test refer to kafka AlterPartitionManagerTest, See: FLUSS-56278513
+ // TODO add more test refer to kafka AlterPartitionManagerTest, See: FLUSS-56278513
@Test
void testSubmitShrinkIsr() throws Exception {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java
index b089efd30..b093787f0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java
@@ -92,7 +92,7 @@ final class HighWatermarkPersistenceTest extends ReplicaTestBase {
assertThat(highWatermark0).isEqualTo(10L);
assertThat(replica0.getLogTablet().getHighWatermark()).isEqualTo(10L);
- // addColumn another replica and set highWatermark.
+ // add another replica and set highWatermark.
TableBucket tableBucket1 = new TableBucket(DATA2_TABLE_ID, 0);
replicaManager.becomeLeaderOrFollower(
INITIAL_COORDINATOR_EPOCH,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java
index a7c65acda..9e13b1688 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java
@@ -130,7 +130,7 @@ class RemoveOfflineReplicaFromIsrITCase {
FLUSS_CLUSTER_EXTENSION.startTabletServer(follower);
isr.add(follower);
- // make sure the stopped follower can addColumn back to isr after restart
+ // make sure the stopped follower can add back to isr after restart
retry(
Duration.ofMinutes(1),
() ->
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java
index 92c24f871..6ec71bc79 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java
@@ -193,7 +193,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
void testFetchLog() throws Exception {
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
makeLogTableAsLeader(tb.getBucket());
@@ -309,7 +309,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
void testFetchLogWithMaxBytesLimit() throws Exception {
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
makeLogTableAsLeader(tb.getBucket());
@@ -432,7 +432,8 @@ class ReplicaManagerTest extends ReplicaTestBase {
assertThat(records1).isNotNull();
assertThat(records2).isNotNull();
SchemaGetter schemaGetter =
- serverMetadataCache.subscribeWithInitialSchema(DATA1_TABLE_PATH, 1, DATA1_SCHEMA);
+ serverMetadataCache.subscribeWithInitialSchema(
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA);
if (records1.sizeInBytes() == 0) {
assertThat(records2.sizeInBytes() > 0).isTrue();
assertMemoryRecordsEquals(
@@ -498,7 +499,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
makeKvTableAsLeader(DATA1_TABLE_ID_PK, DATA1_TABLE_PATH_PK, tb.getBucket());
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH_PK, 1, DATA1_SCHEMA_PK);
+ DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK, 1, DATA1_SCHEMA_PK);
// 1. put kv records to kv store.
List<Tuple2<Object[], Object[]>> data1 =
@@ -628,7 +629,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
makeKvTableAsLeader(DATA1_TABLE_ID_PK, DATA1_TABLE_PATH_PK, tb.getBucket());
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH_PK, 1, DATA1_SCHEMA_PK);
+ DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK, 1, DATA1_SCHEMA_PK);
// put 10 batches delete non-exists key batch to kv store.
CompletableFuture<List<PutKvResultForBucket>> future;
@@ -906,7 +907,8 @@ class ReplicaManagerTest extends ReplicaTestBase {
CompletableFuture<LimitScanResultForBucket> limitFuture = new CompletableFuture<>();
replicaManager.limitScan(tb, 10, limitFuture::complete);
SchemaGetter schemaGetter =
- serverMetadataCache.subscribeWithInitialSchema(DATA1_TABLE_PATH, 1, DATA1_SCHEMA);
+ serverMetadataCache.subscribeWithInitialSchema(
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA);
assertMemoryRecordsEquals(
DATA1_ROW_TYPE,
schemaGetter,
@@ -1463,7 +1465,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
FetchLogResultForBucket r = r1.getValue();
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH, 1, DATA1_SCHEMA);
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA);
assertLogRecordsEqualsWithRowKind(
DEFAULT_SCHEMA_ID,
DATA1_ROW_TYPE,
@@ -1591,11 +1593,12 @@ class ReplicaManagerTest extends ReplicaTestBase {
zkClient.registerTable(
nonePartitionTablePath,
TableRegistration.newTable(nonePartitionTableId, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(nonePartitionTablePath, DATA1_TABLE_DESCRIPTOR.getSchema());
+ zkClient.registerFirstSchema(nonePartitionTablePath, DATA1_TABLE_DESCRIPTOR.getSchema());
zkClient.registerTable(
partitionTablePath,
TableRegistration.newTable(partitionTableId, DATA1_PARTITIONED_TABLE_DESCRIPTOR));
- zkClient.registerSchema(partitionTablePath, DATA1_PARTITIONED_TABLE_DESCRIPTOR.getSchema());
+ zkClient.registerFirstSchema(
+ partitionTablePath, DATA1_PARTITIONED_TABLE_DESCRIPTOR.getSchema());
expectedCoordinatorServer.put(
"INTERNAL", new ServerNode(0, "localhost", 1235, ServerType.COORDINATOR));
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
index e45a5557c..c8e26ea7b 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
@@ -31,6 +31,7 @@ import org.apache.fluss.record.KvRecordTestUtils;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecords;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.server.entity.NotifyLeaderAndIsrData;
import org.apache.fluss.server.kv.KvTablet;
import org.apache.fluss.server.kv.snapshot.CompletedSnapshot;
@@ -94,8 +95,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link Replica}. */
final class ReplicaTest extends ReplicaTestBase {
- // TODO addColumn more tests refer to kafka's PartitionTest.
- // TODO addColumn more tests to cover partition table
+ // TODO add more tests refer to kafka's PartitionTest.
+ // TODO add more tests to cover partition table
@Test
void testMakeLeader() throws Exception {
@@ -137,15 +138,22 @@ final class ReplicaTest extends ReplicaTestBase {
(int)
conf.get(ConfigOptions.CLIENT_SCANNER_LOG_FETCH_MAX_BYTES)
.getBytes());
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
fetchParams.setCurrentFetch(
- DATA1_TABLE_ID, 0, Integer.MAX_VALUE, schemaGetter, DEFAULT_COMPRESSION, null);
+ DATA1_TABLE_ID,
+ 0,
+ Integer.MAX_VALUE,
+ schemaGetter,
+ DEFAULT_COMPRESSION,
+ null,
+ projectionCache);
LogReadInfo logReadInfo = logReplica.fetchRecords(fetchParams);
assertLogRecordsEquals(
DATA1_ROW_TYPE, logReadInfo.getFetchedData().getRecords(), DATA1, schemaGetter);
// schema evolution.
serverMetadataCache.updateLatestSchema(
- DATA1_TABLE_PATH, new SchemaInfo(DATA2_SCHEMA, (short) 2));
+ DATA1_TABLE_ID, new SchemaInfo(DATA2_SCHEMA, (short) 2));
mr =
createRecordsWithoutBaseLogOffset(
DATA2_ROW_TYPE,
@@ -163,7 +171,8 @@ final class ReplicaTest extends ReplicaTestBase {
Integer.MAX_VALUE,
schemaGetter,
DEFAULT_COMPRESSION,
- null);
+ null,
+ projectionCache);
logReadInfo = logReplica.fetchRecords(fetchParams);
assertLogRecordsEquals(
2, DATA2_ROW_TYPE, logReadInfo.getFetchedData().getRecords(), DATA2, schemaGetter);
@@ -393,7 +402,7 @@ final class ReplicaTest extends ReplicaTestBase {
currentOffset += 2;
short newSchemaId = 2;
serverMetadataCache.updateLatestSchema(
- DATA1_TABLE_PATH_PK, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
+ DATA1_TABLE_ID, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
KvRecordTestUtils.KvRecordBatchFactory kvRecordBatchFactory2 =
KvRecordTestUtils.KvRecordBatchFactory.of(newSchemaId);
KvRecordTestUtils.KvRecordFactory kvRecordFactory2 =
@@ -718,7 +727,7 @@ final class ReplicaTest extends ReplicaTestBase {
// update schema.
zkClient.registerSchema(DATA1_TABLE_PATH_PK, DATA2_SCHEMA, newSchemaId);
serverMetadataCache.updateLatestSchema(
- DATA1_TABLE_PATH_PK, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
+ DATA1_TABLE_ID, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
// write data with new schema
putRecordsToLeader(
kvReplica,
@@ -808,7 +817,8 @@ final class ReplicaTest extends ReplicaTestBase {
Integer.MAX_VALUE,
replica.getSchemaGetter(),
DEFAULT_COMPRESSION,
- null);
+ null,
+ new ProjectionPushdownCache());
LogReadInfo logReadInfo = replica.fetchRecords(fetchParams);
return logReadInfo.getFetchedData().getRecords();
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java
index c438da529..ae7e14a87 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java
@@ -254,16 +254,16 @@ public class ReplicaTestBase {
zkClient.registerTable(
DATA1_TABLE_PATH,
TableRegistration.newTable(DATA1_TABLE_ID, data1NonPkTableDescriptor));
- zkClient.registerSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
zkClient.registerTable(
DATA1_TABLE_PATH_PK,
TableRegistration.newTable(DATA1_TABLE_ID_PK, DATA1_TABLE_DESCRIPTOR_PK));
- zkClient.registerSchema(DATA1_TABLE_PATH_PK, DATA1_SCHEMA_PK);
+ zkClient.registerFirstSchema(DATA1_TABLE_PATH_PK, DATA1_SCHEMA_PK);
zkClient.registerTable(
DATA2_TABLE_PATH,
TableRegistration.newTable(DATA2_TABLE_ID, DATA2_TABLE_DESCRIPTOR));
- zkClient.registerSchema(DATA2_TABLE_PATH, DATA2_SCHEMA);
+ zkClient.registerFirstSchema(DATA2_TABLE_PATH, DATA2_SCHEMA);
}
protected long registerTableInZkClient(
@@ -282,7 +282,7 @@ public class ReplicaTestBase {
zkClient.deleteTable(tablePath);
}
zkClient.registerTable(tablePath, TableRegistration.newTable(tableId, tableDescriptor));
- zkClient.registerSchema(tablePath, schema);
+ zkClient.registerFirstSchema(tablePath, schema);
return tableId;
}
@@ -342,7 +342,7 @@ public class ReplicaTestBase {
}
// TODO this is only for single tablet server unit test.
- // TODO addColumn more test cases for partition table which make leader by this method.
+ // TODO add more test cases for partition table which make leader by this method.
protected void makeLogTableAsLeader(int bucketId) {
makeLogTableAsLeader(new TableBucket(DATA1_TABLE_ID, bucketId), false);
}
@@ -375,7 +375,7 @@ public class ReplicaTestBase {
}
// TODO this is only for single tablet server unit test.
- // TODO addColumn more test cases for partition table which make leader by this method.
+ // TODO add more test cases for partition table which make leader by this method.
protected void makeKvTableAsLeader(long tableId, TablePath tablePath, int bucketId) {
makeKvTableAsLeader(
new TableBucket(tableId, bucketId), tablePath, INITIAL_LEADER_EPOCH, false);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java
index ab8bde9df..b1fa9067e 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java
@@ -356,7 +356,7 @@ public class ReplicaFetcherThreadTest {
zkClient.registerTable(
DATA1_TABLE_PATH,
TableRegistration.newTable(DATA1_TABLE_ID, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
}
private void makeLeaderAndFollower() {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
index 5a5910d1e..01cf2701a 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
@@ -177,7 +177,7 @@ public class TabletServiceITCase {
}
@Test
- @Disabled("TODO: addColumn back in https://github.com/apache/fluss/issues/771")
+ @Disabled("TODO: add back in https://github.com/apache/fluss/issues/771")
void testProduceLogResponseReturnInOrder() throws Exception {
long tableId =
createTable(
@@ -330,11 +330,11 @@ public class TabletServiceITCase {
tableId,
0,
Errors.INVALID_COLUMN_PROJECTION.code(),
- "Projected field id 2 is not contains in [0, 1]");
+ "Projected field id 2 is not contained in [0, 1]");
}
@Test
- @Disabled("TODO: addColumn back in https://github.com/apache/fluss/issues/777")
+ @Disabled("TODO: add back in https://github.com/apache/fluss/issues/777")
void testFetchLogWithMinFetchSizeAndTimeout() throws Exception {
long tableId =
createTable(FLUSS_CLUSTER_EXTENSION, DATA1_TABLE_PATH, DATA1_TABLE_DESCRIPTOR);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
index 37f3dd6cb..a1ee83fa6 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
@@ -51,8 +51,8 @@ import org.apache.fluss.server.coordinator.MetadataManager;
import org.apache.fluss.server.entity.NotifyLeaderAndIsrData;
import org.apache.fluss.server.kv.snapshot.CompletedSnapshot;
import org.apache.fluss.server.kv.snapshot.CompletedSnapshotHandle;
-import org.apache.fluss.server.metadata.SchemaMetadataManager;
import org.apache.fluss.server.metadata.ServerInfo;
+import org.apache.fluss.server.metadata.ServerSchemaCache;
import org.apache.fluss.server.metadata.TabletServerMetadataCache;
import org.apache.fluss.server.replica.Replica;
import org.apache.fluss.server.replica.ReplicaManager;
@@ -286,7 +286,7 @@ public final class FlussClusterExtension
}
private void startTabletServers() throws Exception {
- // addColumn tablet server to make generate assignment for table possible
+ // add tablet server to make generate assignment for table possible
for (int i = 0; i < initialNumOfTabletServers; i++) {
startTabletServer(i);
}
@@ -670,20 +670,20 @@ public final class FlussClusterExtension
() -> {
TableRegistration tableRegistration = zkClient.getTable(tablePath).get();
int bucketCount = tableRegistration.bucketCount;
+ long tableId = tableRegistration.tableId;
for (int bucketId = 0; bucketId < bucketCount; bucketId++) {
- TableBucket tableBucket =
- new TableBucket(tableRegistration.tableId, bucketId);
+ TableBucket tableBucket = new TableBucket(tableId, bucketId);
Optional<LeaderAndIsr> leaderAndIsrOpt =
zkClient.getLeaderAndIsr(tableBucket);
assertThat(leaderAndIsrOpt).isPresent();
int leader = leaderAndIsrOpt.get().leader();
TabletServer tabletServer = getTabletServerById(leader);
- SchemaMetadataManager schemaMetadataManager =
- tabletServer.getMetadataCache().getSchemaMetadataManager();
- Map<TablePath, SchemaInfo> latestSchemaByTablePath =
- schemaMetadataManager.getLatestSchemaByTablePath();
- assertThat(latestSchemaByTablePath).containsKey(tablePath);
- assertThat(latestSchemaByTablePath.get(tablePath).getSchemaId())
+ ServerSchemaCache serverSchemaCache =
+ tabletServer.getMetadataCache().getServerSchemaCache();
+ Map<Long, SchemaInfo> latestSchemaByTablePath =
+ serverSchemaCache.getLatestSchemaByTableId();
+ assertThat(latestSchemaByTablePath).containsKey(tableId);
+ assertThat(latestSchemaByTablePath.get(tableId).getSchemaId())
.isEqualTo(schemaId);
}
});
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java b/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java
index b87ed418e..f42d381fa 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java
@@ -387,7 +387,7 @@ class ZooKeeperClientTest {
.withComment("c is third column")
.primaryKey("a")
.build();
- int registeredSchemaId = zookeeperClient.registerSchema(tablePath, schema);
+ int registeredSchemaId = zookeeperClient.registerFirstSchema(tablePath, schema);
assertThat(registeredSchemaId).isEqualTo(schemaId);
assertThat(zookeeperClient.getCurrentSchemaId(tablePath)).isEqualTo(schemaId);
@@ -406,7 +406,7 @@ class ZooKeeperClientTest {
.withComment("b is second column")
.primaryKey("a")
.build();
- registeredSchemaId = zookeeperClient.registerSchema(tablePath, schema2);
+ registeredSchemaId = zookeeperClient.registerSchema(tablePath, schema2, 2);
assertThat(registeredSchemaId).isEqualTo(2);
assertThat(zookeeperClient.getCurrentSchemaId(tablePath)).isEqualTo(2);
diff --git a/fluss-server/src/test/resources/log4j2-test.properties b/fluss-server/src/test/resources/log4j2-test.properties
index 3db036fea..7acf6e146 100644
--- a/fluss-server/src/test/resources/log4j2-test.properties
+++ b/fluss-server/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level = INFO
+rootLogger.level = OFF
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml
index ca167c2d9..0c6f54cd5 100644
--- a/fluss-test-coverage/pom.xml
+++ b/fluss-test-coverage/pom.xml
@@ -164,7 +164,7 @@
<excludes>
<exclude>fluss-test-coverage/**</exclude>
<exclude>fluss-test-utils/**</exclude>
- <!-- exclude adapter classes to avoid Jacoco error: "Can'tadd different class with same name" -->
+ <!-- exclude adapter classes to avoid Jacoco error: "Can't add different class with same name" -->
<exclude>fluss-flink/**/target/classes/org/apache/fluss/flink/adapter/**</exclude>
</excludes>
</resource>
@@ -206,7 +206,7 @@
<excludes>
<exclude>fluss-test-coverage/**</exclude>
<exclude>fluss-test-utils/**</exclude>
- <!-- exclude adapter classes to avoid Jacoco error: "Can'tadd different class with same name" -->
+ <!-- exclude adapter classes to avoid Jacoco error: "Can't add different class with same name" -->
<exclude>fluss-flink/**/target/classes/org/apache/fluss/flink/adapter/**</exclude>
</excludes>
</resource>