You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2025/12/02 04:17:04 UTC
(fluss) 01/02: [server][client] Support Schema Evolution (ADD COLUMN LAST) in Fluss and connector (#2010)
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
commit ea2d61c76ca22ebb926618f7d06d4ced12c0f1d6
Author: Hongshun Wang <lo...@gmail.com>
AuthorDate: Wed Nov 26 19:32:11 2025 +0800
[server][client] Support Schema Evolution (ADD COLUMN LAST) in Fluss and connector (#2010)
---
.../org/apache/fluss/client/FlussConnection.java | 1 -
.../org/apache/fluss/client/admin/FlussAdmin.java | 8 +-
.../fluss/client/lookup/PrefixKeyLookuper.java | 55 ++--
.../fluss/client/lookup/PrimaryKeyLookuper.java | 54 ++--
.../apache/fluss/client/lookup/TableLookup.java | 18 +-
.../fluss/client/metadata/ClientSchemaGetter.java | 80 ++++++
.../org/apache/fluss/client/table/FlussTable.java | 14 +-
.../fluss/client/table/scanner/TableScan.java | 29 ++-
.../client/table/scanner/batch/BatchScanUtils.java | 3 +-
.../scanner/batch/KvSnapshotBatchScanner.java | 22 +-
.../table/scanner/batch/LimitBatchScanner.java | 33 ++-
.../table/scanner/batch/SnapshotFilesReader.java | 31 ++-
.../client/table/scanner/log/BucketScanStatus.java | 3 +-
.../fluss/client/table/scanner/log/LogFetcher.java | 14 +-
.../client/table/scanner/log/LogScannerImpl.java | 9 +-
.../client/table/writer/AbstractTableWriter.java | 2 +
.../fluss/client/utils/ClientRpcMessageUtils.java | 80 ++++++
.../apache/fluss/client/write/KvWriteBatch.java | 9 +
.../fluss/client/write/RecordAccumulator.java | 5 +-
.../java/org/apache/fluss/client/write/Sender.java | 6 +-
.../org/apache/fluss/client/write/WriteRecord.java | 6 +-
.../apache/fluss/client/write/WriterClient.java | 1 +
.../client/admin/ClientToServerITCaseBase.java | 4 +
.../fluss/client/admin/FlussAdminITCase.java | 86 ++++++-
.../fluss/client/lookup/LookupSenderTest.java | 3 +-
.../client/metadata/TestingMetadataUpdater.java | 4 +-
.../fluss/client/table/FlussTableITCase.java | 205 +++++++++++++++
.../fluss/client/table/PartitionedTableITCase.java | 2 +-
.../batch/KvSnapshotBatchScannerITCase.java | 83 +++++-
.../scanner/log/DefaultCompletedFetchTest.java | 21 +-
.../table/scanner/log/LogFetchBufferTest.java | 4 +-
.../table/scanner/log/LogFetchCollectorTest.java | 7 +-
.../client/table/scanner/log/LogFetcherTest.java | 130 +++++++++-
.../scanner/log/RemoteCompletedFetchTest.java | 20 +-
.../fluss/client/write/ArrowLogWriteBatchTest.java | 11 +-
.../client/write/IndexedLogWriteBatchTest.java | 11 +-
.../fluss/client/write/KvWriteBatchTest.java | 5 +-
.../fluss/client/write/RecordAccumulatorTest.java | 10 +-
.../cluster/ColumnPositionType.java} | 37 +--
.../fluss/exception/SchemaChangeException.java | 24 +-
.../java/org/apache/fluss/metadata/Schema.java | 97 +++++--
.../org/apache/fluss/metadata/SchemaGetter.java | 25 +-
.../org/apache/fluss/metadata/TableChange.java | 279 +++++++++++++++++++++
.../java/org/apache/fluss/metadata/TableInfo.java | 17 ++
.../apache/fluss/record/DefaultLogRecordBatch.java | 87 ++++++-
.../fluss/record/DefaultValueRecordBatch.java | 1 +
.../org/apache/fluss/record/FileLogProjection.java | 199 ++++++++++-----
.../org/apache/fluss/record/IndexedLogRecord.java | 2 +-
.../apache/fluss/record/KvRecordReadContext.java | 29 ++-
.../org/apache/fluss/record/LogRecordBatch.java | 16 ++
.../apache/fluss/record/LogRecordReadContext.java | 138 ++++++----
.../fluss/record/ValueRecordReadContext.java | 29 ++-
.../java/org/apache/fluss/row/ProjectedRow.java | 24 +-
.../org/apache/fluss/row/encode/ValueDecoder.java | 33 ++-
.../org/apache/fluss/row/encode/ValueEncoder.java | 23 ++
.../main/java/org/apache/fluss/types/RowType.java | 8 +-
.../java/org/apache/fluss/utils/Projection.java | 88 +++++--
.../java/org/apache/fluss/utils/SchemaUtil.java | 101 ++++++++
.../java/org/apache/fluss/utils/UnsafeUtils.java | 4 +
.../apache/fluss/utils/json/ColumnJsonSerde.java | 14 +-
.../apache/fluss/utils/json/SchemaJsonSerde.java | 7 +
.../fluss/record/DefaultKvRecordBatchTest.java | 3 +-
.../apache/fluss/record/DefaultKvRecordTest.java | 7 +-
.../fluss/record/DefaultLogRecordBatchTest.java | 21 +-
.../fluss/record/FileLogInputStreamTest.java | 5 +-
.../apache/fluss/record/FileLogProjectionTest.java | 209 +++++++++++----
.../org/apache/fluss/record/LogRecordsTest.java | 10 +-
.../java/org/apache/fluss/record/LogTestBase.java | 8 +-
.../record/MemoryLogRecordsArrowBuilderTest.java | 6 +-
.../java/org/apache/fluss/record/TestData.java | 3 +
.../apache/fluss/record/TestingSchemaGetter.java | 73 ++++++
.../org/apache/fluss/row/ProjectedRowTest.java | 68 +++++
.../org/apache/fluss/testutils/DataTestUtils.java | 94 +++++--
.../fluss/testutils/LogRecordBatchAssert.java | 11 +-
.../apache/fluss/testutils/LogRecordsAssert.java | 9 +
.../org/apache/fluss/utils/ProjectionTest.java | 14 +-
.../fluss/utils/json/ColumnJsonSerdeTest.java | 31 ++-
.../fluss/utils/json/SchemaJsonSerdeTest.java | 40 ++-
.../src/test/resources/log4j2-test.properties | 2 +-
.../apache/fluss/flink/row/FlinkAsFlussRow.java | 100 ++++++--
.../flink/sink/FlinkRowDataChannelComputer.java | 2 +
.../sink/serializer/FlussSerializationSchema.java | 9 +-
.../serializer/RowDataSerializationSchema.java | 2 +-
.../sink/serializer/SerializerInitContextImpl.java | 19 +-
.../fluss/flink/sink/writer/FlinkSinkWriter.java | 8 +-
.../org/apache/fluss/flink/source/FlinkSource.java | 7 +-
.../fluss/flink/source/FlinkTableSource.java | 1 -
.../org/apache/fluss/flink/source/FlussSource.java | 4 -
.../fluss/flink/source/FlussSourceBuilder.java | 1 -
.../deserializer/RowDataDeserializationSchema.java | 5 +-
.../source/lookup/FlinkAsyncLookupFunction.java | 20 +-
.../flink/source/lookup/FlinkLookupFunction.java | 18 +-
.../flink/source/reader/FlinkSourceReader.java | 4 -
.../source/reader/FlinkSourceSplitReader.java | 68 ++---
.../apache/fluss/flink/utils/FlinkConversions.java | 67 +++++
.../flink/utils/FlussRowToFlinkRowConverter.java | 39 ++-
.../fluss/flink/catalog/FlinkCatalogITCase.java | 21 +-
.../fluss/flink/row/FlinkAsFlussRowTest.java | 3 +-
.../fluss/flink/sink/FlinkTableSinkITCase.java | 152 +++++++++++
.../flink/sink/writer/FlinkSinkWriterTest.java | 52 +++-
.../FlussDeserializationSchemaTest.java | 13 +-
.../source/lookup/FlinkLookupFunctionTest.java | 123 ++++++++-
.../flink/source/reader/FlinkSourceReaderTest.java | 1 -
.../source/reader/FlinkSourceSplitReaderTest.java | 42 ++--
.../committer/TieringCommitOperatorTest.java | 4 +-
.../fluss/lake/iceberg/conf/HadoopUtils.java | 5 +-
.../iceberg/maintenance/IcebergRewriteITCase.java | 2 +-
.../fluss/rpc/netty/client/ServerConnection.java | 3 +-
fluss-rpc/src/main/proto/FlussApi.proto | 31 ++-
.../org/apache/fluss/server/RpcServiceBase.java | 2 +-
.../coordinator/CompletedSnapshotStoreManager.java | 2 +-
.../coordinator/CoordinatorEventProcessor.java | 45 ++++
.../coordinator/CoordinatorRequestBatch.java | 5 +-
.../server/coordinator/CoordinatorService.java | 43 +++-
.../fluss/server/coordinator/MetadataManager.java | 37 ++-
.../fluss/server/coordinator/SchemaUpdate.java | 119 +++++++++
.../fluss/server/coordinator/UpdateSchema.java | 26 +-
.../coordinator/event/SchemaChangeEvent.java | 62 +++++
.../event/watcher/TableChangeWatcher.java | 38 +++
.../server/entity/BatchRegisterLeadAndIsr.java | 2 +-
.../java/org/apache/fluss/server/kv/KvManager.java | 22 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 47 ++--
.../java/org/apache/fluss/server/kv/KvTablet.java | 140 ++++++++---
.../kv/partialupdate/PartialUpdaterCache.java | 8 +-
.../server/kv/rowmerger/DefaultRowMerger.java | 24 +-
.../server/kv/rowmerger/FirstRowRowMerger.java | 10 +-
.../fluss/server/kv/rowmerger/RowMerger.java | 15 +-
.../server/kv/rowmerger/VersionedRowMerger.java | 28 ++-
.../server/kv/snapshot/CompletedSnapshotStore.java | 4 +-
.../fluss/server/kv/snapshot/SnapshotLocation.java | 2 +-
.../org/apache/fluss/server/log/FetchParams.java | 9 +-
.../java/org/apache/fluss/server/log/LocalLog.java | 2 +
.../server/log/remote/FsRemoteLogOutputStream.java | 2 +-
.../fluss/server/log/remote/LogSegmentFiles.java | 2 +-
.../fluss/server/log/remote/LogTieringTask.java | 4 +-
.../server/metadata/SchemaMetadataManager.java | 193 ++++++++++++++
.../server/metadata/TabletServerMetadataCache.java | 30 +++
.../fluss/server/replica/AdjustIsrManager.java | 2 +-
.../org/apache/fluss/server/replica/Replica.java | 70 ++----
.../fluss/server/replica/ReplicaManager.java | 17 +-
.../replica/fetcher/ReplicaFetcherManager.java | 2 +-
.../replica/fetcher/ReplicaFetcherThread.java | 2 +-
.../fluss/server/utils/RpcGatewayManager.java | 2 +-
.../fluss/server/utils/ServerRpcMessageUtils.java | 84 ++++++-
.../org/apache/fluss/server/utils/timer/Timer.java | 2 +-
.../fluss/server/utils/timer/TimingWheel.java | 4 +-
.../apache/fluss/server/zk/ZooKeeperClient.java | 14 +-
.../org/apache/fluss/server/zk/data/ZkData.java | 28 +++
.../server/authorizer/DefaultAuthorizerTest.java | 4 +-
.../CompletedSnapshotStoreManagerTest.java | 8 +-
.../coordinator/CoordinatorChannelManagerTest.java | 6 +-
.../coordinator/CoordinatorEventProcessorTest.java | 108 +++++++-
.../server/coordinator/CoordinatorTestUtils.java | 23 +-
.../fluss/server/coordinator/TableManagerTest.java | 2 +
.../event/watcher/TableChangeWatcherTest.java | 92 ++++++-
.../org/apache/fluss/server/kv/KvManagerTest.java | 84 ++++++-
.../org/apache/fluss/server/kv/KvTabletTest.java | 273 ++++++++++++++++----
.../server/kv/prewrite/KvPreWriteBufferTest.java | 4 +-
.../server/kv/rowmerger/DefaultRowMergerTest.java | 56 ++++-
.../kv/rowmerger/VersionedRowMergerTest.java | 63 +++--
.../kv/snapshot/CompletedSnapshotStoreTest.java | 2 +-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 2 +-
.../ZooKeeperCompletedSnapshotStoreTest.java | 4 +-
.../fluss/server/log/DroppedTableRecoveryTest.java | 9 +-
.../apache/fluss/server/log/FetchParamsTest.java | 30 ++-
.../apache/fluss/server/log/LogManagerTest.java | 2 +-
.../apache/fluss/server/log/LogSegmentTest.java | 4 +-
.../org/apache/fluss/server/log/LogTabletTest.java | 11 +-
.../server/log/remote/RemoteLogTabletTest.java | 4 +-
.../server/metadata/MetadataUpdateITCase.java | 2 +-
.../server/metadata/SchemaMetadataManagerTest.java | 179 +++++++++++++
.../fluss/server/replica/AdjustIsrITCase.java | 2 +-
.../fluss/server/replica/AdjustIsrManagerTest.java | 2 +-
.../replica/HighWatermarkPersistenceTest.java | 2 +-
.../replica/RemoveOfflineReplicaFromIsrITCase.java | 2 +-
.../fluss/server/replica/ReplicaManagerTest.java | 72 ++++--
.../apache/fluss/server/replica/ReplicaTest.java | 134 +++++++++-
.../fluss/server/replica/ReplicaTestBase.java | 4 +-
.../replica/fetcher/ReplicaFetcherManagerTest.java | 2 +-
.../fluss/server/tablet/TabletServiceITCase.java | 13 +-
.../server/tablet/TestTabletServerGateway.java | 15 +-
.../server/testutils/FlussClusterExtension.java | 31 ++-
.../server/testutils/RpcMessageTestUtils.java | 37 ++-
.../fluss/server/zk/ZooKeeperClientTest.java | 5 +
.../apache/fluss/server/zk/data/ZkDataTest.java | 18 ++
.../src/test/resources/log4j2-test.properties | 2 +-
fluss-test-coverage/pom.xml | 4 +-
187 files changed, 5282 insertions(+), 996 deletions(-)
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
index f29d90ba9..7af5d6b89 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/FlussConnection.java
@@ -100,7 +100,6 @@ public final class FlussConnection implements Connection {
public Table getTable(TablePath tablePath) {
// force to update the table info from server to avoid stale data in cache.
metadataUpdater.updateTableOrPartitionMetadata(tablePath, null);
-
Admin admin = getOrCreateAdmin();
return new FlussTable(this, tablePath, admin.getTableInfo(tablePath).join());
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
index 57dc52008..25bfcfbd8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
@@ -90,6 +90,7 @@ import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
+import static org.apache.fluss.client.utils.ClientRpcMessageUtils.addPbAlterSchemas;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeCreatePartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeDropPartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeListOffsetsRequest;
@@ -249,12 +250,17 @@ public class FlussAdmin implements Admin {
TablePath tablePath, List<TableChange> tableChanges, boolean ignoreIfNotExists) {
tablePath.validate();
AlterTableRequest request = new AlterTableRequest();
-
List<PbAlterConfig> pbFlussTableChanges =
tableChanges.stream()
+ .filter(tableChange -> !(tableChange instanceof TableChange.SchemaChange))
.map(ClientRpcMessageUtils::toPbAlterConfigs)
.collect(Collectors.toList());
+ List<TableChange> schemaChanges =
+ tableChanges.stream()
+ .filter(tableChange -> tableChange instanceof TableChange.SchemaChange)
+ .collect(Collectors.toList());
+ addPbAlterSchemas(request, schemaChanges);
request.addAllConfigChanges(pbFlussTableChanges)
.setIgnoreIfNotExists(ignoreIfNotExists)
.setTablePath()
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
index 61645374f..c9f6308f8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
@@ -22,13 +22,14 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.decode.RowDecoder;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -70,8 +71,11 @@ class PrefixKeyLookuper implements Lookuper {
/** Decode the lookup bytes to result row. */
private final ValueDecoder kvValueDecoder;
+ private final SchemaGetter schemaGetter;
+
public PrefixKeyLookuper(
TableInfo tableInfo,
+ SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient,
List<String> lookupColumnNames) {
@@ -93,10 +97,8 @@ class PrefixKeyLookuper implements Lookuper {
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
this.kvValueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- tableInfo.getTableConfig().getKvFormat(),
- tableInfo.getRowType().getChildren().toArray(new DataType[0])));
+ new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
+ this.schemaGetter = schemaGetter;
}
private void validatePrefixLookup(TableInfo tableInfo, List<String> lookupColumns) {
@@ -167,18 +169,35 @@ class PrefixKeyLookuper implements Lookuper {
}
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- return lookupClient
- .prefixLookup(tableBucket, bucketKeyBytes)
- .thenApply(
- result -> {
- List<InternalRow> rowList = new ArrayList<>(result.size());
- for (byte[] valueBytes : result) {
- if (valueBytes == null) {
- continue;
- }
- rowList.add(kvValueDecoder.decodeValue(valueBytes).row);
+ CompletableFuture<LookupResult> future = new CompletableFuture<>();
+
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ List<byte[]> result =
+ lookupClient.prefixLookup(tableBucket, bucketKeyBytes).get();
+ List<InternalRow> rowList = new ArrayList<>(result.size());
+ for (byte[] valueBytes : result) {
+ if (valueBytes == null) {
+ continue;
+ }
+ ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
+ InternalRow row;
+ if (value.schemaId == tableInfo.getSchemaId()) {
+ row = value.row;
+ } else {
+ Schema schema = schemaGetter.getSchema(value.schemaId);
+ row =
+ ProjectedRow.from(schema, tableInfo.getSchema())
+ .replaceRow(value.row);
}
- return new LookupResult(rowList);
- });
+ rowList.add(row);
+ }
+ future.complete(new LookupResult(rowList));
+ } catch (Exception e) {
+ future.complete(new LookupResult(Collections.emptyList()));
+ }
+ });
+ return future;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
index 2945f1ab4..091a8a38e 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
@@ -22,13 +22,14 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.decode.RowDecoder;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -39,6 +40,7 @@ import java.util.concurrent.CompletableFuture;
import static org.apache.fluss.client.utils.ClientUtils.getPartitionId;
import static org.apache.fluss.utils.Preconditions.checkArgument;
+import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** An implementation of {@link Lookuper} that lookups by primary key. */
@NotThreadSafe
@@ -67,8 +69,13 @@ class PrimaryKeyLookuper implements Lookuper {
/** Decode the lookup bytes to result row. */
private final ValueDecoder kvValueDecoder;
+ private final SchemaGetter schemaGetter;
+
public PrimaryKeyLookuper(
- TableInfo tableInfo, MetadataUpdater metadataUpdater, LookupClient lookupClient) {
+ TableInfo tableInfo,
+ SchemaGetter schemaGetter,
+ MetadataUpdater metadataUpdater,
+ LookupClient lookupClient) {
checkArgument(
tableInfo.hasPrimaryKey(),
"Log table %s doesn't support lookup",
@@ -96,10 +103,8 @@ class PrimaryKeyLookuper implements Lookuper {
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
this.kvValueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- tableInfo.getTableConfig().getKvFormat(),
- tableInfo.getRowType().getChildren().toArray(new DataType[0])));
+ new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
+ this.schemaGetter = schemaGetter;
}
@Override
@@ -127,15 +132,30 @@ class PrimaryKeyLookuper implements Lookuper {
int bucketId = bucketingFunction.bucketing(bkBytes, numBuckets);
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- return lookupClient
- .lookup(tableBucket, pkBytes)
- .thenApply(
- valueBytes -> {
- InternalRow row =
- valueBytes == null
- ? null
- : kvValueDecoder.decodeValue(valueBytes).row;
- return new LookupResult(row);
- });
+ CompletableFuture<LookupResult> future = new CompletableFuture<>();
+
+ CompletableFuture.runAsync(
+ () -> {
+ try {
+ byte[] valueBytes = lookupClient.lookup(tableBucket, pkBytes).get();
+ InternalRow row = null;
+ if (valueBytes != null) {
+ ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
+ if (value.schemaId == tableInfo.getSchemaId()) {
+ row = value.row;
+ } else {
+ Schema schema = schemaGetter.getSchema(value.schemaId);
+ checkNotNull(schema, "schema is null");
+ row =
+ ProjectedRow.from(schema, tableInfo.getSchema())
+ .replaceRow(value.row);
+ }
+ }
+ future.complete(new LookupResult(row));
+ } catch (Exception e) {
+ future.completeExceptionally(e);
+ }
+ });
+ return future;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java
index 2212f123c..84c9bdb21 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/TableLookup.java
@@ -18,6 +18,7 @@
package org.apache.fluss.client.lookup;
import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableInfo;
import javax.annotation.Nullable;
@@ -28,22 +29,28 @@ import java.util.List;
public class TableLookup implements Lookup {
private final TableInfo tableInfo;
+ private final SchemaGetter schemaGetter;
private final MetadataUpdater metadataUpdater;
private final LookupClient lookupClient;
@Nullable private final List<String> lookupColumnNames;
public TableLookup(
- TableInfo tableInfo, MetadataUpdater metadataUpdater, LookupClient lookupClient) {
- this(tableInfo, metadataUpdater, lookupClient, null);
+ TableInfo tableInfo,
+ SchemaGetter schemaGetter,
+ MetadataUpdater metadataUpdater,
+ LookupClient lookupClient) {
+ this(tableInfo, schemaGetter, metadataUpdater, lookupClient, null);
}
private TableLookup(
TableInfo tableInfo,
+ SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient,
@Nullable List<String> lookupColumnNames) {
this.tableInfo = tableInfo;
+ this.schemaGetter = schemaGetter;
this.metadataUpdater = metadataUpdater;
this.lookupClient = lookupClient;
this.lookupColumnNames = lookupColumnNames;
@@ -51,16 +58,17 @@ public class TableLookup implements Lookup {
@Override
public Lookup lookupBy(List<String> lookupColumnNames) {
- return new TableLookup(tableInfo, metadataUpdater, lookupClient, lookupColumnNames);
+ return new TableLookup(
+ tableInfo, schemaGetter, metadataUpdater, lookupClient, lookupColumnNames);
}
@Override
public Lookuper createLookuper() {
if (lookupColumnNames == null) {
- return new PrimaryKeyLookuper(tableInfo, metadataUpdater, lookupClient);
+ return new PrimaryKeyLookuper(tableInfo, schemaGetter, metadataUpdater, lookupClient);
} else {
return new PrefixKeyLookuper(
- tableInfo, metadataUpdater, lookupClient, lookupColumnNames);
+ tableInfo, schemaGetter, metadataUpdater, lookupClient, lookupColumnNames);
}
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
new file mode 100644
index 000000000..181767781
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.metadata;
+
+import org.apache.fluss.annotation.Internal;
+import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TablePath;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.fluss.utils.MapUtils.newConcurrentHashMap;
+
+/** Schema getter for client. */
+@Internal
+public class ClientSchemaGetter implements SchemaGetter {
+ private static final Logger LOG = LoggerFactory.getLogger(ClientSchemaGetter.class);
+
+ private final TablePath tablePath;
+ private final Map<Integer, Schema> schemasById;
+ private final Admin admin;
+ private SchemaInfo latestSchemaInfo;
+
+ public ClientSchemaGetter(TablePath tablePath, SchemaInfo latestSchemaInfo, Admin admin) {
+ this.tablePath = tablePath;
+ this.latestSchemaInfo = latestSchemaInfo;
+ this.admin = admin;
+ this.schemasById = newConcurrentHashMap();
+ schemasById.put(latestSchemaInfo.getSchemaId(), latestSchemaInfo.getSchema());
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ return schemasById.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ try {
+ SchemaInfo schemaInfo =
+ admin.getTableSchema(tablePath, schemaId).get(1, TimeUnit.MINUTES);
+ if (id > latestSchemaInfo.getSchemaId()) {
+ latestSchemaInfo = schemaInfo;
+ }
+ return schemaInfo.getSchema();
+
+ } catch (Exception e) {
+ LOG.warn("Failed to get schema for table: " + tablePath);
+ throw new RuntimeException(e);
+ }
+ });
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ return latestSchemaInfo;
+ }
+
+ @Override
+ public void release() {}
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
index 6a982a699..629abe6ab 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
@@ -21,12 +21,15 @@ import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.client.FlussConnection;
import org.apache.fluss.client.lookup.Lookup;
import org.apache.fluss.client.lookup.TableLookup;
+import org.apache.fluss.client.metadata.ClientSchemaGetter;
import org.apache.fluss.client.table.scanner.Scan;
import org.apache.fluss.client.table.scanner.TableScan;
import org.apache.fluss.client.table.writer.Append;
import org.apache.fluss.client.table.writer.TableAppend;
import org.apache.fluss.client.table.writer.TableUpsert;
import org.apache.fluss.client.table.writer.Upsert;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -44,12 +47,18 @@ public class FlussTable implements Table {
private final TablePath tablePath;
private final TableInfo tableInfo;
private final boolean hasPrimaryKey;
+ private final SchemaGetter schemaGetter;
public FlussTable(FlussConnection conn, TablePath tablePath, TableInfo tableInfo) {
this.conn = conn;
this.tablePath = tablePath;
this.tableInfo = tableInfo;
this.hasPrimaryKey = tableInfo.hasPrimaryKey();
+ this.schemaGetter =
+ new ClientSchemaGetter(
+ tableInfo.getTablePath(),
+ new SchemaInfo(tableInfo.getSchema(), tableInfo.getSchemaId()),
+ conn.getAdmin());
}
@Override
@@ -59,13 +68,13 @@ public class FlussTable implements Table {
@Override
public Scan newScan() {
- return new TableScan(conn, tableInfo);
+ return new TableScan(conn, tableInfo, schemaGetter);
}
@Override
public Lookup newLookup() {
return new TableLookup(
- tableInfo, conn.getMetadataUpdater(), conn.getOrCreateLookupClient());
+ tableInfo, schemaGetter, conn.getMetadataUpdater(), conn.getOrCreateLookupClient());
}
@Override
@@ -89,5 +98,6 @@ public class FlussTable implements Table {
@Override
public void close() throws Exception {
// do nothing
+ schemaGetter.release();
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java
index 7fade0547..0ca1952cf 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/TableScan.java
@@ -27,6 +27,7 @@ import org.apache.fluss.client.table.scanner.log.LogScanner;
import org.apache.fluss.client.table.scanner.log.LogScannerImpl;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.exception.FlussRuntimeException;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.types.RowType;
@@ -40,30 +41,33 @@ public class TableScan implements Scan {
private final FlussConnection conn;
private final TableInfo tableInfo;
+ private final SchemaGetter schemaGetter;
/** The projected fields to do projection. No projection if is null. */
@Nullable private final int[] projectedColumns;
/** The limited row number to read. No limit if is null. */
@Nullable private final Integer limit;
- public TableScan(FlussConnection conn, TableInfo tableInfo) {
- this(conn, tableInfo, null, null);
+ public TableScan(FlussConnection conn, TableInfo tableInfo, SchemaGetter schemaGetter) {
+ this(conn, tableInfo, schemaGetter, null, null);
}
private TableScan(
FlussConnection conn,
TableInfo tableInfo,
+ SchemaGetter schemaGetter,
@Nullable int[] projectedColumns,
@Nullable Integer limit) {
this.conn = conn;
this.tableInfo = tableInfo;
this.projectedColumns = projectedColumns;
this.limit = limit;
+ this.schemaGetter = schemaGetter;
}
@Override
public Scan project(@Nullable int[] projectedColumns) {
- return new TableScan(conn, tableInfo, projectedColumns, limit);
+ return new TableScan(conn, tableInfo, schemaGetter, projectedColumns, limit);
}
@Override
@@ -82,12 +86,12 @@ public class TableScan implements Scan {
}
columnIndexes[i] = index;
}
- return new TableScan(conn, tableInfo, columnIndexes, limit);
+ return new TableScan(conn, tableInfo, schemaGetter, columnIndexes, limit);
}
@Override
public Scan limit(int rowNumber) {
- return new TableScan(conn, tableInfo, projectedColumns, rowNumber);
+ return new TableScan(conn, tableInfo, schemaGetter, projectedColumns, rowNumber);
}
@Override
@@ -98,13 +102,15 @@ public class TableScan implements Scan {
"LogScanner doesn't support limit pushdown. Table: %s, requested limit: %d",
tableInfo.getTablePath(), limit));
}
+
return new LogScannerImpl(
conn.getConfiguration(),
tableInfo,
conn.getMetadataUpdater(),
conn.getClientMetricGroup(),
conn.getOrCreateRemoteFileDownloader(),
- projectedColumns);
+ projectedColumns,
+ schemaGetter);
}
@Override
@@ -116,7 +122,12 @@ public class TableScan implements Scan {
tableInfo.getTablePath(), tableBucket));
}
return new LimitBatchScanner(
- tableInfo, tableBucket, conn.getMetadataUpdater(), projectedColumns, limit);
+ tableInfo,
+ tableBucket,
+ schemaGetter,
+ conn.getMetadataUpdater(),
+ projectedColumns,
+ limit);
}
@Override
@@ -142,7 +153,9 @@ public class TableScan implements Scan {
}
return new KvSnapshotBatchScanner(
- tableInfo.getRowType(),
+ tableInfo.getSchemaId(),
+ tableInfo.getSchema(),
+ schemaGetter,
tableBucket,
snapshotMeta.getSnapshotFiles(),
projectedColumns,
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
index d36bf9a7d..f4e150f30 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
@@ -71,7 +71,8 @@ public class BatchScanUtils {
// If the scanner has more data, add it back to the queue
scannerQueue.add(scanner);
} else {
- // Close the scanner if it has no more data, and not add it back to the queue
+ // Close the scanner if it has no more data, and not add it back to the
+ // queue
scanner.close();
}
} catch (Exception e) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java
index 2c749c662..52a2c36fb 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScanner.java
@@ -22,9 +22,10 @@ import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.fs.FsPathAndFileName;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.FileUtils;
@@ -72,7 +73,10 @@ public class KvSnapshotBatchScanner implements BatchScanner {
public static final CloseableIterator<InternalRow> NO_DATA_AVAILABLE =
CloseableIterator.emptyIterator();
- private final RowType tableRowType;
+ private final Schema targetSchema;
+ private final int targetSchemaId;
+ private final SchemaGetter schemaGetter;
+
private final TableBucket tableBucket;
private final List<FsPathAndFileName> fsPathAndFileNames;
@Nullable private final int[] projectedFields;
@@ -93,14 +97,18 @@ public class KvSnapshotBatchScanner implements BatchScanner {
@Nullable private volatile Throwable initSnapshotFilesReaderException = null;
public KvSnapshotBatchScanner(
- RowType tableRowType,
+ int targetSchemaId,
+ Schema targetSchema,
+ SchemaGetter schemaGetter,
TableBucket tableBucket,
List<FsPathAndFileName> fsPathAndFileNames,
@Nullable int[] projectedFields,
String scannerTmpDir,
KvFormat kvFormat,
RemoteFileDownloader remoteFileDownloader) {
- this.tableRowType = tableRowType;
+ this.targetSchema = targetSchema;
+ this.targetSchemaId = targetSchemaId;
+ this.schemaGetter = schemaGetter;
this.tableBucket = tableBucket;
this.fsPathAndFileNames = fsPathAndFileNames;
this.projectedFields = projectedFields;
@@ -209,8 +217,10 @@ public class KvSnapshotBatchScanner implements BatchScanner {
new SnapshotFilesReader(
kvFormat,
snapshotLocalDirectory,
- tableRowType,
- projectedFields);
+ projectedFields,
+ targetSchemaId,
+ targetSchema,
+ schemaGetter);
readerIsReady.signalAll();
} catch (Throwable e) {
IOUtils.closeQuietly(closeableRegistry);
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
index 4915ab918..e6208f368 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
@@ -19,6 +19,8 @@ package org.apache.fluss.client.table.scanner.batch;
import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.exception.LeaderNotAvailableException;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.DefaultValueRecordBatch;
@@ -32,12 +34,9 @@ import org.apache.fluss.record.ValueRecordReadContext;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.ProjectedRow;
-import org.apache.fluss.row.decode.RowDecoder;
-import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
import org.apache.fluss.rpc.messages.LimitScanRequest;
import org.apache.fluss.rpc.messages.LimitScanResponse;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
@@ -61,20 +60,24 @@ public class LimitBatchScanner implements BatchScanner {
private final int limit;
private final InternalRow.FieldGetter[] fieldGetters;
private final CompletableFuture<LimitScanResponse> scanFuture;
- private final ValueDecoder kvValueDecoder;
+ private final SchemaGetter schemaGetter;
+ private final KvFormat kvFormat;
+ private final int targetSchemaId;
private boolean endOfInput;
public LimitBatchScanner(
TableInfo tableInfo,
TableBucket tableBucket,
+ SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
@Nullable int[] projectedFields,
int limit) {
this.tableInfo = tableInfo;
this.projectedFields = projectedFields;
this.limit = limit;
-
+ this.targetSchemaId = tableInfo.getSchemaId();
+ this.schemaGetter = schemaGetter;
RowType rowType = tableInfo.getRowType();
this.fieldGetters = new InternalRow.FieldGetter[rowType.getFieldCount()];
for (int i = 0; i < rowType.getFieldCount(); i++) {
@@ -102,11 +105,7 @@ public class LimitBatchScanner implements BatchScanner {
}
this.scanFuture = gateway.limitScan(limitScanRequest);
- this.kvValueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- tableInfo.getTableConfig().getKvFormat(),
- rowType.getChildren().toArray(new DataType[0])));
+ this.kvFormat = tableInfo.getTableConfig().getKvFormat();
this.endOfInput = false;
}
@@ -139,13 +138,21 @@ public class LimitBatchScanner implements BatchScanner {
DefaultValueRecordBatch valueRecords =
DefaultValueRecordBatch.pointToByteBuffer(recordsBuffer);
ValueRecordReadContext readContext =
- new ValueRecordReadContext(kvValueDecoder.getRowDecoder());
+ ValueRecordReadContext.createReadContext(schemaGetter, kvFormat);
for (ValueRecord record : valueRecords.records(readContext)) {
- scanRows.add(maybeProject(record.getRow()));
+ InternalRow row = record.getRow();
+ if (targetSchemaId != record.schemaId()) {
+ row =
+ ProjectedRow.from(
+ schemaGetter.getSchema(record.schemaId()),
+ schemaGetter.getSchema(targetSchemaId))
+ .replaceRow(row);
+ }
+ scanRows.add(maybeProject(row));
}
} else {
LogRecordReadContext readContext =
- LogRecordReadContext.createReadContext(tableInfo, false, null);
+ LogRecordReadContext.createReadContext(tableInfo, false, null, schemaGetter);
LogRecords records = MemoryLogRecords.pointToByteBuffer(recordsBuffer);
for (LogRecordBatch logRecordBatch : records.batches()) {
// A batch of log record maybe little more than limit, thus we need slice the
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
index 3c95ab39f..10e773609 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
@@ -20,14 +20,13 @@ package org.apache.fluss.client.table.scanner.batch;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.rocksdb.RocksDBHandle;
import org.apache.fluss.rocksdb.RocksIteratorWrapper;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.ProjectedRow;
-import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.types.DataType;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.IOUtils;
@@ -52,6 +51,9 @@ import java.nio.file.Path;
@NotThreadSafe
class SnapshotFilesReader implements CloseableIterator<InternalRow> {
+ private final int targetSchemaId;
+ private final Schema targetSchema;
+ private final SchemaGetter schemaGetter;
private final ValueDecoder valueDecoder;
@Nullable private final int[] projectedFields;
private RocksIteratorWrapper rocksIteratorWrapper;
@@ -65,13 +67,15 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
SnapshotFilesReader(
KvFormat kvFormat,
Path rocksDbPath,
- RowType tableRowType,
- @Nullable int[] projectedFields)
+ @Nullable int[] projectedFields,
+ int targetSchemaId,
+ Schema targetSchema,
+ SchemaGetter schemaGetter)
throws IOException {
- this.valueDecoder =
- new ValueDecoder(
- RowDecoder.create(
- kvFormat, tableRowType.getChildren().toArray(new DataType[0])));
+ this.targetSchemaId = targetSchemaId;
+ this.targetSchema = targetSchema;
+ this.schemaGetter = schemaGetter;
+ this.valueDecoder = new ValueDecoder(schemaGetter, kvFormat);
this.projectedFields = projectedFields;
closeableRegistry = new CloseableRegistry();
try {
@@ -151,7 +155,14 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
byte[] value = rocksIteratorWrapper.value();
rocksIteratorWrapper.next();
- InternalRow originRow = valueDecoder.decodeValue(value).row;
+ ValueDecoder.Value originValue = valueDecoder.decodeValue(value);
+ InternalRow originRow = originValue.row;
+ if (targetSchemaId != originValue.schemaId) {
+ originRow =
+ ProjectedRow.from(schemaGetter.getSchema(originValue.schemaId), targetSchema)
+ .replaceRow(originRow);
+ }
+
if (projectedFields != null) {
ProjectedRow projectedRow = ProjectedRow.from(projectedFields);
projectedRow.replaceRow(originRow);
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
index 738e04a49..bee0910e1 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
@@ -24,7 +24,8 @@ import org.apache.fluss.annotation.Internal;
class BucketScanStatus {
private long offset; // last consumed position
private long highWatermark; // the high watermark from last fetch
- // TODO add resetStrategy and nextAllowedRetryTimeMs.
+ // TODO add
+ // resetStrategy and nextAllowedRetryTimeMs.
public BucketScanStatus() {
this.offset = 0L;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
index f2df66aaa..e8067dd4c 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
@@ -30,6 +30,7 @@ import org.apache.fluss.exception.InvalidMetadataException;
import org.apache.fluss.exception.LeaderNotAvailableException;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePartition;
@@ -115,12 +116,14 @@ public class LogFetcher implements Closeable {
Configuration conf,
MetadataUpdater metadataUpdater,
ScannerMetricGroup scannerMetricGroup,
- RemoteFileDownloader remoteFileDownloader) {
+ RemoteFileDownloader remoteFileDownloader,
+ SchemaGetter schemaGetter) {
this.tablePath = tableInfo.getTablePath();
this.isPartitioned = tableInfo.isPartitioned();
- this.readContext = LogRecordReadContext.createReadContext(tableInfo, false, projection);
+ this.readContext =
+ LogRecordReadContext.createReadContext(tableInfo, false, projection, schemaGetter);
this.remoteReadContext =
- LogRecordReadContext.createReadContext(tableInfo, true, projection);
+ LogRecordReadContext.createReadContext(tableInfo, true, projection, schemaGetter);
this.projection = projection;
this.logScannerStatus = logScannerStatus;
this.maxFetchBytes =
@@ -360,7 +363,8 @@ public class LogFetcher implements Closeable {
LogRecords logRecords = fetchResultForBucket.recordsOrEmpty();
if (!MemoryLogRecords.EMPTY.equals(logRecords)
|| fetchResultForBucket.getErrorCode() != Errors.NONE.code()) {
- // In oder to not signal notEmptyCondition, add completed fetch to
+ // In oder to not signal notEmptyCondition, add completed
+ // fetch to
// buffer until log records is not empty.
DefaultCompletedFetch completedFetch =
new DefaultCompletedFetch(
@@ -479,7 +483,7 @@ public class LogFetcher implements Closeable {
assert projection != null;
reqForTable
.setProjectionPushdownEnabled(true)
- .setProjectedFields(projection.getProjectionInOrder());
+ .setProjectedFields(projection.getProjectionIdInOrder());
} else {
reqForTable.setProjectionPushdownEnabled(false);
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
index cef05808f..b50bc3622 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogScannerImpl.java
@@ -24,6 +24,7 @@ import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.WakeupException;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -82,7 +83,8 @@ public class LogScannerImpl implements LogScanner {
MetadataUpdater metadataUpdater,
ClientMetricGroup clientMetricGroup,
RemoteFileDownloader remoteFileDownloader,
- @Nullable int[] projectedFields) {
+ @Nullable int[] projectedFields,
+ SchemaGetter schemaGetter) {
this.tablePath = tableInfo.getTablePath();
this.tableId = tableInfo.getTableId();
this.isPartitionedTable = tableInfo.isPartitioned();
@@ -100,7 +102,8 @@ public class LogScannerImpl implements LogScanner {
conf,
metadataUpdater,
scannerMetricGroup,
- remoteFileDownloader);
+ remoteFileDownloader,
+ schemaGetter);
}
/**
@@ -120,7 +123,7 @@ public class LogScannerImpl implements LogScanner {
+ tableRowType);
}
}
- return Projection.of(projectedFields);
+ return Projection.of(projectedFields, tableInfo.getSchema());
} else {
return null;
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
index 98a93fdd2..f1811dfa2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
@@ -37,11 +37,13 @@ public abstract class AbstractTableWriter implements TableWriter {
protected final TablePath tablePath;
protected final WriterClient writerClient;
protected final int fieldCount;
+ protected TableInfo tableInfo;
private final @Nullable PartitionGetter partitionFieldGetter;
protected AbstractTableWriter(
TablePath tablePath, TableInfo tableInfo, WriterClient writerClient) {
this.tablePath = tablePath;
+ this.tableInfo = tableInfo;
this.writerClient = writerClient;
this.fieldCount = tableInfo.getRowType().getFieldCount();
this.partitionFieldGetter =
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
index b98b6fb6c..8b36b5cf8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
@@ -26,6 +26,7 @@ import org.apache.fluss.client.metadata.LakeSnapshot;
import org.apache.fluss.client.write.KvWriteBatch;
import org.apache.fluss.client.write.ReadyWriteBatch;
import org.apache.fluss.config.cluster.AlterConfigOpType;
+import org.apache.fluss.config.cluster.ColumnPositionType;
import org.apache.fluss.config.cluster.ConfigEntry;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.FsPathAndFileName;
@@ -36,6 +37,7 @@ import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.rpc.messages.AlterTableRequest;
import org.apache.fluss.rpc.messages.CreatePartitionRequest;
import org.apache.fluss.rpc.messages.DropPartitionRequest;
import org.apache.fluss.rpc.messages.GetFileSystemSecurityTokenResponse;
@@ -46,20 +48,26 @@ import org.apache.fluss.rpc.messages.ListOffsetsRequest;
import org.apache.fluss.rpc.messages.ListPartitionInfosResponse;
import org.apache.fluss.rpc.messages.LookupRequest;
import org.apache.fluss.rpc.messages.MetadataRequest;
+import org.apache.fluss.rpc.messages.PbAddColumn;
import org.apache.fluss.rpc.messages.PbAlterConfig;
import org.apache.fluss.rpc.messages.PbDescribeConfig;
+import org.apache.fluss.rpc.messages.PbDropColumn;
import org.apache.fluss.rpc.messages.PbKeyValue;
import org.apache.fluss.rpc.messages.PbKvSnapshot;
import org.apache.fluss.rpc.messages.PbLakeSnapshotForBucket;
import org.apache.fluss.rpc.messages.PbLookupReqForBucket;
+import org.apache.fluss.rpc.messages.PbModifyColumn;
import org.apache.fluss.rpc.messages.PbPartitionSpec;
import org.apache.fluss.rpc.messages.PbPrefixLookupReqForBucket;
import org.apache.fluss.rpc.messages.PbProduceLogReqForBucket;
import org.apache.fluss.rpc.messages.PbPutKvReqForBucket;
import org.apache.fluss.rpc.messages.PbRemotePathAndLocalFile;
+import org.apache.fluss.rpc.messages.PbRenameColumn;
import org.apache.fluss.rpc.messages.PrefixLookupRequest;
import org.apache.fluss.rpc.messages.ProduceLogRequest;
import org.apache.fluss.rpc.messages.PutKvRequest;
+import org.apache.fluss.utils.json.DataTypeJsonSerde;
+import org.apache.fluss.utils.json.JsonSerdeUtils;
import javax.annotation.Nullable;
@@ -372,6 +380,78 @@ public class ClientRpcMessageUtils {
return info;
}
+ public static void addPbAlterSchemas(
+ AlterTableRequest request, List<TableChange> tableChanges) {
+ List<PbAddColumn> addColumns = new ArrayList<>();
+ List<PbDropColumn> dropColumns = new ArrayList<>();
+ List<PbRenameColumn> renameColumns = new ArrayList<>();
+ List<PbModifyColumn> modifyColumns = new ArrayList<>();
+ for (TableChange tableChange : tableChanges) {
+ if (tableChange instanceof TableChange.AddColumn) {
+ addColumns.add(toPbAddColumn((TableChange.AddColumn) tableChange));
+ } else if (tableChange instanceof TableChange.DropColumn) {
+ dropColumns.add(toPbDropColumn((TableChange.DropColumn) tableChange));
+ } else if (tableChange instanceof TableChange.RenameColumn) {
+ renameColumns.add(toPbRenameColumn((TableChange.RenameColumn) tableChange));
+ } else if (tableChange instanceof TableChange.ModifyColumn) {
+ modifyColumns.add(toPbModifyColumn((TableChange.ModifyColumn) tableChange));
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported table change: " + tableChange.getClass());
+ }
+ }
+
+ request.addAllAddColumns(addColumns)
+ .addAllDropColumns(dropColumns)
+ .addAllRenameColumns(renameColumns)
+ .addAllModifyColumns(modifyColumns);
+ }
+
+ public static PbAddColumn toPbAddColumn(TableChange.AddColumn addColumn) {
+ ColumnPositionType columnPositionType = ColumnPositionType.from(addColumn.getPosition());
+
+ PbAddColumn pbAddColumn =
+ new PbAddColumn()
+ .setColumnName(addColumn.getName())
+ .setDataTypeJson(
+ JsonSerdeUtils.writeValueAsBytes(
+ addColumn.getDataType(), DataTypeJsonSerde.INSTANCE))
+ .setColumnPositionType(columnPositionType.value());
+ if (addColumn.getComment() != null) {
+ pbAddColumn.setComment(addColumn.getComment());
+ }
+
+ return pbAddColumn;
+ }
+
+ public static PbDropColumn toPbDropColumn(TableChange.DropColumn dropColumn) {
+ return new PbDropColumn().setColumnName(dropColumn.getName());
+ }
+
+ public static PbRenameColumn toPbRenameColumn(TableChange.RenameColumn dropColumn) {
+ return new PbRenameColumn()
+ .setOldColumnName(dropColumn.getOldColumnName())
+ .setNewColumnName(dropColumn.getNewColumnName());
+ }
+
+ public static PbModifyColumn toPbModifyColumn(TableChange.ModifyColumn modifyColumn) {
+ PbModifyColumn pbModifyColumn =
+ new PbModifyColumn()
+ .setColumnName(modifyColumn.getName())
+ .setDataTypeJson(
+ JsonSerdeUtils.writeValueAsBytes(
+ modifyColumn.getDataType(), DataTypeJsonSerde.INSTANCE));
+ if (modifyColumn.getNewPosition() != null) {
+ ColumnPositionType columnPositionType =
+ ColumnPositionType.from(modifyColumn.getNewPosition());
+ pbModifyColumn.setColumnPositionType(columnPositionType.value());
+ }
+ if (modifyColumn.getComment() != null) {
+ pbModifyColumn.setComment(modifyColumn.getComment());
+ }
+ return pbModifyColumn;
+ }
+
public static List<ConfigEntry> toConfigEntries(List<PbDescribeConfig> pbDescribeConfigs) {
return pbDescribeConfigs.stream()
.map(
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
index 963e9866b..72f596be9 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/KvWriteBatch.java
@@ -50,6 +50,7 @@ public class KvWriteBatch extends WriteBatch {
private final AbstractPagedOutputView outputView;
private final KvRecordBatchBuilder recordsBuilder;
private final @Nullable int[] targetColumns;
+ private final int schemaId;
public KvWriteBatch(
int bucketId,
@@ -65,6 +66,7 @@ public class KvWriteBatch extends WriteBatch {
this.recordsBuilder =
KvRecordBatchBuilder.builder(schemaId, writeLimit, outputView, kvFormat);
this.targetColumns = targetColumns;
+ this.schemaId = schemaId;
}
@Override
@@ -74,6 +76,13 @@ public class KvWriteBatch extends WriteBatch {
@Override
public boolean tryAppend(WriteRecord writeRecord, WriteCallback callback) throws Exception {
+ if (schemaId != writeRecord.getSchemaId()) {
+ throw new IllegalStateException(
+ String.format(
+ "schema id %d of the write record to append is not the same as the current schema id %d in the batch.",
+ writeRecord.getSchemaId(), schemaId));
+ }
+
// currently, we throw exception directly when the target columns of the write record is
// not the same as the current target columns in the batch.
// this should be quite fast as they should be the same objects.
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
index 9dd8bade6..f7d7d3ef0 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
@@ -867,7 +867,8 @@ public final class RecordAccumulator {
orderedBatches.size());
// Either we have reached a point where there are batches without a sequence (i.e. never
// been drained and are hence in order by default), or the batch at the front of the
- // queue has a sequence greater than the incoming batch. This is the right place to add
+ // queue has a sequence greater than the incoming batch. This is the right place to
+ // add
// the incoming batch.
deque.addFirst(batch);
@@ -920,6 +921,8 @@ public final class RecordAccumulator {
writerBufferPool.close();
arrowWriterPool.close();
+ // Release all the memory segments.
+ bufferAllocator.releaseBytes(bufferAllocator.getAllocatedMemory());
bufferAllocator.close();
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
index e9c285306..1b9c79397 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
@@ -130,7 +130,7 @@ public class Sender implements Runnable {
this.idempotenceManager = idempotenceManager;
this.writerMetricGroup = writerMetricGroup;
- // TODO add retry logic while send failed. See FLUSS-56364375
+ // TODOadd retry logic while send failed. See FLUSS-56364375
}
@VisibleForTesting
@@ -167,7 +167,7 @@ public class Sender implements Runnable {
}
}
- // TODO if force close failed, add logic to abort incomplete batches.
+ // TODO if force close failed,add logic to abort incomplete batches.
LOG.debug("Shutdown of Fluss write sender I/O thread has completed.");
}
@@ -245,7 +245,7 @@ public class Sender implements Runnable {
if (!batches.isEmpty()) {
addToInflightBatches(batches);
- // TODO add logic for batch expire.
+ // TODOadd logic for batch expire.
sendWriteRequests(batches);
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java
index e1ab2ec24..68a3fe8a2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriteRecord.java
@@ -50,7 +50,7 @@ public final class WriteRecord {
@Nullable int[] targetColumns) {
checkNotNull(row, "row must not be null");
checkNotNull(key, "key must not be null");
- checkNotNull(bucketKey, "key must not be null");
+ checkNotNull(bucketKey, "bucketKey must not be null");
int estimatedSizeInBytes =
DefaultKvRecord.sizeOf(key, row) + DefaultKvRecordBatch.RECORD_BATCH_HEADER_SIZE;
return new WriteRecord(
@@ -205,4 +205,8 @@ public final class WriteRecord {
}
return estimatedSizeInBytes;
}
+
+ public int getSchemaId() {
+ return tableInfo.getSchemaId();
+ }
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
index f61e6f544..788d385a1 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/WriterClient.java
@@ -120,6 +120,7 @@ public class WriterClient {
conf.get(ConfigOptions.CLIENT_WRITER_DYNAMIC_CREATE_PARTITION_ENABLED),
this::maybeAbortBatches);
} catch (Throwable t) {
+ LOG.error("Failed to construct writer.", t);
close(Duration.ofMillis(0));
throw new FlussRuntimeException(
String.format(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java
index 57cdd1c8b..a62c1b70d 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/ClientToServerITCaseBase.java
@@ -240,6 +240,10 @@ public abstract class ClientToServerITCaseBase {
}
}
+ public static void waitAllSchemaSync(TablePath tablePath, int schemaId) {
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(tablePath, schemaId);
+ }
+
protected static void verifyRows(
RowType rowType,
Map<Long, List<InternalRow>> actualRows,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
index 10890f947..05f196fdd 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
@@ -228,7 +228,7 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
}
@Test
- void testAlterTable() throws Exception {
+ void testAlterTableConfig() throws Exception {
// create table
TablePath tablePath = TablePath.of("test_db", "alter_table_1");
admin.createTable(tablePath, DEFAULT_TABLE_DESCRIPTOR, false).get();
@@ -302,6 +302,90 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
.get();
}
+ @Test
+ void testAlterTableColumn() throws Exception {
+ // create table
+ TablePath tablePath = TablePath.of("test_db", "alter_table_1");
+ admin.createTable(tablePath, DEFAULT_TABLE_DESCRIPTOR, false).get();
+
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.dropColumn("id")),
+ false)
+ .get())
+ .hasMessageContaining("Not support drop column now.");
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.renameColumn("id", "id2")),
+ false)
+ .get())
+ .hasMessageContaining("Not support rename column now.");
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.modifyColumn(
+ "id",
+ DataTypes.INT(),
+ "person id",
+ null)),
+ false)
+ .get())
+ .hasMessageContaining("Not support modify column now.");
+
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING().copy(false),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get())
+ .hasMessageContaining("Column c1 must be nullable");
+
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+
+ Schema expectedSchema =
+ Schema.newBuilder()
+ .primaryKey("id")
+ .fromColumns(
+ Arrays.asList(
+ new Schema.Column(
+ "id", DataTypes.INT(), "person id", (short) 0),
+ new Schema.Column(
+ "name",
+ DataTypes.STRING(),
+ "person name",
+ (short) 1),
+ new Schema.Column(
+ "age", DataTypes.INT(), "person age", (short) 2),
+ new Schema.Column(
+ "c1", DataTypes.STRING(), null, (short) 3)))
+ .build();
+ SchemaInfo schemaInfo = admin.getTableSchema(tablePath).get();
+ assertThat(schemaInfo).isEqualTo(new SchemaInfo(expectedSchema, 2));
+ }
+
@Test
void testCreateInvalidDatabaseAndTable() throws Exception {
assertThatThrownBy(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java b/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java
index 61dc5cbc8..f56535b01 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/lookup/LookupSenderTest.java
@@ -44,6 +44,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Duration;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -478,7 +479,7 @@ public class LookupSenderTest {
prefixLookupHandler;
public ConfigurableTestTabletServerGateway() {
- super(false);
+ super(false, Collections.emptySet());
}
public void setLookupHandler(
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java b/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java
index 6ca025f33..b89025a78 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/metadata/TestingMetadataUpdater.java
@@ -74,7 +74,9 @@ public class TestingMetadataUpdater extends MetadataUpdater {
} else {
tabletServerGatewayMap = new HashMap<>();
for (ServerNode tabletServer : tabletServers) {
- tabletServerGatewayMap.put(tabletServer.id(), new TestTabletServerGateway(false));
+ tabletServerGatewayMap.put(
+ tabletServer.id(),
+ new TestTabletServerGateway(false, Collections.emptySet()));
}
}
}
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java
index 3e7484f19..cf2084675 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/FlussTableITCase.java
@@ -40,6 +40,7 @@ import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -241,6 +242,7 @@ class FlussTableITCase extends ClientToServerITCaseBase {
Table table = conn.getTable(tablePath);
verifyPutAndLookup(table, new Object[] {1, "a"});
+ TableInfo tableInfo = table.getTableInfo();
// test put/lookup data for primary table with pk index is not 0
Schema schema =
@@ -261,6 +263,30 @@ class FlussTableITCase extends ClientToServerITCaseBase {
// now, check put/lookup data
Table table2 = conn.getTable(data1PkTablePath2);
verifyPutAndLookup(table2, new Object[] {"a", 1});
+
+ // Test schema change: add new column which equals to DATA2_ROW_TYPE
+ admin.alterTable(
+ tableInfo.getTablePath(),
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ waitAllSchemaSync(tablePath, 2);
+ Table newSchemaTable = conn.getTable(tableInfo.getTablePath());
+ // schema change case1: read new data with new schema.
+ verifyPutAndLookup(newSchemaTable, new Object[] {2, "b", "bb"});
+ // schema change case2: read new data with old schema.
+ assertThatRow(lookupRow(table.newLookup().createLookuper(), row(2)))
+ .withSchema(tableInfo.getSchema().getRowType())
+ .isEqualTo(row(2, "b"));
+ // schema change case3: read old data with new schema.
+ assertThatRow(lookupRow(newSchemaTable.newLookup().createLookuper(), row(1)))
+ .withSchema(newSchemaTable.getTableInfo().getSchema().getRowType())
+ .isEqualTo(row(1, "a", null));
}
@Test
@@ -278,6 +304,7 @@ class FlussTableITCase extends ClientToServerITCaseBase {
TableDescriptor.builder().schema(schema).distributedBy(3, "a", "b").build();
createTable(tablePath, descriptor, false);
Table table = conn.getTable(tablePath);
+ TableInfo tableInfo = table.getTableInfo();
verifyPutAndLookup(table, new Object[] {1, "a", 1L, "value1"});
verifyPutAndLookup(table, new Object[] {1, "a", 2L, "value2"});
verifyPutAndLookup(table, new Object[] {1, "a", 3L, "value3"});
@@ -315,6 +342,64 @@ class FlussTableITCase extends ClientToServerITCaseBase {
assertThat(prefixLookupResult).isNotNull();
rowList = prefixLookupResult.getRowList();
assertThat(rowList.size()).isEqualTo(0);
+
+ // Test schema change: add new column.
+ Schema newSchema =
+ Schema.newBuilder()
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.STRING())
+ .column("c", DataTypes.BIGINT())
+ .column("d", DataTypes.STRING())
+ .column("e", DataTypes.STRING())
+ .primaryKey("a", "b", "c")
+ .build();
+ admin.alterTable(
+ tableInfo.getTablePath(),
+ Collections.singletonList(
+ TableChange.addColumn(
+ "e",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ waitAllSchemaSync(tablePath, 2);
+ try (Connection connection = ConnectionFactory.createConnection(clientConf);
+ Table newSchemaTable = connection.getTable(tableInfo.getTablePath())) {
+ // schema change case1: read new data with new schema.
+ verifyPutAndLookup(
+ newSchemaTable, new Object[] {1, "a", 4L, "value4", "add_column_value"});
+ // schema change case2: read new data with old schema.
+ result = prefixLookuper.lookup(row(1, "a"));
+ prefixLookupResult = result.get();
+ assertThat(prefixLookupResult).isNotNull();
+ rowList = prefixLookupResult.getRowList();
+ assertThat(rowList.size()).isEqualTo(4);
+ for (int i = 0; i < rowList.size(); i++) {
+ assertRowValueEquals(
+ rowType, rowList.get(i), new Object[] {1, "a", i + 1L, "value" + (i + 1)});
+ }
+ admin.getTableSchema(tablePath, 1).get();
+ // schema change case3: read old data with new schema.
+ Lookuper newPrefixLookuper =
+ newSchemaTable
+ .newLookup()
+ .lookupBy(prefixKeyRowType.getFieldNames())
+ .createLookuper();
+ result = newPrefixLookuper.lookup(row(1, "a"));
+ prefixLookupResult = result.get();
+ assertThat(prefixLookupResult).isNotNull();
+ rowList = prefixLookupResult.getRowList();
+ assertThat(rowList.size()).isEqualTo(4);
+ for (int i = 0; i < rowList.size(); i++) {
+ assertRowValueEquals(
+ rowType,
+ rowList.get(i),
+ new Object[] {
+ 1, "a", i + 1L, "value" + (i + 1), i == 3 ? "add_column_value" : null
+ });
+ }
+ }
}
@Test
@@ -908,6 +993,126 @@ class FlussTableITCase extends ClientToServerITCaseBase {
}
}
+ @Test
+ void testPutAndProjectDuringAddColumn() throws Exception {
+ Schema schema =
+ Schema.newBuilder()
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.INT())
+ .column("c", DataTypes.STRING())
+ .primaryKey("a")
+ .build();
+ TableDescriptor tableDescriptor = TableDescriptor.builder().schema(schema).build();
+ TablePath tablePath = TablePath.of("test_db_1", "test_pk_table_1");
+ createTable(tablePath, tableDescriptor, false);
+
+ int batches = 3;
+ int keyId = 0;
+ int expectedSize = 0;
+
+ try (Table table = conn.getTable(tablePath)) {
+ // produce data with new schema.
+ // Test schema change: add new column which equals to DATA2_ROW_TYPE
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "d",
+ DataTypes.BIGINT(),
+ "add new column",
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ waitAllSchemaSync(tablePath, 2);
+ try (Connection connection = ConnectionFactory.createConnection(clientConf);
+ Table newSchemaTable = connection.getTable(tablePath)) {
+ UpsertWriter oldSchemaUpsertWriter = table.newUpsert().createWriter();
+ UpsertWriter newSchemaUpsertWriter = newSchemaTable.newUpsert().createWriter();
+ for (int b = 0; b < batches; b++) {
+ // insert 10 rows with old schema.
+ for (int i = keyId; i < keyId + 10; i++) {
+ InternalRow row = row(i, 100, "hello, friend" + i);
+ oldSchemaUpsertWriter.upsert(row);
+ expectedSize += 1;
+ oldSchemaUpsertWriter.flush();
+ }
+ // update 5 rows with new schema: [keyId, keyId+4]
+ for (int i = keyId; i < keyId + 5; i++) {
+ InternalRow row = row(i, 100, "HELLO, FRIEND" + i, i * 10L);
+ newSchemaUpsertWriter.upsert(row);
+ expectedSize += 2;
+ newSchemaUpsertWriter.flush();
+ }
+ // delete 1 row with old schema: [keyId+5]
+ int deleteKey = keyId + 5;
+ InternalRow row = row(deleteKey, 100, "hello, friend" + deleteKey);
+ oldSchemaUpsertWriter.delete(row);
+ expectedSize += 1;
+ // flush the mutation batch
+ oldSchemaUpsertWriter.flush();
+ keyId += 10;
+ }
+
+ // read with new schema
+ try (LogScanner logScanner = createLogScanner(newSchemaTable, new int[] {2, 0})) {
+ subscribeFromBeginning(logScanner, table);
+ int count = 0;
+ int id = 0;
+ while (count < expectedSize) {
+ ScanRecords scanRecords = logScanner.poll(Duration.ofSeconds(1));
+ Iterator<ScanRecord> iterator = scanRecords.iterator();
+ while (iterator.hasNext()) {
+ // 10 inserts
+ for (int i = 0; i < 10; i++) {
+ ScanRecord scanRecord = iterator.next();
+ assertThat(scanRecord.getChangeType()).isEqualTo(ChangeType.INSERT);
+ assertThat(scanRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(scanRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(scanRecord.getRow().getString(0).toString())
+ .isEqualTo("hello, friend" + id);
+ count++;
+ id++;
+ }
+ id -= 10;
+ // 10 updates
+ for (int i = 0; i < 5; i++) {
+ ScanRecord beforeRecord = iterator.next();
+ assertThat(beforeRecord.getChangeType())
+ .isEqualTo(ChangeType.UPDATE_BEFORE);
+ assertThat(beforeRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(beforeRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(beforeRecord.getRow().getString(0).toString())
+ .isEqualTo("hello, friend" + id);
+
+ ScanRecord afterRecord = iterator.next();
+ assertThat(afterRecord.getChangeType())
+ .isEqualTo(ChangeType.UPDATE_AFTER);
+ assertThat(afterRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(afterRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(afterRecord.getRow().getString(0).toString())
+ .isEqualTo("HELLO, FRIEND" + id);
+
+ id++;
+ count += 2;
+ }
+
+ // 1 delete
+ ScanRecord beforeRecord = iterator.next();
+ assertThat(beforeRecord.getChangeType()).isEqualTo(ChangeType.DELETE);
+ assertThat(beforeRecord.getRow().getFieldCount()).isEqualTo(2);
+ assertThat(beforeRecord.getRow().getInt(1)).isEqualTo(id);
+ assertThat(beforeRecord.getRow().getString(0).toString())
+ .isEqualTo("hello, friend" + id);
+ count++;
+ id += 5;
+ }
+ }
+ assertThat(count).isEqualTo(expectedSize);
+ }
+ }
+ }
+ }
+
@Test
void testInvalidColumnProjection() throws Exception {
TableDescriptor tableDescriptor =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java
index 7797437e0..d204d087a 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/PartitionedTableITCase.java
@@ -170,7 +170,7 @@ class PartitionedTableITCase extends ClientToServerITCaseBase {
AppendWriter appendWriter = table.newAppend().createWriter();
int partitionSize = 5;
- // first try to add records add wait partition created.
+ // first try to add records and wait partition created.
Map<String, List<InternalRow>> expectPartitionNameAndAppendRows = new HashMap<>();
for (int i = 0; i < partitionSize; i++) {
String partitionName = String.valueOf(i);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
index b3f1eee8f..527994777 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
@@ -27,9 +27,11 @@ import org.apache.fluss.client.write.HashBucketAssigner;
import org.apache.fluss.metadata.DataLakeFormat;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.CompactedKeyEncoder;
import org.apache.fluss.row.encode.KeyEncoder;
import org.apache.fluss.types.DataTypes;
@@ -39,6 +41,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -116,13 +119,85 @@ class KvSnapshotBatchScannerITCase extends ClientToServerITCaseBase {
testSnapshotRead(tablePath, expectedRowByBuckets);
}
- private Map<TableBucket, List<InternalRow>> putRows(long tableId, TablePath tablePath, int rows)
- throws Exception {
+ @Test
+ void testScanSnapshotDuringSchemaChange() throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "test-table-snapshot-schema-change");
+ long tableId = createTable(tablePath, DEFAULT_TABLE_DESCRIPTOR, true);
+
+ // put into values with old schema.
+ Map<TableBucket, List<InternalRow>> oldSchemaRowByBuckets = putRows(tableId, tablePath, 10);
+ waitUntilAllSnapshotFinished(oldSchemaRowByBuckets.keySet(), 0);
+
+ // add a new column and rename an existing column
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "new_column",
+ DataTypes.BIGINT().copy(false).copy(true),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(tablePath, 2);
+
+ Schema newSchema =
+ Schema.newBuilder()
+ .primaryKey("id")
+ .column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ .column("new_column", DataTypes.BIGINT())
+ .build();
+ // put into values with old schema.
+ List<InternalRow> rows = new ArrayList<>();
+ for (int i = 10; i < 20; i++) {
+ InternalRow row =
+ compactedRow(newSchema.getRowType(), new Object[] {i, "v" + i, (long) i});
+ rows.add(row);
+ }
+ Map<TableBucket, List<InternalRow>> newSchemaByBuckets = putRows(tableId, tablePath, rows);
+
+ Map<TableBucket, List<InternalRow>> expectedRowByBuckets = new HashMap<>();
+ for (TableBucket tableBucket : oldSchemaRowByBuckets.keySet()) {
+ List<InternalRow> expectedRows =
+ expectedRowByBuckets.computeIfAbsent(tableBucket, k -> new ArrayList<>());
+ oldSchemaRowByBuckets
+ .get(tableBucket)
+ .forEach(
+ row ->
+ expectedRows.add(
+ ProjectedRow.from(DEFAULT_SCHEMA, newSchema)
+ .replaceRow(row)));
+ }
+ for (TableBucket tableBucket : newSchemaByBuckets.keySet()) {
+ expectedRowByBuckets
+ .computeIfAbsent(tableBucket, k -> new ArrayList<>())
+ .addAll(newSchemaByBuckets.get(tableBucket));
+ }
+
+ // wait snapshot finish
+ waitUntilAllSnapshotFinished(expectedRowByBuckets.keySet(), 1);
+
+ // test read snapshot with new Schema
+ testSnapshotRead(tablePath, expectedRowByBuckets);
+ }
+
+ private Map<TableBucket, List<InternalRow>> putRows(
+ long tableId, TablePath tablePath, int rowNumber) throws Exception {
+ List<InternalRow> rows = new ArrayList<>();
+ for (int i = 0; i < rowNumber; i++) {
+ InternalRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {i, "v" + i});
+ rows.add(row);
+ }
+ return putRows(tableId, tablePath, rows);
+ }
+
+ private Map<TableBucket, List<InternalRow>> putRows(
+ long tableId, TablePath tablePath, List<InternalRow> rows) throws Exception {
Map<TableBucket, List<InternalRow>> rowsByBuckets = new HashMap<>();
try (Table table = conn.getTable(tablePath)) {
UpsertWriter upsertWriter = table.newUpsert().createWriter();
- for (int i = 0; i < rows; i++) {
- InternalRow row = compactedRow(DATA1_ROW_TYPE, new Object[] {i, "v" + i});
+ for (InternalRow row : rows) {
upsertWriter.upsert(row);
TableBucket tableBucket = new TableBucket(tableId, getBucketId(row));
rowsByBuckets.computeIfAbsent(tableBucket, k -> new ArrayList<>()).add(row);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
index 3502dcc3c..bfd686dc1 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
@@ -20,6 +20,7 @@ package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
@@ -28,6 +29,7 @@ import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
import org.apache.fluss.types.DataTypes;
@@ -71,9 +73,13 @@ public class DefaultCompletedFetchTest {
private @TempDir File tempDir;
private TableInfo tableInfo;
private RowType rowType;
+ private TestingSchemaGetter testingSchemaGetter;
@BeforeEach
void beforeEach() {
+ testingSchemaGetter =
+ new TestingSchemaGetter(
+ new SchemaInfo(DATA2_TABLE_INFO.getSchema(), DEFAULT_SCHEMA_ID));
tableInfo = DATA2_TABLE_INFO;
rowType = DATA2_ROW_TYPE;
Map<TableBucket, Long> scanBuckets = new HashMap<>();
@@ -162,7 +168,7 @@ public class DefaultCompletedFetchTest {
long fetchOffset = 0L;
int bucketId = 0; // records for 0-10.
TableBucket tb = new TableBucket(DATA2_TABLE_ID, bucketId);
- Projection projection = Projection.of(new int[] {0, 2});
+ Projection projection = Projection.of(new int[] {0, 2}, schema);
MemoryLogRecords memoryLogRecords;
if (logFormat == LogFormat.ARROW) {
memoryLogRecords = genRecordsWithProjection(DATA2, projection, magic);
@@ -198,7 +204,7 @@ public class DefaultCompletedFetchTest {
// test projection reorder.
defaultCompletedFetch =
makeCompletedFetch(
- tb, resultForBucket0, fetchOffset, Projection.of(new int[] {2, 0}));
+ tb, resultForBucket0, fetchOffset, Projection.of(new int[] {2, 0}, schema));
scanRecords = defaultCompletedFetch.fetchRecords(8);
assertThat(scanRecords.size()).isEqualTo(8);
for (int i = 0; i < scanRecords.size(); i++) {
@@ -225,7 +231,11 @@ public class DefaultCompletedFetchTest {
return new DefaultCompletedFetch(
tableBucket,
resultForBucket,
- LogRecordReadContext.createReadContext(tableInfo, false, projection),
+ LogRecordReadContext.createReadContext(
+ tableInfo,
+ false,
+ projection,
+ new TestingSchemaGetter(tableInfo.getSchemaId(), tableInfo.getSchema())),
logScannerStatus,
true,
offset);
@@ -257,7 +267,10 @@ public class DefaultCompletedFetchTest {
FileLogProjection fileLogProjection = new FileLogProjection();
fileLogProjection.setCurrentProjection(
- DATA2_TABLE_ID, rowType, DEFAULT_COMPRESSION, projection.getProjectionInOrder());
+ DATA2_TABLE_ID,
+ testingSchemaGetter,
+ DEFAULT_COMPRESSION,
+ projection.getProjectionIdInOrder());
ByteBuffer buffer =
toByteBuffer(
fileLogProjection
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
index f7a1a07c4..095131b1e 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchBufferTest.java
@@ -42,6 +42,7 @@ import java.util.concurrent.atomic.AtomicReference;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.assertj.core.api.Assertions.assertThat;
@@ -63,7 +64,8 @@ public class LogFetchBufferTest {
logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(scanBuckets);
readContext =
- LogRecordReadContext.createArrowReadContext(DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createArrowReadContext(
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
}
@AfterEach
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
index 9888fe03a..83878a816 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
@@ -40,6 +40,7 @@ import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.assertj.core.api.Assertions.assertThat;
@@ -66,7 +67,8 @@ public class LogFetchCollectorTest {
new LogFetchCollector(
DATA1_TABLE_PATH, logScannerStatus, new Configuration(), metadataUpdater);
readContext =
- LogRecordReadContext.createArrowReadContext(DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createArrowReadContext(
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, TEST_SCHEMA_GETTER);
}
@AfterEach
@@ -91,7 +93,8 @@ public class LogFetchCollectorTest {
logFetchBuffer.add(completedFetch);
assertThat(logFetchBuffer.isEmpty()).isFalse();
- // Validate that the completed fetch isn't initialized just because we add it to the buffer.
+ // Validate that the completed fetch isn't initialized just because we add it to the
+ // buffer.
assertThat(completedFetch.isInitialized()).isFalse();
// Fetch the data and validate that we get all the records we want back.
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
index b131986e7..03f908733 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
@@ -18,18 +18,26 @@
package org.apache.fluss.client.table.scanner.log;
import org.apache.fluss.client.admin.ClientToServerITCaseBase;
+import org.apache.fluss.client.admin.FlussAdmin;
+import org.apache.fluss.client.metadata.ClientSchemaGetter;
import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.metrics.TestingScannerMetricGroup;
import org.apache.fluss.client.table.scanner.RemoteFileDownloader;
import org.apache.fluss.client.table.scanner.ScanRecord;
import org.apache.fluss.cluster.Cluster;
import org.apache.fluss.cluster.ServerNode;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.row.GenericRow;
import org.apache.fluss.rpc.RpcClient;
import org.apache.fluss.rpc.gateway.TabletServerGateway;
import org.apache.fluss.rpc.messages.PbProduceLogRespForBucket;
import org.apache.fluss.rpc.messages.ProduceLogResponse;
+import org.apache.fluss.testutils.DataTestUtils;
+import org.apache.fluss.types.DataTypes;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -43,11 +51,17 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE;
import static org.apache.fluss.record.TestData.DATA1;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DATA1_TABLE_DESCRIPTOR;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA1_TABLE_PATH;
+import static org.apache.fluss.record.TestData.DATA2;
+import static org.apache.fluss.record.TestData.DATA2_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA2_SCHEMA;
import static org.apache.fluss.server.testutils.RpcMessageTestUtils.newProduceLogRequest;
import static org.apache.fluss.testutils.DataTestUtils.genMemoryLogRecordsByObject;
import static org.apache.fluss.testutils.common.CommonTestUtils.retry;
@@ -60,6 +74,9 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
private final int bucketId0 = 0;
private final int bucketId1 = 1;
private LogScannerStatus logScannerStatus;
+ private FlussAdmin admin;
+ private MetadataUpdater metadataUpdater;
+ private ClientSchemaGetter clientSchemaGetter;
// TODO covert this test to UT as kafka.
@@ -72,7 +89,7 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
FLUSS_CLUSTER_EXTENSION.waitUntilTableReady(tableId);
RpcClient rpcClient = FLUSS_CLUSTER_EXTENSION.getRpcClient();
- MetadataUpdater metadataUpdater = new MetadataUpdater(clientConf, rpcClient);
+ metadataUpdater = new MetadataUpdater(clientConf, rpcClient);
metadataUpdater.checkAndUpdateTableMetadata(Collections.singleton(DATA1_TABLE_PATH));
Map<TableBucket, Long> scanBuckets = new HashMap<>();
@@ -81,6 +98,9 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
scanBuckets.put(new TableBucket(tableId, bucketId1), 0L);
logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(scanBuckets);
+ admin = new FlussAdmin(rpcClient, metadataUpdater);
+ clientSchemaGetter =
+ new ClientSchemaGetter(DATA1_TABLE_PATH, new SchemaInfo(DATA1_SCHEMA, 1), admin);
logFetcher =
new LogFetcher(
DATA1_TABLE_INFO,
@@ -89,7 +109,96 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
clientConf,
metadataUpdater,
TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1));
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
+ }
+
+ @Test
+ void testFetchWithSchemaChange() throws Exception {
+ // add one batch records to tb0.
+ TableBucket tb0 = new TableBucket(tableId, bucketId0);
+ addRecordsToBucket(tb0, genMemoryLogRecordsByObject(DATA1), 0L);
+
+ // add new column(which equals to DATA2_ROW_TYPE)
+ admin.alterTable(
+ DATA1_TABLE_PATH,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ // add one batch records with new schema to tb0.
+ addRecordsToBucket(
+ tb0,
+ genMemoryLogRecordsByObject(DATA2_ROW_TYPE, 2, CURRENT_LOG_MAGIC_VALUE, DATA2),
+ 10L);
+
+ // Read data with old schema, thus DATA2 will be truncated as DATA1
+ List<GenericRow> expectedRows =
+ DATA1.stream().map(DataTestUtils::row).collect(Collectors.toList());
+ expectedRows.addAll(DATA1.stream().map(DataTestUtils::row).collect(Collectors.toList()));
+ logFetcher.sendFetches();
+ // The fetcher is async to fetch data, so we need to wait the result write to the
+ // logFetchBuffer.
+ retry(
+ Duration.ofMinutes(1),
+ () -> {
+ assertThat(logFetcher.hasAvailableFetches()).isTrue();
+ assertThat(logFetcher.getCompletedFetchesSize()).isEqualTo(2);
+ });
+ Map<TableBucket, List<ScanRecord>> records = logFetcher.collectFetch();
+ assertThat(records.size()).isEqualTo(1);
+ List<ScanRecord> scanRecords = records.get(tb0);
+ assertThat(scanRecords.stream().map(ScanRecord::getRow).collect(Collectors.toList()))
+ .isEqualTo(expectedRows);
+
+ // read data with new schema, thus DATA2 will be appended with null value.
+ expectedRows =
+ DATA1.stream()
+ .map(row -> DataTestUtils.row(row[0], row[1], null))
+ .collect(Collectors.toList());
+ expectedRows.addAll(DATA2.stream().map(DataTestUtils::row).collect(Collectors.toList()));
+ logScannerStatus.assignScanBuckets(Collections.singletonMap(tb0, 0L));
+ LogFetcher newSchemaLogFetcher =
+ new LogFetcher(
+ new TableInfo(
+ DATA1_TABLE_INFO.getTablePath(),
+ tableId,
+ 2,
+ DATA2_SCHEMA,
+ DATA1_TABLE_INFO.getBucketKeys(),
+ DATA1_TABLE_INFO.getPartitionKeys(),
+ DATA1_TABLE_INFO.getNumBuckets(),
+ DATA1_TABLE_INFO.getProperties(),
+ DATA1_TABLE_INFO.getCustomProperties(),
+ DATA1_TABLE_INFO.getComment().orElse(null),
+ DATA1_TABLE_INFO.getCreatedTime(),
+ DATA1_TABLE_INFO.getModifiedTime()),
+ null,
+ logScannerStatus,
+ clientConf,
+ metadataUpdater,
+ TestingScannerMetricGroup.newInstance(),
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
+ newSchemaLogFetcher.sendFetches();
+ // The fetcher is async to fetch data, so we need to wait the result write to the
+ // logFetchBuffer.
+ retry(
+ Duration.ofMinutes(1),
+ () -> {
+ assertThat(newSchemaLogFetcher.hasAvailableFetches()).isTrue();
+ assertThat(newSchemaLogFetcher.getCompletedFetchesSize()).isEqualTo(2);
+ });
+ records = newSchemaLogFetcher.collectFetch();
+ assertThat(records.size()).isEqualTo(1);
+ assertThat(records.get(tb0)).hasSize(20);
+ scanRecords = records.get(tb0);
+ assertThat(scanRecords.stream().map(ScanRecord::getRow).collect(Collectors.toList()))
+ .isEqualTo(expectedRows);
}
@Test
@@ -157,6 +266,12 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
LogScannerStatus logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(Collections.singletonMap(tb0, 0L));
+ ClientSchemaGetter clientSchemaGetter =
+ new ClientSchemaGetter(
+ DATA1_TABLE_PATH,
+ new SchemaInfo(DATA1_SCHEMA, 1),
+ new FlussAdmin(rpcClient, metadataUpdater));
+
LogFetcher logFetcher =
new LogFetcher(
DATA1_TABLE_INFO,
@@ -165,7 +280,8 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
clientConf,
metadataUpdater,
TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1));
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
// send fetches to fetch data, should have no available fetch.
logFetcher.sendFetches();
@@ -191,6 +307,11 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
void testFetchWithInvalidTableOrPartitions() throws Exception {
MetadataUpdater metadataUpdater1 =
new MetadataUpdater(clientConf, FLUSS_CLUSTER_EXTENSION.getRpcClient());
+ ClientSchemaGetter clientSchemaGetter =
+ new ClientSchemaGetter(
+ DATA1_TABLE_PATH,
+ new SchemaInfo(DATA1_SCHEMA, 1),
+ new FlussAdmin(FLUSS_CLUSTER_EXTENSION.getRpcClient(), metadataUpdater1));
logFetcher =
new LogFetcher(
DATA1_TABLE_INFO,
@@ -199,7 +320,8 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
clientConf,
metadataUpdater1,
TestingScannerMetricGroup.newInstance(),
- new RemoteFileDownloader(1));
+ new RemoteFileDownloader(1),
+ clientSchemaGetter);
ExecutorService executor = Executors.newSingleThreadExecutor();
Future<?> future =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
index f52a86945..edc47a7ec 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/RemoteCompletedFetchTest.java
@@ -22,12 +22,14 @@ import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.DataTypes;
@@ -55,6 +57,7 @@ import java.util.concurrent.atomic.AtomicBoolean;
import static org.apache.fluss.record.TestData.DATA2;
import static org.apache.fluss.record.TestData.DATA2_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA2_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA2_SCHEMA;
import static org.apache.fluss.record.TestData.DATA2_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA2_TABLE_INFO;
import static org.apache.fluss.record.TestData.DATA2_TABLE_PATH;
@@ -69,6 +72,7 @@ class RemoteCompletedFetchTest {
private LogRecordReadContext remoteReadContext;
private @TempDir File tempDir;
private TableInfo tableInfo;
+ private SchemaGetter schemaGetter;
@BeforeEach
void beforeEach() {
@@ -79,8 +83,10 @@ class RemoteCompletedFetchTest {
scanBuckets.put(new TableBucket(DATA2_TABLE_ID, 2), 0L);
logScannerStatus = new LogScannerStatus();
logScannerStatus.assignScanBuckets(scanBuckets);
+ schemaGetter = new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA2_SCHEMA);
remoteReadContext =
- LogRecordReadContext.createArrowReadContext(DATA2_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createArrowReadContext(
+ DATA2_ROW_TYPE, DEFAULT_SCHEMA_ID, schemaGetter);
}
@AfterEach
@@ -220,7 +226,10 @@ class RemoteCompletedFetchTest {
createFileLogRecords(tableBucket, DATA2_PHYSICAL_TABLE_PATH, DATA2, logFormat);
RemoteCompletedFetch completedFetch =
makeCompletedFetch(
- tableBucket, fileLogRecords, fetchOffset, Projection.of(new int[] {0, 2}));
+ tableBucket,
+ fileLogRecords,
+ fetchOffset,
+ Projection.of(new int[] {0, 2}, schema));
List<ScanRecord> scanRecords = completedFetch.fetchRecords(8);
List<Object[]> expectedObjects =
@@ -246,7 +255,10 @@ class RemoteCompletedFetchTest {
completedFetch =
makeCompletedFetch(
- tableBucket, fileLogRecords, fetchOffset, Projection.of(new int[] {2, 0}));
+ tableBucket,
+ fileLogRecords,
+ fetchOffset,
+ Projection.of(new int[] {2, 0}, schema));
scanRecords = completedFetch.fetchRecords(8);
assertThat(scanRecords.size()).isEqualTo(8);
for (int i = 0; i < scanRecords.size(); i++) {
@@ -292,7 +304,7 @@ class RemoteCompletedFetchTest {
tableBucket,
fileLogRecords,
10L,
- LogRecordReadContext.createReadContext(tableInfo, true, projection),
+ LogRecordReadContext.createReadContext(tableInfo, true, projection, schemaGetter),
logScannerStatus,
true,
fetchOffset,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java
index 6b7eb6fe7..cb76d7688 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/ArrowLogWriteBatchTest.java
@@ -49,6 +49,7 @@ import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.DataTestUtils.row;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -95,7 +96,10 @@ public class ArrowLogWriteBatchTest {
LogRecordBatch batch = records.batches().iterator().next();
assertThat(batch.getRecordCount()).isEqualTo(count);
try (LogRecordReadContext readContext =
- createArrowReadContext(DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ createArrowReadContext(
+ DATA1_ROW_TYPE,
+ DATA1_TABLE_INFO.getSchemaId(),
+ TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> recordsIter = batch.records(readContext)) {
int readCount = 0;
while (recordsIter.hasNext()) {
@@ -153,7 +157,10 @@ public class ArrowLogWriteBatchTest {
LogRecordBatch batch = records.batches().iterator().next();
assertThat(batch.getRecordCount()).isEqualTo(count);
try (LogRecordReadContext readContext =
- createArrowReadContext(DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ createArrowReadContext(
+ DATA1_ROW_TYPE,
+ DATA1_TABLE_INFO.getSchemaId(),
+ TEST_SCHEMA_GETTER);
CloseableIterator<LogRecord> recordsIter = batch.records(readContext)) {
int readCount = 0;
while (recordsIter.hasNext()) {
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java
index 3081814a5..db39493bd 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/IndexedLogWriteBatchTest.java
@@ -19,6 +19,7 @@ package org.apache.fluss.client.write;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.memory.PreAllocatedPagedOutputView;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.IndexedLogRecord;
@@ -26,6 +27,7 @@ import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.record.bytesview.BytesView;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.utils.CloseableIterator;
@@ -43,6 +45,7 @@ import static org.apache.fluss.record.LogRecordBatch.CURRENT_LOG_MAGIC_VALUE;
import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize;
import static org.apache.fluss.record.TestData.DATA1_PHYSICAL_TABLE_PATH;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO;
import static org.apache.fluss.testutils.DataTestUtils.indexedRow;
@@ -98,7 +101,8 @@ public class IndexedLogWriteBatchTest {
BytesView bytesView = logProducerBatch.build();
MemoryLogRecords logRecords = MemoryLogRecords.pointToBytesView(bytesView);
Iterator<LogRecordBatch> iterator = logRecords.batches().iterator();
- assertDefaultLogRecordBatchEquals(iterator.next());
+ assertDefaultLogRecordBatchEquals(
+ iterator.next(), new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA, (short) 1)));
assertThat(iterator.hasNext()).isFalse();
}
@@ -219,13 +223,14 @@ public class IndexedLogWriteBatchTest {
System.currentTimeMillis());
}
- private void assertDefaultLogRecordBatchEquals(LogRecordBatch recordBatch) {
+ private void assertDefaultLogRecordBatchEquals(
+ LogRecordBatch recordBatch, TestingSchemaGetter schemaGetter) {
assertThat(recordBatch.getRecordCount()).isEqualTo(1);
assertThat(recordBatch.baseLogOffset()).isEqualTo(0L);
assertThat(recordBatch.schemaId()).isEqualTo((short) DATA1_TABLE_INFO.getSchemaId());
try (LogRecordReadContext readContext =
LogRecordReadContext.createIndexedReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId(), schemaGetter);
CloseableIterator<LogRecord> iterator = recordBatch.records(readContext)) {
assertThat(iterator.hasNext()).isTrue();
LogRecord record = iterator.next();
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java
index 5321da0f2..82f6ee590 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/KvWriteBatchTest.java
@@ -30,6 +30,7 @@ import org.apache.fluss.record.DefaultKvRecord;
import org.apache.fluss.record.DefaultKvRecordBatch;
import org.apache.fluss.record.KvRecord;
import org.apache.fluss.record.KvRecordReadContext;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.encode.CompactedKeyEncoder;
import org.apache.fluss.types.DataType;
@@ -44,6 +45,7 @@ import java.util.List;
import java.util.concurrent.CompletableFuture;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK;
import static org.apache.fluss.record.TestData.DATA1_TABLE_ID_PK;
import static org.apache.fluss.record.TestData.DATA1_TABLE_INFO_PK;
@@ -243,7 +245,8 @@ class KvWriteBatchTest {
recordBatch
.records(
KvRecordReadContext.createReadContext(
- KvFormat.COMPACTED, dataTypes))
+ KvFormat.COMPACTED,
+ new TestingSchemaGetter(1, DATA1_SCHEMA)))
.iterator();
assertThat(iterator.hasNext()).isTrue();
KvRecord kvRecord = iterator.next();
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
index 5aeee2d10..bc4e53b50 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
@@ -27,6 +27,8 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.config.MemorySize;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TableInfo;
@@ -38,6 +40,7 @@ import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.arrow.ArrowWriter;
@@ -127,12 +130,15 @@ class RecordAccumulatorTest {
private final ManualClock clock = new ManualClock(System.currentTimeMillis());
private Configuration conf;
private Cluster cluster;
+ private SchemaGetter schemaGetter;
@BeforeEach
public void start() {
conf = new Configuration();
// init cluster.
cluster = updateCluster(Arrays.asList(bucket1, bucket2, bucket3));
+ // todo: 从cluster中获取schema getter
+ schemaGetter = new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA, (short) 1));
}
// TODO Add more tests to test lingMs, retryBackoffMs, deliveryTimeoutMs and
@@ -292,7 +298,7 @@ class RecordAccumulatorTest {
Iterator<LogRecordBatch> iterator = memoryLogRecords.batches().iterator();
try (LogRecordReadContext readContext =
LogRecordReadContext.createIndexedReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId());
+ DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId(), schemaGetter);
CloseableIterator<LogRecord> iter = iterator.next().records(readContext)) {
for (int i = 0; i < appends; i++) {
LogRecord record = iter.next();
@@ -329,7 +335,7 @@ class RecordAccumulatorTest {
LogRecordBatch logRecordBatch = iterator.next();
try (LogRecordReadContext readContext =
LogRecordReadContext.createIndexedReadContext(
- DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId())) {
+ DATA1_ROW_TYPE, DATA1_TABLE_INFO.getSchemaId(), schemaGetter)) {
LogRecord record = logRecordBatch.records(readContext).next();
assertThat(record.getChangeType()).isEqualTo(ChangeType.APPEND_ONLY);
assertThat(record.logOffset()).isEqualTo(0L);
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/config/cluster/ColumnPositionType.java
similarity index 51%
copy from fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
copy to fluss-common/src/main/java/org/apache/fluss/config/cluster/ColumnPositionType.java
index a0cc396b1..0f73c50b0 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/config/cluster/ColumnPositionType.java
@@ -15,28 +15,35 @@
* limitations under the License.
*/
-package org.apache.fluss.record;
+package org.apache.fluss.config.cluster;
-import org.apache.fluss.metadata.KvFormat;
-import org.apache.fluss.row.decode.RowDecoder;
-import org.apache.fluss.types.DataType;
+import org.apache.fluss.metadata.TableChange;
-/** A default implementation of {@link KvRecordBatch.ReadContext} . */
-public class KvRecordReadContext implements KvRecordBatch.ReadContext {
+/** ColumnPositionType. */
+public enum ColumnPositionType {
+ LAST(0);
- private final RowDecoder rowDecoder;
+ public final int value;
- private KvRecordReadContext(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
+ ColumnPositionType(int value) {
+ this.value = value;
}
- public static KvRecordReadContext createReadContext(KvFormat kvFormat, DataType[] fieldTypes) {
- return new KvRecordReadContext(RowDecoder.create(kvFormat, fieldTypes));
+ public static ColumnPositionType from(int opType) {
+ if (opType == 0) {
+ return LAST;
+ }
+ throw new IllegalArgumentException("Unsupported ColumnPositionType: " + opType);
}
- @Override
- public RowDecoder getRowDecoder(int schemaId) {
- // assume schema id are always not changed for now.
- return rowDecoder;
+ public static ColumnPositionType from(TableChange.ColumnPosition opType) {
+ if (opType == TableChange.ColumnPosition.last()) {
+ return LAST;
+ }
+ throw new IllegalArgumentException("Unsupported ColumnPositionType: " + opType);
+ }
+
+ public int value() {
+ return this.value;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
similarity index 57%
copy from fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
copy to fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
index b2cea1382..8c9446682 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
@@ -15,26 +15,16 @@
* limitations under the License.
*/
-package org.apache.fluss.flink.sink.serializer;
+package org.apache.fluss.exception;
-import org.apache.fluss.types.RowType;
+/** Exception thrown when schema change failed. */
+public class SchemaChangeException extends ApiException {
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
-
-/**
- * Contextual information that can be used during initialization of {@link
- * FlussSerializationSchema}.
- */
-public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
-
- private final RowType rowSchema;
-
- public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
+ public SchemaChangeException(String message, Throwable cause) {
+ super(message, cause);
}
- @Override
- public RowType getRowSchema() {
- return rowSchema;
+ public SchemaChangeException(String message) {
+ this(message, null);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
index 2de841936..367013363 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
@@ -38,6 +38,7 @@ import java.util.List;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import java.util.stream.IntStream;
@@ -55,14 +56,19 @@ import static org.apache.fluss.utils.Preconditions.checkState;
public final class Schema implements Serializable {
private static final long serialVersionUID = 1L;
-
private static final Schema EMPTY = Schema.newBuilder().build();
private final List<Column> columns;
private final @Nullable PrimaryKey primaryKey;
private final RowType rowType;
- private Schema(List<Column> columns, @Nullable PrimaryKey primaryKey) {
+ /**
+ * The highest field ID in this schema, this can only increase during the life cycle of the
+ * schema. Otherwise, the removed columns will influence it.
+ */
+ private final int highestFieldId;
+
+ private Schema(List<Column> columns, @Nullable PrimaryKey primaryKey, int highestFieldId) {
this.columns = normalizeColumns(columns, primaryKey);
this.primaryKey = primaryKey;
// pre-create the row type as it is the most frequently used part of the schema
@@ -74,6 +80,7 @@ public final class Schema implements Serializable {
new DataField(
column.getName(), column.getDataType()))
.collect(Collectors.toList()));
+ this.highestFieldId = highestFieldId;
}
public List<Column> getColumns() {
@@ -125,6 +132,10 @@ public final class Schema implements Serializable {
return columns.stream().map(Column::getName).collect(Collectors.toList());
}
+ public List<Integer> getColumnIds() {
+ return columns.stream().map(Column::getColumnId).collect(Collectors.toList());
+ }
+
/** Returns the column names in given column indexes. */
public List<String> getColumnNames(int[] columnIndexes) {
List<String> columnNames = new ArrayList<>();
@@ -143,6 +154,10 @@ public final class Schema implements Serializable {
return keyIndexes;
}
+ public int getHighestFieldId() {
+ return highestFieldId;
+ }
+
@Override
public String toString() {
final List<Object> components = new ArrayList<>(columns);
@@ -188,9 +203,11 @@ public final class Schema implements Serializable {
public static final class Builder {
private final List<Column> columns;
private @Nullable PrimaryKey primaryKey;
+ private AtomicInteger highestFieldId;
private Builder() {
columns = new ArrayList<>();
+ highestFieldId = new AtomicInteger(-1);
}
/** Adopts all members from the given schema. */
@@ -199,10 +216,15 @@ public final class Schema implements Serializable {
if (schema.primaryKey != null) {
primaryKeyNamed(schema.primaryKey.constraintName, schema.primaryKey.columnNames);
}
+ this.highestFieldId = new AtomicInteger(schema.highestFieldId);
+ return this;
+ }
+
+ public Builder highestFieldId(int highestFieldId) {
+ this.highestFieldId = new AtomicInteger(highestFieldId);
return this;
}
- /** Adopts all fields of the given row as columns of the schema. */
public Builder fromRowType(RowType rowType) {
checkNotNull(rowType, "rowType must not be null.");
final List<DataType> fieldDataTypes = rowType.getChildren();
@@ -227,7 +249,33 @@ public final class Schema implements Serializable {
/** Adopts all columns from the given list. */
public Builder fromColumns(List<Column> inputColumns) {
- columns.addAll(inputColumns);
+ boolean nonMatchColumnId =
+ inputColumns.stream()
+ .noneMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
+ boolean allMatchColumnId =
+ inputColumns.stream()
+ .allMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
+ checkState(
+ nonMatchColumnId || allMatchColumnId,
+ "All columns must have columnId or none of them must have columnId.");
+
+ if (allMatchColumnId) {
+ columns.addAll(inputColumns);
+ highestFieldId =
+ new AtomicInteger(
+ inputColumns.stream()
+ .mapToInt(Column::getColumnId)
+ .max()
+ .orElse(-1));
+ } else {
+ // if all columnId is not set, this maybe from old version schema. Just use its
+ // position as columnId.
+ inputColumns.forEach(
+ column ->
+ this.column(column.columnName, column.dataType)
+ .withComment(column.comment));
+ }
+
return this;
}
@@ -239,12 +287,11 @@ public final class Schema implements Serializable {
* from and written to an external system.
*
* @param columnName column name
- * @param dataType data type of the column
*/
public Builder column(String columnName, DataType dataType) {
checkNotNull(columnName, "Column name must not be null.");
checkNotNull(dataType, "Data type must not be null.");
- columns.add(new Column(columnName, dataType));
+ columns.add(new Column(columnName, dataType, null, highestFieldId.incrementAndGet()));
return this;
}
@@ -316,7 +363,17 @@ public final class Schema implements Serializable {
/** Returns an instance of an {@link Schema}. */
public Schema build() {
- return new Schema(columns, primaryKey);
+ Integer maximumColumnId =
+ columns.stream().map(Column::getColumnId).max(Integer::compareTo).orElse(0);
+
+ checkState(
+ columns.isEmpty() || highestFieldId.get() >= maximumColumnId,
+ "Highest field id must be greater than or equal to the maximum column id.");
+
+ checkState(
+ columns.stream().map(Column::getColumnId).distinct().count() == columns.size(),
+ "Column ids must be unique.");
+ return new Schema(columns, primaryKey, highestFieldId.get());
}
}
@@ -331,19 +388,27 @@ public final class Schema implements Serializable {
*/
@PublicStable
public static final class Column implements Serializable {
+ public static final int UNKNOWN_COLUMN_ID = -1;
private static final long serialVersionUID = 1L;
+ private final int columnId;
private final String columnName;
private final DataType dataType;
private final @Nullable String comment;
public Column(String columnName, DataType dataType) {
- this(columnName, dataType, null);
+ this(columnName, dataType, null, UNKNOWN_COLUMN_ID);
}
public Column(String columnName, DataType dataType, @Nullable String comment) {
+ this(columnName, dataType, comment, UNKNOWN_COLUMN_ID);
+ }
+
+ public Column(
+ String columnName, DataType dataType, @Nullable String comment, int columnId) {
this.columnName = columnName;
this.dataType = dataType;
this.comment = comment;
+ this.columnId = columnId;
}
public String getName() {
@@ -354,12 +419,16 @@ public final class Schema implements Serializable {
return Optional.ofNullable(comment);
}
+ public int getColumnId() {
+ return columnId;
+ }
+
public DataType getDataType() {
return dataType;
}
public Column withComment(String comment) {
- return new Column(columnName, dataType, comment);
+ return new Column(columnName, dataType, comment, columnId);
}
@Override
@@ -387,12 +456,13 @@ public final class Schema implements Serializable {
Column that = (Column) o;
return Objects.equals(columnName, that.columnName)
&& Objects.equals(dataType, that.dataType)
- && Objects.equals(comment, that.comment);
+ && Objects.equals(comment, that.comment)
+ && Objects.equals(columnId, that.columnId);
}
@Override
public int hashCode() {
- return Objects.hash(columnName, dataType, comment);
+ return Objects.hash(columnName, dataType, comment, columnId);
}
}
@@ -492,9 +562,8 @@ public final class Schema implements Serializable {
new Column(
column.getName(),
column.getDataType().copy(false),
- column.getComment().isPresent()
- ? column.getComment().get()
- : null));
+ column.getComment().isPresent() ? column.getComment().get() : null,
+ column.getColumnId()));
} else {
newColumns.add(column);
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
similarity index 56%
copy from fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
copy to fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
index b2cea1382..8cdc14670 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
@@ -15,26 +15,13 @@
* limitations under the License.
*/
-package org.apache.fluss.flink.sink.serializer;
+package org.apache.fluss.metadata;
-import org.apache.fluss.types.RowType;
+/** A schema getter interface. */
+public interface SchemaGetter {
+ Schema getSchema(int schemaId);
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
+ SchemaInfo getLatestSchemaInfo();
-/**
- * Contextual information that can be used during initialization of {@link
- * FlussSerializationSchema}.
- */
-public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
-
- private final RowType rowSchema;
-
- public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
- }
-
- @Override
- public RowType getRowSchema() {
- return rowSchema;
- }
+ void release();
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
index 98e907e5f..36162f30c 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
@@ -17,11 +17,98 @@
package org.apache.fluss.metadata;
+import org.apache.fluss.types.DataType;
+
+import javax.annotation.Nullable;
+
import java.util.Objects;
/** {@link TableChange} represents the modification of the Fluss Table. */
public interface TableChange {
+ /**
+ * A table change toadd the column with specified position.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> ADD <column_definition> <column_position>
+ * </pre>
+ *
+ * @return a TableChange represents the modification.
+ */
+ static AddColumn addColumn(
+ String columnName,
+ DataType dataType,
+ @Nullable String comment,
+ ColumnPosition position) {
+ return new AddColumn(columnName, dataType, comment, position);
+ }
+
+ /**
+ * A table change to modify a column. The modification includes:
+ *
+ * <ul>
+ * <li>change column data type
+ * <li>reorder column position
+ * <li>modify column comment
+ * <li>rename column name
+ * <li>change the computed expression
+ * <li>change the metadata column expression
+ * </ul>
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> MODIFY <column_definition> COMMENT '<column_comment>' <column_position>
+ * </pre>
+ *
+ * @param oldColumn the definition of the old column.
+ * @param newColumn the definition of the new column.
+ * @param columnPosition the new position of the column.
+ * @return a TableChange represents the modification.
+ */
+ static ModifyColumn modifyColumn(
+ String columnName,
+ DataType dataType,
+ @Nullable String comment,
+ @Nullable ColumnPosition columnPosition) {
+ return new ModifyColumn(columnName, dataType, comment, columnPosition);
+ }
+
+ /**
+ * A table change to modify the column name.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> RENAME <old_column_name> TO <new_column_name>
+ * </pre>
+ *
+ * @param oldColumn the definition of the old column.
+ * @param newName the name of the new column.
+ * @return a TableChange represents the modification.
+ */
+ static RenameColumn renameColumn(String oldColumnName, String newColumnName) {
+ return new RenameColumn(oldColumnName, newColumnName);
+ }
+
+ /**
+ * A table change to drop column.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> DROP COLUMN <column_name>
+ * </pre>
+ *
+ * @param columnName the column to drop.
+ * @return a TableChange represents the modification.
+ */
+ static DropColumn dropColumn(String columnName) {
+ return new DropColumn(columnName);
+ }
+
static SetOption set(String key, String value) {
return new SetOption(key, value);
}
@@ -126,4 +213,196 @@ public interface TableChange {
return "ResetOption{" + "key='" + key + '\'' + '}';
}
}
+
+ /** A table change to modify the table schema. */
+ interface SchemaChange extends TableChange {}
+
+ /**
+ * A table change to add a column.
+ *
+ * <p>It is equal to the following statement:
+ *
+ * <pre>
+ * ALTER TABLE <table_name> ADD <column_definition> <column_position>
+ * </pre>
+ */
+ class AddColumn implements SchemaChange {
+ private final String name;
+ private final DataType dataType;
+ private final @Nullable String comment;
+
+ private final ColumnPosition position;
+
+ private AddColumn(
+ String name, DataType dataType, @Nullable String comment, ColumnPosition position) {
+ this.name = name;
+ this.dataType = dataType;
+ this.comment = comment;
+ this.position = position;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ @Nullable
+ public String getComment() {
+ return comment;
+ }
+
+ public ColumnPosition getPosition() {
+ return position;
+ }
+ }
+
+ /** A table change to drop a column. */
+ class DropColumn implements SchemaChange {
+ private final String name;
+
+ private DropColumn(String name) {
+ this.name = name;
+ }
+
+ public String getName() {
+ return name;
+ }
+ }
+
+ /** A table change to modify a column. */
+ class ModifyColumn implements SchemaChange {
+ private final String name;
+ private final DataType dataType;
+ private final @Nullable String comment;
+
+ private final @Nullable ColumnPosition newPosition;
+
+ private ModifyColumn(
+ String name,
+ DataType dataType,
+ @Nullable String comment,
+ @Nullable ColumnPosition newPosition) {
+ this.name = name;
+ this.dataType = dataType;
+ this.comment = comment;
+ this.newPosition = newPosition;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ @Nullable
+ public String getComment() {
+ return comment;
+ }
+
+ @Nullable
+ public ColumnPosition getNewPosition() {
+ return newPosition;
+ }
+ }
+
+ /** A table change to modify a column's name. */
+ class RenameColumn implements SchemaChange {
+ private final String oldColumnName;
+ private final String newColumnName;
+
+ private RenameColumn(String oldColumnName, String newColumnName) {
+ this.oldColumnName = oldColumnName;
+ this.newColumnName = newColumnName;
+ }
+
+ public String getOldColumnName() {
+ return oldColumnName;
+ }
+
+ public String getNewColumnName() {
+ return newColumnName;
+ }
+ }
+
+ /** The position of the modified or added column. */
+ interface ColumnPosition {
+ /** Get the position to place the column at the first. */
+ static ColumnPosition last() {
+ return Last.INSTANCE;
+ }
+
+ /** Get the position to place the column at the first. */
+ static ColumnPosition first() {
+ return First.INSTANCE;
+ }
+
+ /** Get the position to place the column after the specified column. */
+ static ColumnPosition after(String column) {
+ return new After(column);
+ }
+ }
+
+ /** Column position FIRST means the specified column should be the first column. */
+ final class First implements ColumnPosition {
+ private static final First INSTANCE = new First();
+
+ private First() {}
+
+ @Override
+ public String toString() {
+ return "FIRST";
+ }
+ }
+
+ /** Column position Last means the specified column should be the last column. */
+ final class Last implements ColumnPosition {
+ private static final Last INSTANCE = new Last();
+
+ private Last() {}
+
+ @Override
+ public String toString() {
+ return "LAST";
+ }
+ }
+
+ /** Column position AFTER means the specified column should be put after the given `column`. */
+ final class After implements ColumnPosition {
+ private final String columnName;
+
+ private After(String columnName) {
+ this.columnName = columnName;
+ }
+
+ public String columnName() {
+ return columnName;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (!(o instanceof After)) {
+ return false;
+ }
+ After after = (After) o;
+ return Objects.equals(columnName, after.columnName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(columnName);
+ }
+
+ @Override
+ public String toString() {
+ return String.format("AFTER %s", columnName);
+ }
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
index 329d113c2..af293f120 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
@@ -335,6 +335,23 @@ public final class TableInfo {
modifiedTime);
}
+ /** Replace a TableInfo with a new SchemaInfo. */
+ public TableInfo withNewSchema(SchemaInfo schemaInfo) {
+ return new TableInfo(
+ tablePath,
+ tableId,
+ schemaInfo.getSchemaId(),
+ schemaInfo.getSchema(),
+ bucketKeys,
+ partitionKeys,
+ numBuckets,
+ properties,
+ customProperties,
+ comment,
+ createdTime,
+ modifiedTime);
+ }
+
@Override
public boolean equals(Object o) {
if (o == null || getClass() != o.getClass()) {
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
index eaacc48a6..b6ed40eeb 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
@@ -21,7 +21,10 @@ import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.exception.CorruptMessageException;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.arrow.ArrowReader;
+import org.apache.fluss.row.columnar.ColumnarRow;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.fluss.types.DataType;
@@ -33,9 +36,11 @@ import org.apache.fluss.utils.crc.Crc32C;
import java.nio.ByteBuffer;
import java.util.NoSuchElementException;
+import java.util.function.Function;
import static org.apache.fluss.record.LogRecordBatchFormat.BASE_OFFSET_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.COMMIT_TIMESTAMP_OFFSET;
+import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_LENGTH;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_OVERHEAD;
@@ -208,6 +213,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
@Override
+ @SuppressWarnings("unchecked")
public CloseableIterator<LogRecord> records(ReadContext context) {
if (getRecordCount() == 0) {
return CloseableIterator.emptyIterator();
@@ -217,15 +223,26 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
long timestamp = commitTimestamp();
LogFormat logFormat = context.getLogFormat();
RowType rowType = context.getRowType(schemaId);
+ Function<InternalRow, InternalRow> outputRowProcessor =
+ (row) -> {
+ ProjectedRow projectedRow = context.getOutputProjectedRow(schemaId);
+ if (projectedRow != null) {
+ return projectedRow.replaceRow(row);
+ } else {
+ return row;
+ }
+ };
+
switch (logFormat) {
case ARROW:
return columnRecordIterator(
rowType,
+ outputRowProcessor,
context.getVectorSchemaRoot(schemaId),
context.getBufferAllocator(),
timestamp);
case INDEXED:
- return rowRecordIterator(rowType, timestamp);
+ return rowRecordIterator(rowType, outputRowProcessor, timestamp);
default:
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
@@ -252,7 +269,10 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
return MurmurHashUtils.hashBytes(segment, position, sizeInBytes());
}
- private CloseableIterator<LogRecord> rowRecordIterator(RowType rowType, long timestamp) {
+ private CloseableIterator<LogRecord> rowRecordIterator(
+ RowType rowType,
+ Function<InternalRow, InternalRow> outputRowProcessor,
+ long timestamp) {
DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
return new LogRecordIterator() {
int position = DefaultLogRecordBatch.this.position + recordBatchHeaderSize(magic);
@@ -261,8 +281,13 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
@Override
protected LogRecord readNext(long baseOffset) {
IndexedLogRecord logRecord =
- IndexedLogRecord.readFrom(
- segment, position, baseOffset + rowId, timestamp, fieldTypes);
+ ProjectedIndexedLogRecord.readFrom(
+ segment,
+ position,
+ baseOffset + rowId,
+ timestamp,
+ fieldTypes,
+ outputRowProcessor);
rowId++;
position += logRecord.getSizeInBytes();
return logRecord;
@@ -279,7 +304,11 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
private CloseableIterator<LogRecord> columnRecordIterator(
- RowType rowType, VectorSchemaRoot root, BufferAllocator allocator, long timestamp) {
+ RowType rowType,
+ Function<InternalRow, InternalRow> outputRowProcessor,
+ VectorSchemaRoot root,
+ BufferAllocator allocator,
+ long timestamp) {
boolean isAppendOnly = (attributes() & APPEND_ONLY_FLAG_MASK) > 0;
if (isAppendOnly) {
// append only batch, no change type vector,
@@ -290,7 +319,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
@Override
protected ChangeType getChangeType(int rowId) {
return ChangeType.APPEND_ONLY;
@@ -308,7 +337,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
@Override
protected ChangeType getChangeType(int rowId) {
return changeTypeVector.getChangeType(rowId);
@@ -322,10 +351,15 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
private final ArrowReader reader;
private final long timestamp;
private int rowId = 0;
+ private final Function<InternalRow, InternalRow> outputRowProcessor;
- private ArrowLogRecordIterator(ArrowReader reader, long timestamp) {
+ private ArrowLogRecordIterator(
+ ArrowReader reader,
+ long timestamp,
+ Function<InternalRow, InternalRow> outputRowProcessor) {
this.reader = reader;
this.timestamp = timestamp;
+ this.outputRowProcessor = outputRowProcessor;
}
protected abstract ChangeType getChangeType(int rowId);
@@ -337,12 +371,13 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
@Override
protected LogRecord readNext(long baseOffset) {
+ ColumnarRow originalRow = reader.read(rowId);
LogRecord record =
new GenericRecord(
baseOffset + rowId,
timestamp,
getChangeType(rowId),
- reader.read(rowId));
+ outputRowProcessor.apply(originalRow));
rowId++;
return record;
}
@@ -412,4 +447,38 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
throw new UnsupportedOperationException();
}
}
+
+ private static class ProjectedIndexedLogRecord extends IndexedLogRecord {
+ private final Function<InternalRow, InternalRow> outputRowProcessor;
+
+ ProjectedIndexedLogRecord(
+ long logOffset,
+ long timestamp,
+ DataType[] fieldTypes,
+ Function<InternalRow, InternalRow> outputRowProcessor) {
+ super(logOffset, timestamp, fieldTypes);
+ this.outputRowProcessor = outputRowProcessor;
+ }
+
+ public static IndexedLogRecord readFrom(
+ MemorySegment segment,
+ int position,
+ long logOffset,
+ long logTimestamp,
+ DataType[] colTypes,
+ Function<InternalRow, InternalRow> outputRowProcessor) {
+ int sizeInBytes = segment.getInt(position);
+ ProjectedIndexedLogRecord logRecord =
+ new ProjectedIndexedLogRecord(
+ logOffset, logTimestamp, colTypes, outputRowProcessor);
+ logRecord.pointTo(segment, position, sizeInBytes + LENGTH_LENGTH);
+ return logRecord;
+ }
+
+ @Override
+ public InternalRow getRow() {
+ InternalRow row = super.getRow();
+ return outputRowProcessor.apply(row);
+ }
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java
index b8aefc75b..8d41ec4c8 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultValueRecordBatch.java
@@ -163,6 +163,7 @@ public class DefaultValueRecordBatch implements ValueRecordBatch {
}
readRecords++;
ValueRecord rec = readNext();
+
if (readRecords == numRecords) {
// Validate that the actual size of the batch is equal to declared size
// by checking that after reading declared number of items, there no items left
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
index 1f889fff1..11c50bb3d 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
@@ -20,6 +20,7 @@ package org.apache.fluss.record;
import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.compression.ArrowCompressionInfo;
import org.apache.fluss.exception.InvalidColumnProjectionException;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.bytesview.MultiBytesView;
import org.apache.fluss.shaded.arrow.com.google.flatbuffers.FlatBufferBuilder;
import org.apache.fluss.shaded.arrow.org.apache.arrow.flatbuf.Buffer;
@@ -53,6 +54,7 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import static org.apache.fluss.record.DefaultLogRecordBatch.APPEND_ONLY_FLAG_MASK;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
@@ -66,6 +68,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.arrowChangeTypeOffset
import static org.apache.fluss.record.LogRecordBatchFormat.attributeOffset;
import static org.apache.fluss.record.LogRecordBatchFormat.recordBatchHeaderSize;
import static org.apache.fluss.record.LogRecordBatchFormat.recordsCountOffset;
+import static org.apache.fluss.record.LogRecordBatchFormat.schemaIdOffset;
import static org.apache.fluss.utils.FileUtils.readFully;
import static org.apache.fluss.utils.FileUtils.readFullyOrFail;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
@@ -82,7 +85,7 @@ public class FileLogProjection {
private static final int ARROW_HEADER_SIZE =
ARROW_IPC_CONTINUATION_LENGTH + ARROW_IPC_METADATA_SIZE_LENGTH;
- final Map<Long, ProjectionInfo> projectionsCache = new HashMap<>();
+ final Map<ProjectionKey, ProjectionInfo> projectionsCache = new HashMap<>();
ProjectionInfo currentProjection;
// shared resources for multiple projections
@@ -97,6 +100,10 @@ public class FileLogProjection {
private final ByteBuffer arrowHeaderBuffer = ByteBuffer.allocate(ARROW_HEADER_SIZE);
private ByteBuffer arrowMetadataBuffer;
+ private SchemaGetter schemaGetter;
+ private long tableId;
+ private ArrowCompressionInfo compressionInfo;
+ private int[] selectedFieldPositions;
public FileLogProjection() {
this.outputStream = new ByteArrayOutputStream();
@@ -109,60 +116,20 @@ public class FileLogProjection {
public void setCurrentProjection(
long tableId,
- RowType schema,
+ SchemaGetter schemaGetter,
ArrowCompressionInfo compressionInfo,
- int[] selectedFields) {
- if (projectionsCache.containsKey(tableId)) {
- // the schema and projection should identical for the same table id.
- currentProjection = projectionsCache.get(tableId);
- if (!Arrays.equals(currentProjection.selectedFields, selectedFields)
- || !currentProjection.schema.equals(schema)) {
- throw new InvalidColumnProjectionException(
- "The schema and projection should be identical for the same table id.");
- }
- return;
- }
-
- // initialize the projection util information
- Schema arrowSchema = ArrowUtils.toArrowSchema(schema);
- BitSet selection = toBitSet(arrowSchema.getFields().size(), selectedFields);
- List<Tuple2<Field, Boolean>> flattenedFields = new ArrayList<>();
- flattenFields(arrowSchema.getFields(), selection, flattenedFields);
- int totalFieldNodes = flattenedFields.size();
- int[] bufferLayoutCount = new int[totalFieldNodes];
- BitSet nodesProjection = new BitSet(totalFieldNodes);
- int totalBuffers = 0;
- for (int i = 0; i < totalFieldNodes; i++) {
- Field fieldNode = flattenedFields.get(i).f0;
- boolean selected = flattenedFields.get(i).f1;
- nodesProjection.set(i, selected);
- bufferLayoutCount[i] = TypeLayout.getTypeBufferCount(fieldNode.getType());
- totalBuffers += bufferLayoutCount[i];
- }
- BitSet buffersProjection = new BitSet(totalBuffers);
- int bufferIndex = 0;
- for (int i = 0; i < totalFieldNodes; i++) {
- if (nodesProjection.get(i)) {
- buffersProjection.set(bufferIndex, bufferIndex + bufferLayoutCount[i]);
- }
- bufferIndex += bufferLayoutCount[i];
- }
-
- Schema projectedArrowSchema = ArrowUtils.toArrowSchema(schema.project(selectedFields));
- ArrowBodyCompression bodyCompression =
- CompressionUtil.createBodyCompression(compressionInfo.createCompressionCodec());
- int metadataLength =
- ArrowUtils.estimateArrowMetadataLength(projectedArrowSchema, bodyCompression);
- currentProjection =
- new ProjectionInfo(
- nodesProjection,
- buffersProjection,
- bufferIndex,
- schema,
- metadataLength,
- bodyCompression,
- selectedFields);
- projectionsCache.put(tableId, currentProjection);
+ int[] selectedFieldIds) {
+ this.tableId = tableId;
+ this.schemaGetter = schemaGetter;
+ this.compressionInfo = compressionInfo;
+
+ // Currently, only add last column is supported.Thus selectedFieldPositions is always same
+ // from same selectedFieldIds.
+ // TODO: if support drop column or add column in middle, this selectedFieldPositions should
+ // be re-calculated for each schema.
+ this.selectedFieldPositions =
+ selectedFieldPositions(
+ schemaGetter.getLatestSchemaInfo().getSchema(), selectedFieldIds);
}
/**
@@ -173,7 +140,7 @@ public class FileLogProjection {
*/
public BytesViewLogRecords project(FileChannel channel, int start, int end, int maxBytes)
throws IOException {
- checkNotNull(currentProjection, "There is no projection registered yet.");
+
MultiBytesView.Builder builder = MultiBytesView.builder();
int position = start;
@@ -194,6 +161,10 @@ public class FileLogProjection {
byte magic = logHeaderBuffer.get(MAGIC_OFFSET);
int recordBatchHeaderSize = recordBatchHeaderSize(magic);
int batchSizeInBytes = LOG_OVERHEAD + logHeaderBuffer.getInt(LENGTH_OFFSET);
+ short schemaId = logHeaderBuffer.getShort(schemaIdOffset(magic));
+ setCurrentSchema(schemaId);
+ checkNotNull(currentProjection, "There is no projection registered yet.");
+
if (position > end - batchSizeInBytes) {
// the remaining bytes in the file are not enough to read a full batch
return new BytesViewLogRecords(builder.build());
@@ -434,6 +405,118 @@ public class FileLogProjection {
return logHeaderBuffer;
}
+ private void setCurrentSchema(short schemaId) {
+ org.apache.fluss.metadata.Schema schema = schemaGetter.getSchema(schemaId);
+ RowType rowType = schema.getRowType();
+ ProjectionKey projectionKey = new ProjectionKey(tableId, schemaId);
+ if (projectionsCache.containsKey(projectionKey)) {
+ // the schema and projection should identical for the same table id.
+ currentProjection = projectionsCache.get(projectionKey);
+ if (!Arrays.equals(currentProjection.selectedFieldPositions, selectedFieldPositions)
+ || !currentProjection.schema.equals(rowType)) {
+ throw new InvalidColumnProjectionException(
+ "The schema and projection should be identical for the same table id.");
+ }
+ return;
+ }
+
+ // initialize the projection util information
+ Schema arrowSchema = ArrowUtils.toArrowSchema(rowType);
+ BitSet selection = toBitSet(arrowSchema.getFields().size(), selectedFieldPositions);
+ List<Tuple2<Field, Boolean>> flattenedFields = new ArrayList<>();
+ flattenFields(arrowSchema.getFields(), selection, flattenedFields);
+ int totalFieldNodes = flattenedFields.size();
+ int[] bufferLayoutCount = new int[totalFieldNodes];
+ BitSet nodesProjection = new BitSet(totalFieldNodes);
+ int totalBuffers = 0;
+ for (int i = 0; i < totalFieldNodes; i++) {
+ Field fieldNode = flattenedFields.get(i).f0;
+ boolean selected = flattenedFields.get(i).f1;
+ nodesProjection.set(i, selected);
+ bufferLayoutCount[i] = TypeLayout.getTypeBufferCount(fieldNode.getType());
+ totalBuffers += bufferLayoutCount[i];
+ }
+ BitSet buffersProjection = new BitSet(totalBuffers);
+ int bufferIndex = 0;
+ for (int i = 0; i < totalFieldNodes; i++) {
+ if (nodesProjection.get(i)) {
+ buffersProjection.set(bufferIndex, bufferIndex + bufferLayoutCount[i]);
+ }
+ bufferIndex += bufferLayoutCount[i];
+ }
+
+ Schema projectedArrowSchema =
+ ArrowUtils.toArrowSchema(rowType.project(selectedFieldPositions));
+ ArrowBodyCompression bodyCompression =
+ CompressionUtil.createBodyCompression(compressionInfo.createCompressionCodec());
+ int metadataLength =
+ ArrowUtils.estimateArrowMetadataLength(projectedArrowSchema, bodyCompression);
+ currentProjection =
+ new ProjectionInfo(
+ nodesProjection,
+ buffersProjection,
+ bufferIndex,
+ rowType,
+ metadataLength,
+ bodyCompression,
+ selectedFieldPositions);
+ projectionsCache.put(projectionKey, currentProjection);
+ }
+
+ int[] selectedFieldPositions(org.apache.fluss.metadata.Schema schema, int[] projectedFields) {
+ Map<Integer, Integer> columnIdPositions = new HashMap<>();
+ List<Integer> columnIds = schema.getColumnIds();
+ for (int i = 0; i < columnIds.size(); i++) {
+ columnIdPositions.put(columnIds.get(i), i);
+ }
+
+ int prev = -1;
+ int[] selectedFieldPositions = new int[projectedFields.length];
+ for (int i = 0; i < projectedFields.length; i++) {
+ int fieldId = projectedFields[i];
+ Integer position = columnIdPositions.get(fieldId);
+ if (position == null) {
+ throw new InvalidColumnProjectionException(
+ String.format(
+ "Projected field id %s is not contains in %s", fieldId, columnIds));
+ }
+
+ selectedFieldPositions[i] = position;
+ if (position < prev) {
+ throw new InvalidColumnProjectionException(
+ "The projection indexes should be in field order, but is "
+ + Arrays.toString(projectedFields));
+ }
+
+ prev = position;
+ }
+ return selectedFieldPositions;
+ }
+
+ static final class ProjectionKey {
+ private final long tableId;
+ private final short schemaId;
+
+ ProjectionKey(long tableId, short schemaId) {
+ this.tableId = tableId;
+ this.schemaId = schemaId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ProjectionKey)) {
+ return false;
+ }
+ ProjectionKey that = (ProjectionKey) o;
+ return tableId == that.tableId && schemaId == that.schemaId;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableId, schemaId);
+ }
+ }
+
static final class ProjectionInfo {
final BitSet nodesProjection;
final BitSet buffersProjection;
@@ -441,7 +524,7 @@ public class FileLogProjection {
final RowType schema;
final int arrowMetadataLength;
final ArrowBodyCompression bodyCompression;
- final int[] selectedFields;
+ final int[] selectedFieldPositions;
private ProjectionInfo(
BitSet nodesProjection,
@@ -450,14 +533,14 @@ public class FileLogProjection {
RowType schema,
int arrowMetadataLength,
ArrowBodyCompression bodyCompression,
- int[] selectedFields) {
+ int[] selectedFieldPositions) {
this.nodesProjection = nodesProjection;
this.buffersProjection = buffersProjection;
this.bufferCount = bufferCount;
this.schema = schema;
this.arrowMetadataLength = arrowMetadataLength;
this.bodyCompression = bodyCompression;
- this.selectedFields = selectedFields;
+ this.selectedFieldPositions = selectedFieldPositions;
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java b/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java
index 385e457b8..88bc6a3c7 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/IndexedLogRecord.java
@@ -73,7 +73,7 @@ public class IndexedLogRecord implements LogRecord {
this.timestamp = timestamp;
}
- private void pointTo(MemorySegment segment, int offset, int sizeInBytes) {
+ void pointTo(MemorySegment segment, int offset, int sizeInBytes) {
this.segment = segment;
this.offset = offset;
this.sizeInBytes = sizeInBytes;
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
index a0cc396b1..5fffc3f4f 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/KvRecordReadContext.java
@@ -18,25 +18,40 @@
package org.apache.fluss.record;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.types.DataType;
+import java.util.HashMap;
+import java.util.Map;
+
/** A default implementation of {@link KvRecordBatch.ReadContext} . */
public class KvRecordReadContext implements KvRecordBatch.ReadContext {
- private final RowDecoder rowDecoder;
+ private final KvFormat kvFormat;
+ private final SchemaGetter schemaGetter;
+ private final Map<Integer, RowDecoder> rowDecoderCache;
- private KvRecordReadContext(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
+ private KvRecordReadContext(KvFormat kvFormat, SchemaGetter schemaGetter) {
+ this.kvFormat = kvFormat;
+ this.schemaGetter = schemaGetter;
+ this.rowDecoderCache = new HashMap<>();
}
- public static KvRecordReadContext createReadContext(KvFormat kvFormat, DataType[] fieldTypes) {
- return new KvRecordReadContext(RowDecoder.create(kvFormat, fieldTypes));
+ public static KvRecordReadContext createReadContext(
+ KvFormat kvFormat, SchemaGetter schemaGetter) {
+ return new KvRecordReadContext(kvFormat, schemaGetter);
}
@Override
public RowDecoder getRowDecoder(int schemaId) {
- // assume schema id are always not changed for now.
- return rowDecoder;
+ return rowDecoderCache.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema schema = schemaGetter.getSchema((short) schemaId);
+ return RowDecoder.create(
+ kvFormat, schema.getRowType().getChildren().toArray(new DataType[0]));
+ });
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java
index cadc1ef9d..06536e582 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordBatch.java
@@ -19,11 +19,14 @@ package org.apache.fluss.record;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
+import javax.annotation.Nullable;
+
import java.util.Iterator;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V0;
@@ -195,5 +198,18 @@ public interface LogRecordBatch {
/** Gets the buffer allocator. */
BufferAllocator getBufferAllocator();
+
+ /**
+ * If the read context defines an output projection (for example, log records may add new
+ * columns or reorder columns, but reader need a static schema for the output rows), return
+ * a {@link ProjectedRow} that describes the projected output row for the given schemaId.
+ * The returned object is used by readers to transform or materialize rows according to the
+ * output projection. Returns {@code null} if no output projection is configured.
+ *
+ * @param schemaId the current row schema id
+ * @return a {@link ProjectedRow} describing the output projection, or {@code null} if none
+ */
+ @Nullable
+ ProjectedRow getOutputProjectedRow(int schemaId);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
index 1158dbc35..50ec7ee66 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
@@ -19,9 +19,12 @@ package org.apache.fluss.record;
import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.InternalRow.FieldGetter;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.RootAllocator;
import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
@@ -32,10 +35,11 @@ import org.apache.fluss.utils.Projection;
import javax.annotation.Nullable;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.stream.IntStream;
-import static org.apache.fluss.utils.Preconditions.checkArgument;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** A simple implementation for {@link LogRecordBatch.ReadContext}. */
@@ -46,7 +50,7 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
// the schema of the date read form server or remote. (which is projected in the server side)
private final RowType dataRowType;
// the static schemaId of the table, should support dynamic schema evolution in the future
- private final int schemaId;
+ private final int expectedSchemaId;
// the Arrow vector schema root of the table, should be null if not ARROW log format
@Nullable private final VectorSchemaRoot vectorSchemaRoot;
// the Arrow memory buffer allocator for the table, should be null if not ARROW log format
@@ -55,12 +59,14 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
private final FieldGetter[] selectedFieldGetters;
// whether the projection is push downed to the server side and the returned data is pruned.
private final boolean projectionPushDowned;
+ private final SchemaGetter schemaGetter;
+ private final Map<Integer, VectorSchemaRoot> vectorSchemaRootMap;
- /**
- * Creates a LogRecordReadContext for the given table information and projection information.
- */
public static LogRecordReadContext createReadContext(
- TableInfo tableInfo, boolean readFromRemote, @Nullable Projection projection) {
+ TableInfo tableInfo,
+ boolean readFromRemote,
+ @Nullable Projection projection,
+ SchemaGetter schemaGetter) {
RowType rowType = tableInfo.getRowType();
LogFormat logFormat = tableInfo.getTableConfig().getLogFormat();
// only for arrow log format, the projection can be push downed to the server side
@@ -68,33 +74,45 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
int schemaId = tableInfo.getSchemaId();
if (projection == null) {
// set a default dummy projection to simplify code
- projection = Projection.of(IntStream.range(0, rowType.getFieldCount()).toArray());
+ projection =
+ Projection.of(
+ IntStream.range(0, rowType.getFieldCount()).toArray(),
+ tableInfo.getSchema());
}
if (logFormat == LogFormat.ARROW) {
if (readFromRemote) {
// currently, for remote read, arrow log doesn't support projection pushdown,
// so set the rowType as is.
- int[] selectedFields = projection.getProjection();
- return createArrowReadContext(rowType, schemaId, selectedFields, false);
+ int[] selectedFields = projection.getProjectionPositions();
+ return createArrowReadContext(
+ rowType, schemaId, selectedFields, false, schemaGetter);
} else {
// arrow data that returned from server has been projected (in order)
RowType projectedRowType = projection.projectInOrder(rowType);
// need to reorder the fields for final output
int[] selectedFields = projection.getReorderingIndexes();
return createArrowReadContext(
- projectedRowType, schemaId, selectedFields, projectionPushDowned);
+ projectedRowType,
+ schemaId,
+ selectedFields,
+ projectionPushDowned,
+ schemaGetter);
}
} else if (logFormat == LogFormat.INDEXED) {
- int[] selectedFields = projection.getProjection();
- return createIndexedReadContext(rowType, schemaId, selectedFields);
+ int[] selectedFields = projection.getProjectionPositions();
+ return createIndexedReadContext(rowType, schemaId, selectedFields, schemaGetter);
} else {
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
}
private static LogRecordReadContext createArrowReadContext(
- RowType dataRowType, int schemaId, int[] selectedFields, boolean projectionPushDowned) {
+ RowType dataRowType,
+ int schemaId,
+ int[] selectedFields,
+ boolean projectionPushDowned,
+ SchemaGetter schemaGetter) {
// TODO: use a more reasonable memory limit
BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
VectorSchemaRoot vectorRoot =
@@ -107,31 +125,28 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
vectorRoot,
allocator,
fieldGetters,
- projectionPushDowned);
+ projectionPushDowned,
+ schemaGetter);
}
/**
- * Creates a testing purpose LogRecordReadContext for ARROW log format, that underlying Arrow
- * resources are not reused.
+ * Creates a LogRecordReadContext for INDEXED log format.
*
* @param rowType the schema of the table
* @param schemaId the schemaId of the table
*/
@VisibleForTesting
- public static LogRecordReadContext createArrowReadContext(RowType rowType, int schemaId) {
+ public static LogRecordReadContext createArrowReadContext(
+ RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createArrowReadContext(rowType, schemaId, selectedFields, false);
+ return createArrowReadContext(rowType, schemaId, selectedFields, false, schemaGetter);
}
- /**
- * Creates a LogRecordReadContext for INDEXED log format.
- *
- * @param rowType the schema of the table
- * @param schemaId the schemaId of the table
- */
- public static LogRecordReadContext createIndexedReadContext(RowType rowType, int schemaId) {
+ @VisibleForTesting
+ public static LogRecordReadContext createIndexedReadContext(
+ RowType rowType, int schemaId, SchemaGetter schemaGetter) {
int[] selectedFields = IntStream.range(0, rowType.getFieldCount()).toArray();
- return createIndexedReadContext(rowType, schemaId, selectedFields);
+ return createIndexedReadContext(rowType, schemaId, selectedFields, schemaGetter);
}
/**
@@ -142,28 +157,38 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
* @param selectedFields the final selected fields of the read data
*/
public static LogRecordReadContext createIndexedReadContext(
- RowType rowType, int schemaId, int[] selectedFields) {
+ RowType rowType, int schemaId, int[] selectedFields, SchemaGetter schemaGetter) {
FieldGetter[] fieldGetters = buildProjectedFieldGetters(rowType, selectedFields);
// for INDEXED log format, the projection is NEVER push downed to the server side
return new LogRecordReadContext(
- LogFormat.INDEXED, rowType, schemaId, null, null, fieldGetters, false);
+ LogFormat.INDEXED,
+ rowType,
+ schemaId,
+ null,
+ null,
+ fieldGetters,
+ false,
+ schemaGetter);
}
private LogRecordReadContext(
LogFormat logFormat,
- RowType dataRowType,
- int schemaId,
+ RowType targetDataRowType,
+ int expectedSchemaId,
VectorSchemaRoot vectorSchemaRoot,
BufferAllocator bufferAllocator,
FieldGetter[] selectedFieldGetters,
- boolean projectionPushDowned) {
+ boolean projectionPushDowned,
+ SchemaGetter schemaGetter) {
this.logFormat = logFormat;
- this.dataRowType = dataRowType;
- this.schemaId = schemaId;
+ this.dataRowType = targetDataRowType;
+ this.expectedSchemaId = expectedSchemaId;
this.vectorSchemaRoot = vectorSchemaRoot;
this.bufferAllocator = bufferAllocator;
this.selectedFieldGetters = selectedFieldGetters;
this.projectionPushDowned = projectionPushDowned;
+ this.schemaGetter = schemaGetter;
+ this.vectorSchemaRootMap = new HashMap<>();
}
@Override
@@ -173,12 +198,12 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
@Override
public RowType getRowType(int schemaId) {
- checkArgument(
- schemaId == this.schemaId,
- "The schemaId (%s) in the record batch is not the same as the context (%s).",
- schemaId,
- this.schemaId);
- return dataRowType;
+ if (isSameRowType(schemaId)) {
+ return dataRowType;
+ }
+
+ Schema schema = schemaGetter.getSchema(schemaId);
+ return schema.getRowType();
}
/** Get the selected field getters for the read data. */
@@ -193,17 +218,22 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
@Override
public VectorSchemaRoot getVectorSchemaRoot(int schemaId) {
- checkArgument(
- schemaId == this.schemaId,
- "The schemaId (%s) in the record batch is not the same as the context (%s).",
- schemaId,
- this.schemaId);
if (logFormat != LogFormat.ARROW) {
throw new IllegalArgumentException(
"Only Arrow log format provides vector schema root.");
}
- checkNotNull(vectorSchemaRoot, "The vector schema root is not available.");
- return vectorSchemaRoot;
+
+ if (schemaId == this.expectedSchemaId) {
+ checkNotNull(vectorSchemaRoot, "The vector schema root is not available.");
+ return vectorSchemaRoot;
+ }
+
+ RowType rowType = getRowType(schemaId);
+ return vectorSchemaRootMap.computeIfAbsent(
+ schemaId,
+ (id) ->
+ VectorSchemaRoot.create(
+ ArrowUtils.toArrowSchema(rowType), bufferAllocator));
}
@Override
@@ -215,15 +245,31 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
return bufferAllocator;
}
+ @Nullable
+ @Override
+ public ProjectedRow getOutputProjectedRow(int schemaId) {
+ if (isSameRowType(schemaId)) {
+ return null;
+ }
+ Schema originSchema = schemaGetter.getSchema(schemaId);
+ Schema expectedSchema = schemaGetter.getSchema(expectedSchemaId);
+ return ProjectedRow.from(originSchema, expectedSchema);
+ }
+
public void close() {
if (vectorSchemaRoot != null) {
vectorSchemaRoot.close();
}
+ vectorSchemaRootMap.values().forEach(VectorSchemaRoot::close);
if (bufferAllocator != null) {
bufferAllocator.close();
}
}
+ private boolean isSameRowType(int schemaId) {
+ return expectedSchemaId == schemaId || isProjectionPushDowned();
+ }
+
private static FieldGetter[] buildProjectedFieldGetters(RowType rowType, int[] selectedFields) {
List<DataType> dataTypeList = rowType.getChildren();
FieldGetter[] fieldGetters = new FieldGetter[selectedFields.length];
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java
index c1a1a0af9..e00559e00 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/ValueRecordReadContext.java
@@ -18,26 +18,39 @@
package org.apache.fluss.record;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.types.DataType;
+import java.util.HashMap;
+import java.util.Map;
+
/** A default implementation of {@link ValueRecordBatch.ReadContext} . */
public class ValueRecordReadContext implements ValueRecordBatch.ReadContext {
+ private final Map<Integer, RowDecoder> rowDecoderCache;
+ private final SchemaGetter schemaGetter;
+ private final KvFormat kvFormat;
- private final RowDecoder rowDecoder;
-
- public ValueRecordReadContext(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
+ private ValueRecordReadContext(SchemaGetter schemaGetter, KvFormat kvFormat) {
+ this.rowDecoderCache = new HashMap<>();
+ this.schemaGetter = schemaGetter;
+ this.kvFormat = kvFormat;
}
public static ValueRecordReadContext createReadContext(
- KvFormat kvFormat, DataType[] fieldTypes) {
- return new ValueRecordReadContext(RowDecoder.create(kvFormat, fieldTypes));
+ SchemaGetter schemaGetter, KvFormat kvFormat) {
+ return new ValueRecordReadContext(schemaGetter, kvFormat);
}
@Override
public RowDecoder getRowDecoder(int schemaId) {
- // assume schema id are always not changed for now.
- return rowDecoder;
+ return rowDecoderCache.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema schema = schemaGetter.getSchema(schemaId);
+ return RowDecoder.create(
+ kvFormat, schema.getRowType().getChildren().toArray(new DataType[0]));
+ });
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
index 0de55354f..6b4c516d1 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
@@ -18,9 +18,12 @@
package org.apache.fluss.row;
import org.apache.fluss.annotation.PublicEvolving;
+import org.apache.fluss.metadata.Schema;
import java.util.Arrays;
+import static org.apache.fluss.utils.SchemaUtil.getIndexMapping;
+
/**
* An implementation of {@link InternalRow} which provides a projected view of the underlying {@link
* InternalRow}.
@@ -31,12 +34,13 @@ import java.util.Arrays;
*/
@PublicEvolving
public class ProjectedRow implements InternalRow {
+ public static final int UNEXIST_MAPPING = -1;
- private final int[] indexMapping;
+ protected final int[] indexMapping;
- private InternalRow row;
+ protected InternalRow row;
- private ProjectedRow(int[] indexMapping) {
+ protected ProjectedRow(int[] indexMapping) {
this.indexMapping = indexMapping;
}
@@ -60,6 +64,9 @@ public class ProjectedRow implements InternalRow {
@Override
public boolean isNullAt(int pos) {
+ if (indexMapping[pos] == UNEXIST_MAPPING) {
+ return true;
+ }
return row.isNullAt(indexMapping[pos]);
}
@@ -143,6 +150,12 @@ public class ProjectedRow implements InternalRow {
@Override
public boolean equals(Object o) {
+ // only can compare when the index mapping and row are equal.
+ if (o instanceof ProjectedRow
+ && Arrays.equals(indexMapping, ((ProjectedRow) o).indexMapping)
+ && row.equals(((ProjectedRow) o).row)) {
+ return true;
+ }
throw new UnsupportedOperationException("Projected row data cannot be compared");
}
@@ -168,4 +181,9 @@ public class ProjectedRow implements InternalRow {
public static ProjectedRow from(int[] projection) {
return new ProjectedRow(projection);
}
+
+ public static ProjectedRow from(Schema originSchema, Schema expectedSchema) {
+ int[] indexMapping = getIndexMapping(originSchema, expectedSchema);
+ return new ProjectedRow(indexMapping);
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
index 2fdc32f8e..e0252ae11 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
@@ -18,10 +18,17 @@
package org.apache.fluss.row.encode;
import org.apache.fluss.memory.MemorySegment;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.decode.RowDecoder;
+import org.apache.fluss.types.DataType;
+
+import java.util.Map;
import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH;
+import static org.apache.fluss.utils.MapUtils.newConcurrentHashMap;
/**
* A decoder to decode a schema id and {@link BinaryRow} from a byte array value which is encoded by
@@ -29,21 +36,31 @@ import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH;
*/
public class ValueDecoder {
- // todo: the row decoder should be inferred from the schema id encoded in the value
- private final RowDecoder rowDecoder;
-
- public ValueDecoder(RowDecoder rowDecoder) {
- this.rowDecoder = rowDecoder;
- }
+ private final Map<Short, RowDecoder> rowDecoders;
+ private final SchemaGetter schemaGetter;
+ private final KvFormat kvFormat;
- public RowDecoder getRowDecoder() {
- return rowDecoder;
+ public ValueDecoder(SchemaGetter schemaGetter, KvFormat kvFormat) {
+ this.rowDecoders = newConcurrentHashMap();
+ this.schemaGetter = schemaGetter;
+ this.kvFormat = kvFormat;
}
/** Decode the value bytes and return the schema id and the row encoded in the value bytes. */
public Value decodeValue(byte[] valueBytes) {
MemorySegment memorySegment = MemorySegment.wrap(valueBytes);
short schemaId = memorySegment.getShort(0);
+
+ RowDecoder rowDecoder =
+ rowDecoders.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema schema = schemaGetter.getSchema(schemaId);
+ return RowDecoder.create(
+ kvFormat,
+ schema.getRowType().getChildren().toArray(new DataType[0]));
+ });
+
BinaryRow row =
rowDecoder.decode(
memorySegment, SCHEMA_ID_LENGTH, valueBytes.length - SCHEMA_ID_LENGTH);
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
index 80fea90fa..bbfc6a7f5 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
@@ -17,7 +17,10 @@
package org.apache.fluss.row.encode;
+import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.UnsafeUtils;
/** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */
@@ -38,4 +41,24 @@ public class ValueEncoder {
row.copyTo(values, SCHEMA_ID_LENGTH);
return values;
}
+
+ public static byte[] encodeRow(
+ short schemaId, KvFormat kvFormat, RowType currentRowType, InternalRow row)
+ throws Exception {
+ if (row instanceof BinaryRow) {
+ return encodeValue(schemaId, (BinaryRow) row);
+ }
+ // todo: reuse the encoder here
+ try (RowEncoder rowEncoder = RowEncoder.create(kvFormat, currentRowType)) {
+ rowEncoder.startNewRow();
+ for (int i = 0; i < currentRowType.getFieldCount(); i++) {
+ rowEncoder.encodeField(
+ i,
+ InternalRow.createFieldGetter(currentRowType.getTypeAt(i), i)
+ .getFieldOrNull(row));
+ }
+ BinaryRow binaryRow = rowEncoder.finishRow();
+ return encodeValue(schemaId, binaryRow);
+ }
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/types/RowType.java b/fluss-common/src/main/java/org/apache/fluss/types/RowType.java
index a282c3286..090fe9655 100644
--- a/fluss-common/src/main/java/org/apache/fluss/types/RowType.java
+++ b/fluss-common/src/main/java/org/apache/fluss/types/RowType.java
@@ -98,17 +98,17 @@ public final class RowType extends DataType {
return -1;
}
- public RowType project(int[] projectFields) {
+ public RowType project(int[] projectFieldPositions) {
List<DataField> projectedFields = new ArrayList<>();
- for (int projectField : projectFields) {
+ for (int projectField : projectFieldPositions) {
projectedFields.add(this.fields.get(projectField));
}
return new RowType(this.isNullable(), projectedFields);
}
- public RowType project(List<String> projectFields) {
+ public RowType project(List<String> projectFieldNames) {
List<DataField> projectedFields = new ArrayList<>();
- for (String projectField : projectFields) {
+ for (String projectField : projectFieldNames) {
int index = getFieldIndex(projectField);
if (index == -1) {
throw new IllegalArgumentException(
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
index c16c6b10b..c597caf81 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
@@ -17,9 +17,11 @@
package org.apache.fluss.utils;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.types.RowType;
import java.util.Arrays;
+import java.util.List;
import static org.apache.fluss.utils.Preconditions.checkState;
@@ -28,12 +30,14 @@ import static org.apache.fluss.utils.Preconditions.checkState;
* projection includes both reducing the accessible fields and reordering them. Currently, this only
* supports top-level projection. Nested projection will be supported in the future.
*
- * <p>For example, given a row with fields [a, b, c, d, e], a projection [2, 0, 3] will project the
- * row to [c, a, d]. The projection indexes are 0-based.
+ * <p>For example, given a row with fields [a, b, c, d, e] with id [0, 1, 3, 4, 5], a projection [2,
+ * 0, 3] will project the row to [c, a, d] with id [3, 0, 4]. The projection indexes are 0-based.
*
* <ul>
- * <li>The {@link #projection} indexes will be [2, 0, 3]
- * <li>The {@link #projectionInOrder} indexes will be [0, 2, 3]
+ * <li>The {@link #projectionPositions} indexes will be [2, 0, 3]
+ * <li>The {@link #projectionPositions} id will be [3, 0, 4]
+ * <li>The {@link #projectionIdsInOrder} indexes will be [0, 3, 4]
+ * <li>The {@link #projectionPositionsOrderById} indexes will be [0, 2, 3]
* <li>The {@link #reorderingIndexes} indexes will be [1, 0, 2]
* </ul>
*
@@ -41,42 +45,80 @@ import static org.apache.fluss.utils.Preconditions.checkState;
*/
public class Projection {
/** the projection indexes including both selected fields and reordering them. */
- final int[] projection;
+ final int[] projectionPositions;
/** the projection indexes that only select fields but not reordering them. */
- final int[] projectionInOrder;
- /** the indexes to reorder the fields of {@link #projectionInOrder} to {@link #projection}. */
+ final int[] projectionPositionsOrderById;
+ /** the projection indexes that only select fields but not reordering them. */
+ final int[] projectionIdsInOrder;
+ /**
+ * the indexes to reorder the fields of {@link #projectionPositionsOrderById} to {@link
+ * #projectionPositions}.
+ */
final int[] reorderingIndexes;
/** the flag to indicate whether reordering is needed. */
final boolean reorderingNeeded;
- /** Create a {@link Projection} of the provided {@code indexes}. */
- public static Projection of(int[] indexes) {
- return new Projection(indexes);
+ public static Projection of(int[] indexes, Schema schema) {
+ int[] ids = new int[indexes.length];
+ List<Integer> columnIds = schema.getColumnIds();
+ for (int i = 0; i < indexes.length; i++) {
+ ids[i] = columnIds.get(indexes[i]);
+ }
+
+ return new Projection(indexes, ids);
}
- private Projection(int[] projection) {
- this.projection = projection;
- this.projectionInOrder = Arrays.copyOf(projection, projection.length);
- Arrays.sort(projectionInOrder);
- this.reorderingNeeded = !Arrays.equals(projection, projectionInOrder);
- this.reorderingIndexes = new int[projection.length];
- for (int i = 0; i < projection.length; i++) {
- int index = Arrays.binarySearch(projectionInOrder, projection[i]);
+ /**
+ * Create a {@link Projection} of the provided {@code indexes} and {@code projectionIds}.
+ *
+ * <p>Typically, {@code projectionIndexes} and {@code projectionIds} are the same. But when
+ * removing a middle column or reordering columns, they won't be the same. In this case, when
+ * the schema on the fluss server side differs from the schema on the client side during user
+ * queries, using {@code projectionIds} ensures correctness by mapping to the actual server-side
+ * column positions.
+ *
+ * @param projectionIndexes the indexes of the projection, which is used for user to read data
+ * from client side. These indexes are based on the schema visible to the client side and
+ * are used to parse the data returned by the fluss server.
+ * @param projectionIds the ids of the projection, which is used for fluss server scan. These
+ * ids are based on the actual schema on the fluss server side and are pushed down to the
+ * server for querying.
+ */
+ private Projection(int[] projectionIndexes, int[] projectionIds) {
+ checkState(
+ projectionIds.length == projectionIndexes.length,
+ "The number of ids must be equal to the number of indexes.");
+ this.projectionPositions = projectionIndexes;
+
+ // reorder the projection id for lookup.
+ this.projectionIdsInOrder = Arrays.copyOf(projectionIds, projectionIds.length);
+ Arrays.sort(projectionIdsInOrder);
+ this.reorderingNeeded = !Arrays.equals(projectionIds, projectionIdsInOrder);
+ this.reorderingIndexes = new int[projectionPositions.length];
+ this.projectionPositionsOrderById = new int[projectionPositions.length];
+ for (int i = 0; i < projectionIds.length; i++) {
+ int index = Arrays.binarySearch(projectionIdsInOrder, projectionIds[i]);
checkState(index >= 0, "The projection index is invalid.");
reorderingIndexes[i] = index;
+ projectionPositionsOrderById[i] = projectionIndexes[index];
}
}
public RowType projectInOrder(RowType rowType) {
- return rowType.project(projectionInOrder);
+ return rowType.project(projectionPositionsOrderById);
}
- public int[] getProjection() {
- return projection;
+ public int[] getProjectionPositions() {
+ return projectionPositions;
}
- public int[] getProjectionInOrder() {
- return projectionInOrder;
+ /**
+ * The id of the projection, which is used for fluss server scan.
+ *
+ * @return
+ */
+ public int[] getProjectionIdInOrder() {
+ return projectionIdsInOrder;
}
public boolean isReorderingNeeded() {
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
new file mode 100644
index 000000000..6b66c0cd6
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.utils;
+
+import org.apache.fluss.exception.SchemaChangeException;
+import org.apache.fluss.metadata.Schema;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+
+import static org.apache.fluss.row.ProjectedRow.UNEXIST_MAPPING;
+
+/** Schema util. */
+public class SchemaUtil {
+
+ /**
+ * get the target columns in expected schema.
+ *
+ * @param targetColumns
+ * @param originSchema
+ * @param expectedSchema
+ * @return
+ */
+ public static int[] getTargetColumns(
+ int[] targetColumns, Schema originSchema, Schema expectedSchema) {
+ if (targetColumns == null) {
+ return null;
+ }
+
+ // get the mapping from expected schema to origin schema
+ int[] indexMapping = getIndexMapping(expectedSchema, originSchema);
+ ArrayList<Integer> targetColumnsInExpectedSchema = new ArrayList<>();
+ for (int i = 0; i < targetColumns.length; i++) {
+ if (indexMapping[targetColumns[i]] != UNEXIST_MAPPING) {
+ targetColumnsInExpectedSchema.add(indexMapping[targetColumns[i]]);
+ }
+ }
+ return targetColumnsInExpectedSchema.stream().mapToInt(value -> value).toArray();
+ }
+
+ /**
+ * Get the index of each value of expectedSchema from originSchema.
+ *
+ * @param originSchema
+ * @param expectedSchema
+ * @return indexMapping the value of i-th means the expectedSchema's i-th column is the
+ * originSchema's indexMapping[i]-th column.
+ */
+ public static int[] getIndexMapping(Schema originSchema, Schema expectedSchema) {
+ List<Schema.Column> originColumns = originSchema.getColumns();
+ Map<Integer, Integer> originColumnIdToIndex = new HashMap<>();
+ for (int i = 0; i < originColumns.size(); i++) {
+ originColumnIdToIndex.put(originColumns.get(i).getColumnId(), i);
+ }
+ List<Schema.Column> expectedColumns = expectedSchema.getColumns();
+
+ int[] indexMapping = new int[expectedColumns.size()];
+ for (int i = 0; i < expectedColumns.size(); i++) {
+ Schema.Column expectedColumn = expectedColumns.get(i);
+ indexMapping[i] =
+ originColumnIdToIndex.getOrDefault(
+ expectedColumn.getColumnId(), UNEXIST_MAPPING);
+ Schema.Column originColumn =
+ indexMapping[i] == UNEXIST_MAPPING ? null : originColumns.get(indexMapping[i]);
+
+ if (originColumn != null
+ && !Objects.equals(
+ expectedColumn.getDataType().copy(true),
+ originColumns.get(indexMapping[i]).getDataType().copy(true))) {
+
+ throw new SchemaChangeException(
+ String.format(
+ "Expected datatype of column(id=%s,name=%s) is [%s], while the actual datatype is [%s]",
+ expectedColumn.getColumnId(),
+ expectedColumn.getName(),
+ expectedColumn.getDataType(),
+ originColumn.getDataType()));
+ }
+ }
+
+ return indexMapping;
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java b/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java
index 258b96007..5c8fdb6c5 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/UnsafeUtils.java
@@ -74,6 +74,10 @@ public class UnsafeUtils {
return MemoryUtils.UNSAFE.getInt(target, BYTE_ARRAY_BASE_OFFSET + offset);
}
+ public static short getShort(byte[] target, long offset) {
+ return MemoryUtils.UNSAFE.getShort(target, BYTE_ARRAY_BASE_OFFSET + offset);
+ }
+
public static long getLong(byte[] target, long offset) {
return MemoryUtils.UNSAFE.getLong(target, BYTE_ARRAY_BASE_OFFSET + offset);
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
index 1994ba61a..5db5b65ed 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/json/ColumnJsonSerde.java
@@ -25,6 +25,8 @@ import org.apache.fluss.types.DataType;
import java.io.IOException;
+import static org.apache.fluss.metadata.Schema.Column.UNKNOWN_COLUMN_ID;
+
/** Json serializer and deserializer for {@link Schema.Column}. */
@Internal
public class ColumnJsonSerde
@@ -32,6 +34,7 @@ public class ColumnJsonSerde
public static final ColumnJsonSerde INSTANCE = new ColumnJsonSerde();
static final String NAME = "name";
+ static final String ID = "id";
static final String DATA_TYPE = "data_type";
static final String COMMENT = "comment";
@@ -46,6 +49,7 @@ public class ColumnJsonSerde
if (column.getComment().isPresent()) {
generator.writeStringField(COMMENT, column.getComment().get());
}
+ generator.writeNumberField(ID, column.getColumnId());
generator.writeEndObject();
}
@@ -55,11 +59,11 @@ public class ColumnJsonSerde
String columnName = node.required(NAME).asText();
DataType dataType = DataTypeJsonSerde.INSTANCE.deserialize(node.get(DATA_TYPE));
- Schema.Column column = new Schema.Column(columnName, dataType);
- if (node.hasNonNull(COMMENT)) {
- column = column.withComment(node.get(COMMENT).asText());
- }
- return column;
+ return new Schema.Column(
+ columnName,
+ dataType,
+ node.hasNonNull(COMMENT) ? node.get(COMMENT).asText() : null,
+ node.has(ID) ? node.get(ID).asInt() : UNKNOWN_COLUMN_ID);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java b/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java
index a4e5fd8ad..527247b6e 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/json/SchemaJsonSerde.java
@@ -37,6 +37,7 @@ public class SchemaJsonSerde implements JsonSerializer<Schema>, JsonDeserializer
private static final String COLUMNS_NAME = "columns";
private static final String PRIMARY_KEY_NAME = "primary_key";
private static final String VERSION_KEY = "version";
+ private static final String HIGHEST_FIELD_ID = "highest_field_id";
private static final int VERSION = 1;
@Override
@@ -62,6 +63,8 @@ public class SchemaJsonSerde implements JsonSerializer<Schema>, JsonDeserializer
generator.writeEndArray();
}
+ generator.writeNumberField(HIGHEST_FIELD_ID, schema.getHighestFieldId());
+
generator.writeEndObject();
}
@@ -83,6 +86,10 @@ public class SchemaJsonSerde implements JsonSerializer<Schema>, JsonDeserializer
builder.primaryKey(primaryKeys);
}
+ if (node.has(HIGHEST_FIELD_ID)) {
+ builder.highestFieldId(node.get(HIGHEST_FIELD_ID).asInt());
+ }
+
return builder.build();
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java
index d3e9cf2f6..6c8b483a0 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordBatchTest.java
@@ -27,6 +27,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link DefaultKvRecordBatch}. */
@@ -67,7 +68,7 @@ class DefaultKvRecordBatchTest extends KvTestBase {
for (KvRecord record :
kvRecords.records(
KvRecordReadContext.createReadContext(
- KvFormat.COMPACTED, baseRowFieldTypes))) {
+ KvFormat.COMPACTED, new TestingSchemaGetter(1, DATA1_SCHEMA)))) {
assertThat(keyToBytes(record)).isEqualTo(keys.get(i));
assertThat(record.getRow()).isEqualTo(rows.get(i));
i++;
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java
index ee3859be9..8e6774221 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultKvRecordTest.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.Test;
import java.util.Random;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link DefaultKvRecord}. */
@@ -39,7 +40,8 @@ class DefaultKvRecordTest extends KvTestBase {
@Test
void testBase() throws Exception {
KvRecordReadContext kvRecordReadContext =
- KvRecordReadContext.createReadContext(KvFormat.COMPACTED, baseRowFieldTypes);
+ KvRecordReadContext.createReadContext(
+ KvFormat.COMPACTED, new TestingSchemaGetter(1, DATA1_SCHEMA));
// create row.
CompactedRow row;
@@ -107,7 +109,8 @@ class DefaultKvRecordTest extends KvTestBase {
MemorySegment.wrap(outputView.getCopyOfBuffer()),
0,
schemaId,
- KvRecordReadContext.createReadContext(KvFormat.COMPACTED, colTypes));
+ KvRecordReadContext.createReadContext(
+ KvFormat.COMPACTED, new TestingSchemaGetter(1, DATA1_SCHEMA)));
// check key
assertThat(keyToBytes(kvRecord)).isEqualTo(key);
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java b/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
index 70f881e06..fb55be89a 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/DefaultLogRecordBatchTest.java
@@ -18,6 +18,9 @@
package org.apache.fluss.record;
import org.apache.fluss.memory.UnmanagedPagedOutputView;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.row.TestInternalRowGenerator;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.testutils.DataTestUtils;
@@ -87,10 +90,15 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.isValid()).isTrue();
assertThat(logRecordBatch.schemaId()).isEqualTo(schemaId);
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(
+ new SchemaInfo(
+ Schema.newBuilder().fromRowType(allRowType).build(), schemaId));
// verify record.
int i = 0;
try (LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(allRowType, schemaId);
+ LogRecordReadContext.createIndexedReadContext(
+ allRowType, schemaId, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
while (iter.hasNext()) {
LogRecord record = iter.next();
@@ -125,8 +133,13 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.lastLogOffset()).isEqualTo(0);
assertThat(logRecordBatch.nextLogOffset()).isEqualTo(1);
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(0);
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(
+ new SchemaInfo(
+ Schema.newBuilder().fromRowType(baseRowType).build(), schemaId));
try (LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(baseRowType, schemaId);
+ LogRecordReadContext.createIndexedReadContext(
+ baseRowType, schemaId, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
@@ -153,8 +166,10 @@ public class DefaultLogRecordBatchTest extends LogTestBase {
assertThat(logRecordBatch.lastLogOffset()).isEqualTo(100);
assertThat(logRecordBatch.nextLogOffset()).isEqualTo(101);
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(100);
+
try (LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(baseRowType, schemaId);
+ LogRecordReadContext.createIndexedReadContext(
+ baseRowType, schemaId, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java
index 7de046ec7..1e6e51d55 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogInputStreamTest.java
@@ -30,6 +30,7 @@ import java.util.Collections;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V0;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
+import static org.apache.fluss.record.TestData.DATA1_SCHEMA;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
import static org.apache.fluss.testutils.DataTestUtils.createRecordsWithoutBaseLogOffset;
import static org.assertj.core.api.Assertions.assertThat;
@@ -62,8 +63,10 @@ public class FileLogInputStreamTest extends LogTestBase {
LogRecordBatch recordBatch = batch.loadFullBatch();
+ TestingSchemaGetter schemaGetter = new TestingSchemaGetter(schemaId, DATA1_SCHEMA);
try (LogRecordReadContext readContext =
- LogRecordReadContext.createArrowReadContext(DATA1_ROW_TYPE, schemaId);
+ LogRecordReadContext.createArrowReadContext(
+ DATA1_ROW_TYPE, schemaId, schemaGetter);
CloseableIterator<LogRecord> iterator = recordBatch.records(readContext)) {
assertThat(iterator.hasNext()).isTrue();
LogRecord record = iterator.next();
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
index c19ecc882..2c3ff36ef 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
@@ -18,11 +18,14 @@
package org.apache.fluss.record;
import org.apache.fluss.exception.InvalidColumnProjectionException;
+import org.apache.fluss.exception.SchemaNotExistException;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
+import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
@@ -56,77 +59,130 @@ import static org.assertj.core.api.Assertions.fail;
class FileLogProjectionTest {
private @TempDir File tempDir;
+ private TestingSchemaGetter testingSchemaGetter;
+
+ @BeforeEach
+ void beforeEach() {
+ testingSchemaGetter = new TestingSchemaGetter(new SchemaInfo(TestData.DATA1_SCHEMA, 1));
+ testingSchemaGetter.updateLatestSchemaInfo(new SchemaInfo(TestData.DATA2_SCHEMA, 2));
+ }
- // TODO: add tests for nested types
@Test
- void testSetCurrentProjection() {
+ void testSetCurrentProjection() throws Exception {
+ short schemaId = (short) 2;
+ FileLogRecords recordsOfData2RowType =
+ createFileLogRecords(
+ schemaId,
+ LOG_MAGIC_VALUE_V1,
+ TestData.DATA2_ROW_TYPE,
+ TestData.DATA2,
+ TestData.DATA2);
+
FileLogProjection projection = new FileLogProjection();
- projection.setCurrentProjection(
- 1L, TestData.DATA2_ROW_TYPE, DEFAULT_COMPRESSION, new int[] {0, 2});
+ // get schema during running.
+ doProjection(
+ 1L,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {0, 2},
+ recordsOfData2RowType.sizeInBytes());
FileLogProjection.ProjectionInfo info1 = projection.currentProjection;
assertThat(info1).isNotNull();
assertThat(info1.nodesProjection.stream().toArray()).isEqualTo(new int[] {0, 2});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info1.buffersProjection.stream().toArray()).isEqualTo(new int[] {0, 1, 5, 6, 7});
assertThat(projection.projectionsCache).hasSize(1);
- assertThat(projection.projectionsCache.get(1L)).isSameAs(info1);
+ assertThat(
+ projection.projectionsCache.get(
+ new FileLogProjection.ProjectionKey(1L, schemaId)))
+ .isSameAs(info1);
- projection.setCurrentProjection(
- 2L, TestData.DATA2_ROW_TYPE, DEFAULT_COMPRESSION, new int[] {1});
+ doProjection(
+ 2L,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {1},
+ recordsOfData2RowType.sizeInBytes());
FileLogProjection.ProjectionInfo info2 = projection.currentProjection;
assertThat(info2).isNotNull();
assertThat(info2.nodesProjection.stream().toArray()).isEqualTo(new int[] {1});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info2.buffersProjection.stream().toArray()).isEqualTo(new int[] {2, 3, 4});
assertThat(projection.projectionsCache).hasSize(2);
- assertThat(projection.projectionsCache.get(2L)).isSameAs(info2);
+ assertThat(
+ projection.projectionsCache.get(
+ new FileLogProjection.ProjectionKey(2L, schemaId)))
+ .isSameAs(info2);
- projection.setCurrentProjection(
- 1L, TestData.DATA2_ROW_TYPE, DEFAULT_COMPRESSION, new int[] {0, 2});
+ doProjection(
+ 1L,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {0, 2},
+ recordsOfData2RowType.sizeInBytes());
assertThat(projection.currentProjection).isNotNull().isSameAs(info1);
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
+ doProjection(
1L,
- TestData.DATA1_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {1}))
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {1},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
.hasMessage("The schema and projection should be identical for the same table id.");
}
@Test
- void testIllegalSetCurrentProjection() {
+ void testIllegalSetCurrentProjection() throws Exception {
+ long tableId = 1L;
+ short schemaId = (short) 2;
+ FileLogRecords recordsOfData2RowType =
+ createFileLogRecords(
+ schemaId,
+ LOG_MAGIC_VALUE_V1,
+ TestData.DATA2_ROW_TYPE,
+ TestData.DATA2,
+ TestData.DATA2);
FileLogProjection projection = new FileLogProjection();
-
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
- 1L,
- TestData.DATA2_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {3}))
+ doProjection(
+ tableId,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {3},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
- .hasMessage("Projected fields [3] is out of bound for schema with 3 fields.");
+ .hasMessage("Projected field id 3 is not contains in [0, 1, 2]");
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
- 1L,
- TestData.DATA2_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {1, 0}))
+ doProjection(
+ tableId,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {1, 0},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
.hasMessage("The projection indexes should be in field order, but is [1, 0]");
assertThatThrownBy(
() ->
- projection.setCurrentProjection(
- 1L,
- TestData.DATA2_ROW_TYPE,
- DEFAULT_COMPRESSION,
- new int[] {0, 0, 0}))
+ doProjection(
+ tableId,
+ schemaId,
+ projection,
+ recordsOfData2RowType,
+ new int[] {0, 0, 0},
+ recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
.hasMessage(
"The projection indexes should not contain duplicated fields, but is [0, 0, 0]");
@@ -134,28 +190,38 @@ class FileLogProjectionTest {
static Stream<Arguments> projectedFieldsArgs() {
return Stream.of(
- Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V0),
- Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V0),
- Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V0),
- Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V1),
- Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V1),
- Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V1));
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V0, (short) 1),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V0, (short) 1),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V0, (short) 1),
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V1, (short) 1),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V1, (short) 1),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V1, (short) 1),
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V0, (short) 2),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V0, (short) 2),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V0, (short) 2),
+ Arguments.of((Object) new int[] {0}, LOG_MAGIC_VALUE_V1, (short) 2),
+ Arguments.arguments((Object) new int[] {1}, LOG_MAGIC_VALUE_V1, (short) 2),
+ Arguments.arguments((Object) new int[] {0, 1}, LOG_MAGIC_VALUE_V1, (short) 2));
}
@ParameterizedTest
@MethodSource("projectedFieldsArgs")
- void testProject(int[] projectedFields, byte recordBatchMagic) throws Exception {
+ void testProject(int[] projectedFields, byte recordBatchMagic, short schemaId)
+ throws Exception {
+ testingSchemaGetter.updateLatestSchemaInfo(new SchemaInfo(TestData.DATA1_SCHEMA, schemaId));
FileLogRecords fileLogRecords =
createFileLogRecords(
+ schemaId,
recordBatchMagic,
TestData.DATA1_ROW_TYPE,
TestData.DATA1,
TestData.ANOTHER_DATA1);
List<Object[]> results =
doProjection(
+ 1L,
+ schemaId,
new FileLogProjection(),
fileLogRecords,
- TestData.DATA1_ROW_TYPE,
projectedFields,
Integer.MAX_VALUE);
List<Object[]> allData = new ArrayList<>();
@@ -176,7 +242,7 @@ class FileLogProjectionTest {
@ParameterizedTest
@ValueSource(bytes = {LOG_MAGIC_VALUE_V0, LOG_MAGIC_VALUE_V1})
void testIllegalByteOrder(byte recordBatchMagic) throws Exception {
- FileLogRecords fileLogRecords =
+ final FileLogRecords fileLogRecords =
createFileLogRecords(
recordBatchMagic,
TestData.DATA1_ROW_TYPE,
@@ -185,17 +251,38 @@ class FileLogProjectionTest {
FileLogProjection projection = new FileLogProjection();
// overwrite the wrong decoding byte order endian
projection.getLogHeaderBuffer().order(ByteOrder.BIG_ENDIAN);
- // should throw exception.
assertThatThrownBy(
() ->
doProjection(
projection,
fileLogRecords,
- TestData.DATA1_ROW_TYPE,
+ new int[] {0, 1},
+ Integer.MAX_VALUE))
+ .isInstanceOf(SchemaNotExistException.class);
+
+ // The BIG_ENDIAN of schema will be read as 256 wronly. In this case, we use 256 to skip it.
+ short schemaId = 256;
+ testingSchemaGetter.updateLatestSchemaInfo(new SchemaInfo(TestData.DATA2_SCHEMA, schemaId));
+ final FileLogRecords fileLogRecords2 =
+ createFileLogRecords(
+ schemaId,
+ recordBatchMagic,
+ TestData.DATA1_ROW_TYPE,
+ TestData.DATA1,
+ TestData.ANOTHER_DATA1);
+
+ assertThatThrownBy(
+ () ->
+ doProjection(
+ 1L,
+ schemaId,
+ projection,
+ fileLogRecords2,
new int[] {0, 1},
Integer.MAX_VALUE))
.isInstanceOf(EOFException.class)
.hasMessageContaining("Failed to read `arrow header` from file channel");
+ fileLogRecords2.close();
}
@ParameterizedTest
@@ -218,11 +305,7 @@ class FileLogProjectionTest {
int maxBytes = totalSize / i;
List<Object[]> results =
doProjection(
- new FileLogProjection(),
- fileLogRecords,
- TestData.DATA1_ROW_TYPE,
- new int[] {0, 1},
- maxBytes);
+ new FileLogProjection(), fileLogRecords, new int[] {0, 1}, maxBytes);
if (results.isEmpty()) {
hasEmpty = true;
} else if (results.size() == TestData.DATA1.size()) {
@@ -323,13 +406,20 @@ class FileLogProjectionTest {
@SafeVarargs
final FileLogRecords createFileLogRecords(
byte recordBatchMagic, RowType rowType, List<Object[]>... inputs) throws Exception {
+ return createFileLogRecords(DEFAULT_SCHEMA_ID, recordBatchMagic, rowType, inputs);
+ }
+
+ @SafeVarargs
+ final FileLogRecords createFileLogRecords(
+ int schemaId, byte recordBatchMagic, RowType rowType, List<Object[]>... inputs)
+ throws Exception {
FileLogRecords fileLogRecords = FileLogRecords.open(new File(tempDir, "test.tmp"));
long offsetBase = 0L;
for (List<Object[]> input : inputs) {
fileLogRecords.append(
createRecordsWithoutBaseLogOffset(
rowType,
- DEFAULT_SCHEMA_ID,
+ schemaId,
offsetBase,
System.currentTimeMillis(),
recordBatchMagic,
@@ -344,20 +434,33 @@ class FileLogProjectionTest {
private List<Object[]> doProjection(
FileLogProjection projection,
FileLogRecords fileLogRecords,
- RowType rowType,
int[] projectedFields,
int fetchMaxBytes)
throws Exception {
- projection.setCurrentProjection(1L, rowType, DEFAULT_COMPRESSION, projectedFields);
- RowType projectedType = rowType.project(projectedFields);
+ return doProjection(
+ 1L, DEFAULT_SCHEMA_ID, projection, fileLogRecords, projectedFields, fetchMaxBytes);
+ }
+
+ private List<Object[]> doProjection(
+ long tableId,
+ short schemaId,
+ FileLogProjection projection,
+ FileLogRecords fileLogRecords,
+ int[] projectedFields,
+ int fetchMaxBytes)
+ throws Exception {
+ projection.setCurrentProjection(
+ tableId, testingSchemaGetter, DEFAULT_COMPRESSION, projectedFields);
LogRecords project =
projection.project(
fileLogRecords.channel(), 0, fileLogRecords.sizeInBytes(), fetchMaxBytes);
assertThat(project.sizeInBytes()).isLessThanOrEqualTo(fetchMaxBytes);
+ RowType rowType = testingSchemaGetter.getSchema(schemaId).getRowType();
+ RowType projectedType = rowType.project(projectedFields);
List<Object[]> results = new ArrayList<>();
long expectedOffset = 0L;
try (LogRecordReadContext context =
- createArrowReadContext(projectedType, DEFAULT_SCHEMA_ID)) {
+ createArrowReadContext(projectedType, schemaId, testingSchemaGetter)) {
for (LogRecordBatch batch : project.batches()) {
try (CloseableIterator<LogRecord> records = batch.records(context)) {
while (records.hasNext()) {
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java b/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java
index 8e7395ca1..a8f851cee 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogRecordsTest.java
@@ -17,6 +17,8 @@
package org.apache.fluss.record;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.TestInternalRowGenerator;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.testutils.DataTestUtils;
@@ -75,8 +77,10 @@ final class LogRecordsTest extends LogTestBase {
assertThat(iterator.hasNext()).isTrue();
assertThat(memoryIterator.hasNext()).isTrue();
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(1, Schema.newBuilder().fromRowType(baseRowType).build());
assertIndexedLogRecordBatchAndRowEquals(
- iterator.next(), memoryIterator.next(), baseRowType, rows1);
+ iterator.next(), memoryIterator.next(), baseRowType, rows1, schemaGetter);
assertThat(iterator.hasNext()).isFalse();
assertThat(memoryIterator.hasNext()).isFalse();
@@ -98,8 +102,10 @@ final class LogRecordsTest extends LogTestBase {
assertThat(iterator.hasNext()).isTrue();
assertThat(memoryIterator.hasNext()).isTrue();
+ SchemaGetter schemaGetter =
+ new TestingSchemaGetter(1, Schema.newBuilder().fromRowType(allRowType).build());
assertIndexedLogRecordBatchAndRowEquals(
- iterator.next(), memoryIterator.next(), allRowType, rows1);
+ iterator.next(), memoryIterator.next(), allRowType, rows1, schemaGetter);
assertThat(iterator.hasNext()).isFalse();
assertThat(memoryIterator.hasNext()).isFalse();
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
index 3183d1acf..108ab2acf 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
@@ -19,6 +19,7 @@ package org.apache.fluss.record;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.memory.MemorySegmentOutputView;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.row.TestInternalRowGenerator;
import org.apache.fluss.row.indexed.IndexedRow;
import org.apache.fluss.testutils.ListLogRecords;
@@ -71,15 +72,18 @@ public abstract class LogTestBase {
assertLogRecordsListEquals(expected, actual, baseRowType);
}
+ // todo: 在这个基础上可以加更多最简单的schema evolution测试,非常好
protected void assertIndexedLogRecordBatchAndRowEquals(
LogRecordBatch actual,
LogRecordBatch expected,
RowType rowType,
- List<IndexedRow> rows) {
+ List<IndexedRow> rows,
+ SchemaGetter schemaGetter) {
assertRecordBatchHeaderEquals(actual);
LogRecordReadContext readContext =
- LogRecordReadContext.createIndexedReadContext(rowType, TestData.DEFAULT_SCHEMA_ID);
+ LogRecordReadContext.createIndexedReadContext(
+ rowType, TestData.DEFAULT_SCHEMA_ID, schemaGetter);
try (CloseableIterator<LogRecord> actualIter = actual.records(readContext);
CloseableIterator<LogRecord> expectIter = expected.records(readContext)) {
int i = 0;
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
index 5c0df6e36..e72b44f14 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/MemoryLogRecordsArrowBuilderTest.java
@@ -58,6 +58,7 @@ import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.TestData.DATA1;
import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.row.arrow.ArrowWriter.BUFFER_USAGE_RATIO;
import static org.apache.fluss.testutils.DataTestUtils.assertLogRecordsEquals;
import static org.apache.fluss.testutils.DataTestUtils.row;
@@ -278,9 +279,10 @@ public class MemoryLogRecordsArrowBuilderTest {
assertThat(logRecordBatch.lastLogOffset()).isEqualTo(0);
assertThat(logRecordBatch.nextLogOffset()).isEqualTo(1);
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(0);
+ TestingSchemaGetter schemaGetter = TEST_SCHEMA_GETTER;
try (LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
@@ -305,7 +307,7 @@ public class MemoryLogRecordsArrowBuilderTest {
assertThat(logRecordBatch.baseLogOffset()).isEqualTo(100);
try (LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID);
+ DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> iter = logRecordBatch.records(readContext)) {
assertThat(iter.hasNext()).isFalse();
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestData.java b/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
index 7752b15b0..874377125 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestData.java
@@ -217,4 +217,7 @@ public final class TestData {
.primaryKey("a")
.build();
// ---------------------------- data3 table info end ------------------------------
+
+ public static final TestingSchemaGetter TEST_SCHEMA_GETTER =
+ new TestingSchemaGetter(DEFAULT_SCHEMA_ID, DATA2_SCHEMA);
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
new file mode 100644
index 000000000..f74f379eb
--- /dev/null
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.record;
+
+import org.apache.fluss.exception.SchemaNotExistException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+
+import java.util.HashMap;
+import java.util.Map;
+
+/** A testing schema metadata subscriber. */
+public class TestingSchemaGetter implements SchemaGetter {
+ private SchemaInfo schemaInfo;
+ private Map<Integer, Schema> schemaCaches;
+
+ public TestingSchemaGetter() {
+ this.schemaInfo = null;
+ this.schemaCaches = new HashMap<>();
+ }
+
+ // todo: remove it(简化测试代码)
+ public TestingSchemaGetter(SchemaInfo schemaInfo) {
+ this.schemaInfo = schemaInfo;
+ this.schemaCaches = new HashMap<>();
+ this.schemaCaches.put(schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+
+ public TestingSchemaGetter(int schemaId, Schema schema) {
+ this.schemaInfo = new SchemaInfo(schema, schemaId);
+ this.schemaCaches = new HashMap<>();
+ this.schemaCaches.put(schemaId, schema);
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ if (!schemaCaches.containsKey(schemaId)) {
+ throw new SchemaNotExistException("Schema not exist");
+ }
+ return schemaCaches.get(schemaId);
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ return schemaInfo;
+ }
+
+ @Override
+ public void release() {}
+
+ public void updateLatestSchemaInfo(SchemaInfo schemaInfo) {
+ if (schemaInfo.getSchemaId() >= this.schemaInfo.getSchemaId()) {
+ this.schemaInfo = schemaInfo;
+ }
+ this.schemaCaches.put(schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+}
diff --git a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
index 5e04e57a9..09467d31a 100644
--- a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
@@ -17,11 +17,18 @@
package org.apache.fluss.row;
+import org.apache.fluss.exception.SchemaChangeException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.types.DataTypes;
+
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
+import java.util.Arrays;
+import java.util.Collections;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link ProjectedRow}. */
class ProjectedRowTest {
@@ -159,4 +166,65 @@ class ProjectedRowTest {
assertThat(projectedRow.isNullAt(2)).isFalse();
assertThat(projectedRow.isNullAt(3)).isTrue();
}
+
+ @Test
+ void testProjectedRowsWithDifferentSchema() {
+ Schema schema =
+ Schema.newBuilder()
+ .column("a", DataTypes.BIGINT())
+ .column("b", DataTypes.STRING())
+ .column("c", DataTypes.DECIMAL(10, 1))
+ .build();
+ assertThatThrownBy(
+ () ->
+ ProjectedRow.from(
+ schema,
+ Schema.newBuilder().column("a", DataTypes.INT()).build()))
+ .isExactlyInstanceOf(SchemaChangeException.class)
+ .hasMessage(
+ "Expected datatype of column(id=0,name=a) is [INT], while the actual datatype is [BIGINT]");
+ assertThatThrownBy(
+ () ->
+ ProjectedRow.from(
+ schema,
+ Schema.newBuilder()
+ .fromColumns(
+ Collections.singletonList(
+ new Schema.Column(
+ "f",
+ DataTypes.DECIMAL(20, 1),
+ null,
+ 2)))
+ .build()))
+ .isExactlyInstanceOf(SchemaChangeException.class)
+ .hasMessage(
+ "Expected datatype of column(id=2,name=f) is [DECIMAL(20, 1)], while the actual datatype is [DECIMAL(10, 1)]");
+
+ Schema newSchema =
+ Schema.newBuilder()
+ .fromColumns(
+ Arrays.asList(
+ new Schema.Column("a", DataTypes.BIGINT(), null, 0),
+ // colum e share same id as the column c in the original
+ // schema.
+ new Schema.Column("e", DataTypes.DECIMAL(10, 1), null, 2),
+ // column b has different id the column b the original
+ // schema.
+ new Schema.Column("b", DataTypes.DECIMAL(10, 1), null, 3),
+ new Schema.Column("f", DataTypes.STRING(), null, 4)))
+ .build();
+
+ ProjectedRow projectedRow = ProjectedRow.from(schema, newSchema);
+ projectedRow.replaceRow(
+ GenericRow.of(
+ 1L,
+ BinaryString.fromString("value"),
+ Decimal.fromBigDecimal(new BigDecimal("13145678.1"), 10, 1)));
+ assertThat(projectedRow.getFieldCount()).isEqualTo(4);
+ assertThat(projectedRow.getLong(0)).isEqualTo(1L);
+ assertThat(projectedRow.getDecimal(1, 10, 1).decimalVal)
+ .isEqualTo(new BigDecimal("13145678.1"));
+ assertThat(projectedRow.isNullAt(2)).isTrue();
+ assertThat(projectedRow.isNullAt(3)).isTrue();
+ }
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java b/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
index 2236ebe21..b9d5c4a27 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/DataTestUtils.java
@@ -27,6 +27,7 @@ import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.DefaultLogRecordBatch;
@@ -81,6 +82,7 @@ import static org.apache.fluss.record.TestData.DATA1_ROW_TYPE;
import static org.apache.fluss.record.TestData.DATA1_SCHEMA_PK;
import static org.apache.fluss.record.TestData.DEFAULT_MAGIC;
import static org.apache.fluss.record.TestData.DEFAULT_SCHEMA_ID;
+import static org.apache.fluss.record.TestData.TEST_SCHEMA_GETTER;
import static org.apache.fluss.testutils.LogRecordBatchAssert.assertThatLogRecordBatch;
import static org.apache.fluss.utils.FlussPaths.remoteLogDir;
import static org.apache.fluss.utils.FlussPaths.remoteLogSegmentDir;
@@ -153,14 +155,13 @@ public class DataTestUtils {
public static MemoryLogRecords genMemoryLogRecordsByObject(byte magic, List<Object[]> objects)
throws Exception {
+ return genMemoryLogRecordsByObject(DATA1_ROW_TYPE, DEFAULT_SCHEMA_ID, magic, objects);
+ }
+
+ public static MemoryLogRecords genMemoryLogRecordsByObject(
+ RowType rowType, int schemaId, byte magic, List<Object[]> objects) throws Exception {
return createRecordsWithoutBaseLogOffset(
- DATA1_ROW_TYPE,
- DEFAULT_SCHEMA_ID,
- 0,
- System.currentTimeMillis(),
- magic,
- objects,
- LogFormat.ARROW);
+ rowType, schemaId, 0, System.currentTimeMillis(), magic, objects, LogFormat.ARROW);
}
public static MemoryLogRecords genMemoryLogRecordsByObject(List<Object[]> objects)
@@ -271,9 +272,14 @@ public class DataTestUtils {
* Generate a KvRecord batch from the values only, whose key will be extracted from the value.
*/
public static KvRecordBatch genKvRecordBatch(Object[]... values) throws Exception {
+ return genKvRecordBatch(DEFAULT_SCHEMA_ID, DATA1_ROW_TYPE, values);
+ }
+
+ public static KvRecordBatch genKvRecordBatch(
+ short schemaId, RowType rowType, Object[]... values) throws Exception {
KvRecordTestUtils.KvRecordBatchFactory kvRecordBatchFactory =
- KvRecordTestUtils.KvRecordBatchFactory.of(DEFAULT_SCHEMA_ID);
- return kvRecordBatchFactory.ofRecords(genKvRecords(values));
+ KvRecordTestUtils.KvRecordBatchFactory.of(schemaId);
+ return kvRecordBatchFactory.ofRecords(genKvRecords(rowType, values));
}
public static KvRecordBatch toKvRecordBatch(List<KvRecord> records) throws Exception {
@@ -344,9 +350,13 @@ public class DataTestUtils {
* the values.
*/
public static List<KvRecord> genKvRecords(Object[]... values) {
+ return genKvRecords(DATA1_ROW_TYPE, values);
+ }
+
+ public static List<KvRecord> genKvRecords(RowType rowType, Object[]... values) {
KvRecordTestUtils.PKBasedKvRecordFactory kvRecordFactory =
KvRecordTestUtils.PKBasedKvRecordFactory.of(
- DATA1_SCHEMA_PK.getRowType(), DATA1_SCHEMA_PK.getPrimaryKeyIndexes());
+ rowType, DATA1_SCHEMA_PK.getPrimaryKeyIndexes());
List<KvRecord> records = new ArrayList<>();
for (Object[] value : values) {
records.add(kvRecordFactory.ofRecord(value));
@@ -358,13 +368,23 @@ public class DataTestUtils {
return getKeyValuePairs(kvRecords.toArray(new KvRecord[0]));
}
+ public static List<Tuple2<byte[], byte[]>> getKeyValuePairs(
+ short schemaId, List<KvRecord> kvRecords) {
+ return getKeyValuePairs(schemaId, kvRecords.toArray(new KvRecord[0]));
+ }
+
public static List<Tuple2<byte[], byte[]>> getKeyValuePairs(KvRecord... kvRecords) {
+ return getKeyValuePairs(DEFAULT_SCHEMA_ID, kvRecords);
+ }
+
+ public static List<Tuple2<byte[], byte[]>> getKeyValuePairs(
+ short schemaId, KvRecord... kvRecords) {
List<Tuple2<byte[], byte[]>> keyValuePairs = new ArrayList<>();
for (KvRecord kvRecord : kvRecords) {
keyValuePairs.add(
Tuple2.of(
BytesUtils.toArray(kvRecord.getKey()),
- ValueEncoder.encodeValue(DEFAULT_SCHEMA_ID, kvRecord.getRow())));
+ ValueEncoder.encodeValue(schemaId, kvRecord.getRow())));
}
return keyValuePairs;
}
@@ -527,7 +547,10 @@ public class DataTestUtils {
}
public static void assertMemoryRecordsEquals(
- RowType rowType, LogRecords records, List<List<Object[]>> expected) {
+ RowType rowType,
+ SchemaGetter schemaGetter,
+ LogRecords records,
+ List<List<Object[]>> expected) {
List<List<Tuple2<ChangeType, Object[]>>> appendOnlyExpectedValue = new ArrayList<>();
for (List<Object[]> expectedRecord : expected) {
List<Tuple2<ChangeType, Object[]>> expectedFieldAndRowKind =
@@ -536,11 +559,13 @@ public class DataTestUtils {
.collect(Collectors.toList());
appendOnlyExpectedValue.add(expectedFieldAndRowKind);
}
- assertMemoryRecordsEqualsWithRowKind(rowType, records, appendOnlyExpectedValue);
+ assertMemoryRecordsEqualsWithRowKind(
+ rowType, schemaGetter, records, appendOnlyExpectedValue);
}
public static void assertMemoryRecordsEqualsWithRowKind(
- RowType rowType,
+ RowType expectRowType,
+ SchemaGetter schemaGetter,
LogRecords records,
List<List<Tuple2<ChangeType, Object[]>>> expected) {
Iterator<LogRecordBatch> iterator = records.batches().iterator();
@@ -548,12 +573,12 @@ public class DataTestUtils {
assertThat(iterator.hasNext()).isTrue();
LogRecordBatch batch = iterator.next();
try (LogRecordReadContext readContext =
- createArrowReadContext(rowType, DEFAULT_SCHEMA_ID);
+ createArrowReadContext(expectRowType, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> logIterator = batch.records(readContext)) {
for (Tuple2<ChangeType, Object[]> expectedFieldAndRowKind : expectedRecord) {
assertThat(logIterator.hasNext()).isTrue();
assertLogRecordsEqualsWithRowKind(
- rowType, logIterator.next(), expectedFieldAndRowKind);
+ expectRowType, logIterator.next(), expectedFieldAndRowKind);
}
assertThat(logIterator.hasNext()).isFalse();
}
@@ -563,9 +588,11 @@ public class DataTestUtils {
public static void assertLogRecordBatchEqualsWithRowKind(
RowType rowType,
+ SchemaGetter schemaGetter,
LogRecordBatch logRecordBatch,
List<Tuple2<ChangeType, Object[]>> expected) {
- try (LogRecordReadContext readContext = createArrowReadContext(rowType, DEFAULT_SCHEMA_ID);
+ try (LogRecordReadContext readContext =
+ createArrowReadContext(rowType, DEFAULT_SCHEMA_ID, schemaGetter);
CloseableIterator<LogRecord> logIterator = logRecordBatch.records(readContext)) {
for (Tuple2<ChangeType, Object[]> expectedFieldAndRowKind : expected) {
assertThat(logIterator.hasNext()).isTrue();
@@ -615,17 +642,46 @@ public class DataTestUtils {
public static void assertLogRecordsEquals(
RowType rowType, LogRecords logRecords, List<Object[]> expectedValue) {
+ assertLogRecordsEquals(
+ DEFAULT_SCHEMA_ID, rowType, logRecords, expectedValue, TEST_SCHEMA_GETTER);
+ }
+
+ public static void assertLogRecordsEquals(
+ RowType rowType,
+ LogRecords logRecords,
+ List<Object[]> expectedValue,
+ SchemaGetter schemaGetter) {
+ assertLogRecordsEquals(DEFAULT_SCHEMA_ID, rowType, logRecords, expectedValue, schemaGetter);
+ }
+
+ public static void assertLogRecordsEquals(
+ int schemaId,
+ RowType rowType,
+ LogRecords logRecords,
+ List<Object[]> expectedValue,
+ SchemaGetter schemaGetter) {
List<Tuple2<ChangeType, Object[]>> expectedValueWithRowKind =
expectedValue.stream()
.map(val -> Tuple2.of(ChangeType.APPEND_ONLY, val))
.collect(Collectors.toList());
- assertLogRecordsEqualsWithRowKind(rowType, logRecords, expectedValueWithRowKind);
+ assertLogRecordsEqualsWithRowKind(
+ schemaId, rowType, logRecords, expectedValueWithRowKind, schemaGetter);
}
public static void assertLogRecordsEqualsWithRowKind(
RowType rowType,
LogRecords logRecords,
List<Tuple2<ChangeType, Object[]>> expectedValue) {
+ assertLogRecordsEqualsWithRowKind(
+ DEFAULT_SCHEMA_ID, rowType, logRecords, expectedValue, TEST_SCHEMA_GETTER);
+ }
+
+ public static void assertLogRecordsEqualsWithRowKind(
+ int schemaId,
+ RowType rowType,
+ LogRecords logRecords,
+ List<Tuple2<ChangeType, Object[]>> expectedValue,
+ SchemaGetter schemaGetter) {
DataType[] dataTypes = rowType.getChildren().toArray(new DataType[0]);
InternalRow.FieldGetter[] fieldGetter = new InternalRow.FieldGetter[dataTypes.length];
for (int i = 0; i < dataTypes.length; i++) {
@@ -634,7 +690,7 @@ public class DataTestUtils {
int i = 0;
try (LogRecordReadContext readContext =
- LogRecordReadContext.createArrowReadContext(rowType, DEFAULT_SCHEMA_ID)) {
+ LogRecordReadContext.createArrowReadContext(rowType, schemaId, schemaGetter)) {
for (LogRecordBatch batch : logRecords.batches()) {
try (CloseableIterator<LogRecord> iterator = batch.records(readContext)) {
while (iterator.hasNext()) {
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java
index ca6ee5d4f..320533ffb 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordBatchAssert.java
@@ -18,6 +18,7 @@
package org.apache.fluss.testutils;
import org.apache.fluss.metadata.LogFormat;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.LogRecord;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecordReadContext;
@@ -35,6 +36,7 @@ public class LogRecordBatchAssert extends AbstractAssert<LogRecordBatchAssert, L
private RowType rowType;
private LogFormat logFormat;
private boolean assertCheckSum = true;
+ private SchemaGetter schemaGetter;
/** Creates assertions for {@link LogRecordBatch}. */
public static LogRecordBatchAssert assertThatLogRecordBatch(LogRecordBatch actual) {
@@ -60,6 +62,11 @@ public class LogRecordBatchAssert extends AbstractAssert<LogRecordBatchAssert, L
return this;
}
+ public LogRecordBatchAssert withSchemaGetter(SchemaGetter schemaGetter) {
+ this.schemaGetter = schemaGetter;
+ return this;
+ }
+
public LogRecordBatchAssert isEqualTo(LogRecordBatch expected) {
if (rowType == null) {
throw new IllegalStateException(
@@ -113,9 +120,9 @@ public class LogRecordBatchAssert extends AbstractAssert<LogRecordBatchAssert, L
private LogRecordReadContext createReadContext(int schemaId) {
if (logFormat != null && logFormat == LogFormat.INDEXED) {
- return LogRecordReadContext.createIndexedReadContext(rowType, schemaId);
+ return LogRecordReadContext.createIndexedReadContext(rowType, schemaId, schemaGetter);
} else {
- return LogRecordReadContext.createArrowReadContext(rowType, schemaId);
+ return LogRecordReadContext.createArrowReadContext(rowType, schemaId, schemaGetter);
}
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
index b487b2138..28df40e34 100644
--- a/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
+++ b/fluss-common/src/test/java/org/apache/fluss/testutils/LogRecordsAssert.java
@@ -17,8 +17,10 @@
package org.apache.fluss.testutils;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecords;
+import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.types.RowType;
import org.assertj.core.api.AbstractAssert;
@@ -38,6 +40,7 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
private RowType rowType;
private boolean assertCheckSum = true;
+ private SchemaGetter schemaGetter = new TestingSchemaGetter();
private LogRecordsAssert(LogRecords actual) {
super(actual, LogRecordsAssert.class);
@@ -48,6 +51,11 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
return this;
}
+ public LogRecordsAssert withSchemaGetter(SchemaGetter schemaGetter) {
+ this.schemaGetter = schemaGetter;
+ return this;
+ }
+
public LogRecordsAssert assertCheckSum(boolean assertCheckSum) {
this.assertCheckSum = assertCheckSum;
return this;
@@ -69,6 +77,7 @@ public class LogRecordsAssert extends AbstractAssert<LogRecordsAssert, LogRecord
assertThatLogRecordBatch(actualIter.next())
.withSchema(rowType)
.assertCheckSum(assertCheckSum)
+ .withSchemaGetter(schemaGetter)
.isEqualTo(expectedNext);
}
assertThat(actualIter.hasNext()).isFalse();
diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java
index e34ff49b1..48fc2ac67 100644
--- a/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/utils/ProjectionTest.java
@@ -17,6 +17,7 @@
package org.apache.fluss.utils;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.row.BinaryString;
import org.apache.fluss.row.GenericRow;
import org.apache.fluss.row.ProjectedRow;
@@ -32,8 +33,15 @@ class ProjectionTest {
@Test
void testProjection() {
- Projection projection = Projection.of(new int[] {2, 0, 3});
- assertThat(projection.getProjectionInOrder()).isEqualTo(new int[] {0, 2, 3});
+ Schema schema =
+ Schema.newBuilder()
+ .column("f0", DataTypes.INT())
+ .column("f1", DataTypes.BIGINT())
+ .column("f2", DataTypes.STRING())
+ .column("f3", DataTypes.DOUBLE())
+ .build();
+ Projection projection = Projection.of(new int[] {2, 0, 3}, schema);
+ assertThat(projection.getProjectionIdInOrder()).isEqualTo(new int[] {0, 2, 3});
RowType rowType =
projection.projectInOrder(
@@ -59,7 +67,7 @@ class ProjectionTest {
DataTypes.FIELD("f3", DataTypes.DOUBLE())));
GenericRow row = GenericRow.of(0, 1L, BinaryString.fromString("2"), 3.0d);
- ProjectedRow p1 = ProjectedRow.from(projection.getProjectionInOrder());
+ ProjectedRow p1 = ProjectedRow.from(projection.getProjectionIdInOrder());
p1.replaceRow(row);
ProjectedRow p2 = ProjectedRow.from(projection.getReorderingIndexes());
p2.replaceRow(p1);
diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java
index b9924fac5..716e04347 100644
--- a/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/utils/json/ColumnJsonSerdeTest.java
@@ -21,6 +21,13 @@ import org.apache.fluss.metadata.Schema;
import org.apache.fluss.types.DataTypes;
import org.apache.fluss.types.IntType;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
/** Test for {@link ColumnJsonSerde}. */
public class ColumnJsonSerdeTest extends JsonSerdeTestBase<Schema.Column> {
protected ColumnJsonSerdeTest() {
@@ -29,19 +36,41 @@ public class ColumnJsonSerdeTest extends JsonSerdeTestBase<Schema.Column> {
@Override
protected Schema.Column[] createObjects() {
- Schema.Column[] columns = new Schema.Column[3];
+ Schema.Column[] columns = new Schema.Column[4];
columns[0] = new Schema.Column("a", DataTypes.STRING());
columns[1] = new Schema.Column("b", DataTypes.INT(), "hello b");
columns[2] = new Schema.Column("c", new IntType(false), "hello c");
+ columns[3] = new Schema.Column("d", new IntType(false), "hello c", (short) 2);
return columns;
}
@Override
protected String[] expectedJsons() {
return new String[] {
+ "{\"name\":\"a\",\"data_type\":{\"type\":\"STRING\"},\"id\":-1}",
+ "{\"name\":\"b\",\"data_type\":{\"type\":\"INTEGER\"},\"comment\":\"hello b\",\"id\":-1}",
+ "{\"name\":\"c\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"hello c\",\"id\":-1}",
+ "{\"name\":\"d\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"hello c\",\"id\":2}"
+ };
+ }
+
+ @Test
+ void testDeserializeCompatibility() throws IOException {
+ String[] jsonWithoutColumnId = {
"{\"name\":\"a\",\"data_type\":{\"type\":\"STRING\"}}",
"{\"name\":\"b\",\"data_type\":{\"type\":\"INTEGER\"},\"comment\":\"hello b\"}",
"{\"name\":\"c\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"hello c\"}"
};
+ Schema.Column[] columns = new Schema.Column[3];
+ columns[0] = new Schema.Column("a", DataTypes.STRING());
+ columns[1] = new Schema.Column("b", DataTypes.INT(), "hello b");
+ columns[2] = new Schema.Column("c", new IntType(false), "hello c");
+ for (int i = 0; i < jsonWithoutColumnId.length; i++) {
+ Schema.Column column =
+ JsonSerdeUtils.readValue(
+ jsonWithoutColumnId[i].getBytes(StandardCharsets.UTF_8),
+ ColumnJsonSerde.INSTANCE);
+ assertThat(column).isEqualTo(columns[i]);
+ }
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java b/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java
index 4b3ddc4c9..a7b6dca62 100644
--- a/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/utils/json/SchemaJsonSerdeTest.java
@@ -20,9 +20,14 @@ package org.apache.fluss.utils.json;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.types.DataTypes;
+import org.junit.jupiter.api.Test;
+
+import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Collections;
+import static org.assertj.core.api.Assertions.assertThat;
+
/** Test for {@link SchemaJsonSerde}. */
public class SchemaJsonSerdeTest extends JsonSerdeTestBase<Schema> {
@@ -62,14 +67,12 @@ public class SchemaJsonSerdeTest extends JsonSerdeTestBase<Schema> {
.build();
static final String SCHEMA_JSON_0 =
- "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"CHAR\",\"nullable\":false,\"length\":10},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\",\"c\"]}";
-
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"a is first column\",\"id\":0},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\",\"id\":1},{\"name\":\"c\",\"data_type\":{\"type\":\"CHAR\",\"nullable\":false,\"length\":10},\"comment\":\"c is third column\",\"id\":2}],\"primary_key\":[\"a\",\"c\"],\"highest_field_id\":2}";
static final String SCHEMA_JSON_1 =
- "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false}},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\"]}";
-
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"id\":0},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\",\"id\":1},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\",\"id\":2}],\"primary_key\":[\"a\"],\"highest_field_id\":2}";
// shouldn't contain primary_key fields
static final String SCHEMA_JSON_3 =
- "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"BIGINT\"},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}]}";
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"BIGINT\"},\"comment\":\"a is first column\",\"id\":0},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\",\"id\":1},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\",\"id\":2}],\"highest_field_id\":2}";
SchemaJsonSerdeTest() {
super(SchemaJsonSerde.INSTANCE);
@@ -84,4 +87,31 @@ public class SchemaJsonSerdeTest extends JsonSerdeTestBase<Schema> {
protected String[] expectedJsons() {
return new String[] {SCHEMA_JSON_0, SCHEMA_JSON_1, SCHEMA_JSON_1, SCHEMA_JSON_3};
}
+
+ @Test
+ void testCompatibilityFromJsonLackOfColumnId() {
+ String[] jsons = jsonLackOfColumnId();
+ Schema[] expectedSchema = new Schema[] {SCHEMA_0, SCHEMA_1, SCHEMA_2, SCHEMA_3};
+ for (int i = 0; i < jsons.length; i++) {
+ assertThat(
+ JsonSerdeUtils.readValue(
+ jsons[i].getBytes(StandardCharsets.UTF_8),
+ SchemaJsonSerde.INSTANCE))
+ .isEqualTo(expectedSchema[i]);
+ }
+ }
+
+ protected String[] jsonLackOfColumnId() {
+ String oldSchemaJson1 =
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"CHAR\",\"nullable\":false,\"length\":10},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\",\"c\"]}";
+
+ String oldSchemaJson2 =
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"INTEGER\",\"nullable\":false}},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}],\"primary_key\":[\"a\"]}";
+
+ // shouldn't contain primary_key fields
+ String oldSchemaJson3 =
+ "{\"version\":1,\"columns\":[{\"name\":\"a\",\"data_type\":{\"type\":\"BIGINT\"},\"comment\":\"a is first column\"},{\"name\":\"b\",\"data_type\":{\"type\":\"STRING\"},\"comment\":\"b is second column\"},{\"name\":\"c\",\"data_type\":{\"type\":\"TIMESTAMP_WITHOUT_TIME_ZONE\",\"precision\":6},\"comment\":\"c is third column\"}]}";
+
+ return new String[] {oldSchemaJson1, oldSchemaJson2, oldSchemaJson2, oldSchemaJson3};
+ }
}
diff --git a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
index 12b05f186..bd1b6af81 100644
--- a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
+++ b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level = OFF
+rootLogger.level = INFO
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
index 003e910ea..314e4b40f 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
@@ -23,17 +23,34 @@ import org.apache.fluss.row.InternalArray;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
+import org.apache.fluss.types.RowType;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.TimestampData;
+import javax.annotation.Nullable;
+
+import java.util.List;
+
/** Wraps a Flink {@link RowData} as a Fluss {@link InternalRow}. */
public class FlinkAsFlussRow implements InternalRow {
private RowData flinkRow;
- public FlinkAsFlussRow() {}
+ /**
+ * Creates a FlinkAsFlussRow with a mapping from Fluss field index to Flink field index. If the
+ * indexMapping is null, flink and flink share same field name.
+ */
+ @Nullable private final int[] indexMapping;
+
+ public FlinkAsFlussRow() {
+ this(null);
+ }
+
+ public FlinkAsFlussRow(int[] indexMapping) {
+ this.indexMapping = indexMapping;
+ }
public FlinkAsFlussRow replace(RowData flinkRow) {
this.flinkRow = flinkRow;
@@ -42,62 +59,94 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public int getFieldCount() {
- return flinkRow.getArity();
+ if (indexMapping == null) {
+ return flinkRow.getArity();
+ } else {
+ return indexMapping.length;
+ }
}
@Override
public boolean isNullAt(int pos) {
- return flinkRow.isNullAt(pos);
+ // if no mapping, means that the field is not found in Flink, just ignore it.
+ if ((indexMapping != null && indexMapping[pos] == -1)) {
+ return true;
+ }
+
+ // If pos is larger than flinkRow.getArity(), it indicates that the Fluss table's data type
+ // is wider than the data provided by Flink.
+ // This often occurs when a schema change happens to the catalog table before job restart.
+ // Only appending columns at the end is compatible, just need to ignore the trailing
+ // columns.
+ // Other types of schema changes (e.g., dropping columns) would lead to incompatibility.
+ // Therefore, Fluss currently only supports appending columns, and here we simply ignore any
+ // extra columns.
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ if (flussPos >= flinkRow.getArity()) {
+ return true;
+ }
+
+ return flinkRow.isNullAt(flussPos);
}
@Override
public boolean getBoolean(int pos) {
- return flinkRow.getBoolean(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getBoolean(flussPos);
}
@Override
public byte getByte(int pos) {
- return flinkRow.getByte(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getByte(flussPos);
}
@Override
public short getShort(int pos) {
- return flinkRow.getShort(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getShort(flussPos);
}
@Override
public int getInt(int pos) {
- return flinkRow.getInt(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getInt(flussPos);
}
@Override
public long getLong(int pos) {
- return flinkRow.getLong(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getLong(flussPos);
}
@Override
public float getFloat(int pos) {
- return flinkRow.getFloat(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getFloat(flussPos);
}
@Override
public double getDouble(int pos) {
- return flinkRow.getDouble(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getDouble(flussPos);
}
@Override
public BinaryString getChar(int pos, int length) {
- return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
}
@Override
public BinaryString getString(int pos) {
- return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
}
@Override
public Decimal getDecimal(int pos, int precision, int scale) {
- return fromFlinkDecimal(flinkRow.getDecimal(pos, precision, scale));
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return fromFlinkDecimal(flinkRow.getDecimal(flussPos, precision, scale));
}
public static Decimal fromFlinkDecimal(DecimalData decimal) {
@@ -112,26 +161,43 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public TimestampNtz getTimestampNtz(int pos, int precision) {
- TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
return TimestampNtz.fromMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public TimestampLtz getTimestampLtz(int pos, int precision) {
- TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
return TimestampLtz.fromEpochMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public byte[] getBinary(int pos, int length) {
- return flinkRow.getBinary(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getBinary(flussPos);
}
@Override
public byte[] getBytes(int pos) {
- return flinkRow.getBinary(pos);
+ int flussPos = indexMapping == null ? pos : indexMapping[pos];
+ return flinkRow.getBinary(flussPos);
+ }
+
+ /** Converts a Flink {@link RowType} to a Fluss {@link RowType}, based on the field name. */
+ public static FlinkAsFlussRow from(
+ org.apache.flink.table.types.logical.RowType flinkRowType, RowType flussRowType) {
+ int[] indexMapping = new int[flussRowType.getFieldCount()];
+ List<String> fieldNames = flussRowType.getFieldNames();
+ for (int i = 0; i < flussRowType.getFieldCount(); i++) {
+ String fieldName = fieldNames.get(i);
+ int fieldIndex = flinkRowType.getFieldIndex(fieldName);
+ indexMapping[i] = fieldIndex;
+ }
+ return new FlinkAsFlussRow(indexMapping);
}
@Override
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java
index b615797a5..1b78292dd 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/FlinkRowDataChannelComputer.java
@@ -93,6 +93,8 @@ public class FlinkRowDataChannelComputer<InputT> implements ChannelComputer<Inpu
partitionGetter != null && numBucket % numChannels != 0;
try {
+ // no need to read real database, thus assume to deserialize the fluss row as same as
+ // flink table type.
this.serializationSchema.open(new SerializerInitContextImpl(flussRowType));
} catch (Exception e) {
throw new FlussRuntimeException(e);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
index 91cee80cd..0f3fff210 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
@@ -62,10 +62,17 @@ public interface FlussSerializationSchema<T> extends Serializable {
@PublicEvolving
interface InitializationContext {
/**
- * Returns the target row schema.
+ * Returns the Fluss physical row schema.
*
* @return The schema of the target row.
*/
RowType getRowSchema();
+
+ /**
+ * Returns the Flink table row schema.
+ *
+ * @return The schema of the Flink table row.
+ */
+ org.apache.flink.table.types.logical.RowType getTableRowType();
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
index adbfb7fb3..05e2921fc 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
@@ -63,7 +63,7 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
*/
@Override
public void open(InitializationContext context) throws Exception {
- this.converter = new FlinkAsFlussRow();
+ this.converter = FlinkAsFlussRow.from(context.getTableRowType(), context.getRowSchema());
}
/**
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
index b2cea1382..93b849e58 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
@@ -19,6 +19,7 @@ package org.apache.fluss.flink.sink.serializer;
import org.apache.fluss.types.RowType;
+import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/**
@@ -27,14 +28,26 @@ import static org.apache.fluss.utils.Preconditions.checkNotNull;
*/
public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
- private final RowType rowSchema;
+ private final RowType flussRowSchema;
+ private final org.apache.flink.table.types.logical.RowType flinkRowType;
public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
+ this(rowSchema, toFlinkRowType(rowSchema));
+ }
+
+ public SerializerInitContextImpl(
+ RowType rowSchema, org.apache.flink.table.types.logical.RowType flinkRowType) {
+ this.flussRowSchema = checkNotNull(rowSchema, "flussRowSchema");
+ this.flinkRowType = checkNotNull(flinkRowType, "flinkRowType");
}
@Override
public RowType getRowSchema() {
- return rowSchema;
+ return flussRowSchema;
+ }
+
+ @Override
+ public org.apache.flink.table.types.logical.RowType getTableRowType() {
+ return flinkRowType;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
index 901a8999f..a4469fd32 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
@@ -110,11 +110,15 @@ public abstract class FlinkSinkWriter<InputT> implements SinkWriter<InputT> {
metricGroup, Collections.singleton(MetricNames.WRITER_SEND_LATENCY_MS));
connection = ConnectionFactory.createConnection(flussConfig, flinkMetricRegistry);
table = connection.getTable(tablePath);
- sanityCheck(table.getTableInfo());
+ LOG.info(
+ "Current Fluss Schema is {}, Table RowType is {}",
+ table.getTableInfo().getSchema(),
+ tableRowType);
+ // sanityCheck(table.getTableInfo());
try {
this.serializationSchema.open(
- new SerializerInitContextImpl(table.getTableInfo().getRowType()));
+ new SerializerInitContextImpl(table.getTableInfo().getRowType(), tableRowType));
} catch (Exception e) {
throw new FlussRuntimeException(e);
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
index bc15390fe..0bf9f187c 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
@@ -60,7 +60,6 @@ public class FlinkSource<OUT>
private final boolean hasPrimaryKey;
private final boolean isPartitioned;
private final RowType sourceOutputType;
- @Nullable private final int[] projectedFields;
protected final OffsetsInitializer offsetsInitializer;
protected final long scanPartitionDiscoveryIntervalMs;
private final boolean streaming;
@@ -74,7 +73,6 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -86,7 +84,6 @@ public class FlinkSource<OUT>
hasPrimaryKey,
isPartitioned,
sourceOutputType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
@@ -101,7 +98,6 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -113,7 +109,6 @@ public class FlinkSource<OUT>
this.hasPrimaryKey = hasPrimaryKey;
this.isPartitioned = isPartitioned;
this.sourceOutputType = sourceOutputType;
- this.projectedFields = projectedFields;
this.offsetsInitializer = offsetsInitializer;
this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs;
this.deserializationSchema = deserializationSchema;
@@ -187,6 +182,7 @@ public class FlinkSource<OUT>
context.getUserCodeClassLoader(),
sourceOutputType));
FlinkRecordEmitter<OUT> recordEmitter = new FlinkRecordEmitter<>(deserializationSchema);
+ // recall to projectedFields
return new FlinkSourceReader<>(
elementsQueue,
@@ -194,7 +190,6 @@ public class FlinkSource<OUT>
tablePath,
sourceOutputType,
context,
- projectedFields,
flinkSourceReaderMetrics,
recordEmitter,
lakeSource);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
index ef4b63812..b110e9f88 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
@@ -325,7 +325,6 @@ public class FlinkTableSource
hasPrimaryKey(),
isPartitioned(),
flussRowType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
new RowDataDeserializationSchema(),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
index 427741834..5f0b2ff8c 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
@@ -24,8 +24,6 @@ import org.apache.fluss.flink.source.enumerator.initializer.OffsetsInitializer;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.types.RowType;
-import javax.annotation.Nullable;
-
/**
* A Flink DataStream source implementation for reading data from Fluss tables.
*
@@ -63,7 +61,6 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -75,7 +72,6 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
index f16639753..4891ab256 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
@@ -296,7 +296,6 @@ public class FlussSourceBuilder<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
- projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java
index 7c6405004..31238cf72 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/deserializer/RowDataDeserializationSchema.java
@@ -19,7 +19,6 @@ package org.apache.fluss.flink.source.deserializer;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.client.table.scanner.ScanRecord;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.record.LogRecord;
import org.apache.fluss.types.RowType;
@@ -28,6 +27,8 @@ import org.apache.flink.api.common.typeinfo.TypeInformation;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.runtime.typeutils.InternalTypeInfo;
+import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
+
/**
* A deserialization schema that converts {@link LogRecord} objects to Flink's {@link RowData}
* format.
@@ -89,6 +90,6 @@ public class RowDataDeserializationSchema implements FlussDeserializationSchema<
/** Returns the TypeInformation for the produced {@link RowData} type. */
@Override
public TypeInformation<RowData> getProducedType(RowType rowSchema) {
- return InternalTypeInfo.of(FlinkConversions.toFlinkRowType(rowSchema));
+ return InternalTypeInfo.of(toFlinkRowType(rowSchema));
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
index e1906712a..6e4647f80 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
@@ -27,12 +27,10 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.TableNotExistException;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
import org.apache.fluss.flink.source.lookup.LookupNormalizer.RemainingFilter;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.utils.ExceptionUtils;
import org.apache.flink.table.data.RowData;
@@ -87,6 +85,10 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
+ org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
+ LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
+ // Currently, only primary key and prefix lookup are supported. These keys must be primary
+ // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
@@ -95,8 +97,8 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
} else {
outputRowType = FlinkUtils.projectRowType(flinkRowType, projection);
}
- flussRowToFlinkRowConverter =
- new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
+
+ flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -162,7 +164,7 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
List<RowData> projectedRow = new ArrayList<>();
for (InternalRow row : lookupResult) {
if (row != null) {
- RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
+ RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRow.add(flinkRow);
}
@@ -171,14 +173,6 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
resultFuture.complete(projectedRow);
}
- private InternalRow maybeProject(InternalRow row) {
- if (projection == null) {
- return row;
- }
- // should not reuse objects for async operations
- return ProjectedRow.from(projection).replaceRow(row);
- }
-
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
index 79418f9b2..e9a5ad915 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
@@ -25,7 +25,6 @@ import org.apache.fluss.client.lookup.Lookuper;
import org.apache.fluss.client.table.Table;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
@@ -83,6 +82,10 @@ public class FlinkLookupFunction extends LookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
+ org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
+ LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
+ // Currently, only primary key and prefix lookup are supported. These keys must be primary
+ // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
@@ -94,8 +97,7 @@ public class FlinkLookupFunction extends LookupFunction {
// reuse the projected row
projectedRow = ProjectedRow.from(projection);
}
- flussRowToFlinkRowConverter =
- new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
+ flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -131,8 +133,7 @@ public class FlinkLookupFunction extends LookupFunction {
List<RowData> projectedRows = new ArrayList<>();
for (InternalRow row : lookupRows) {
if (row != null) {
- RowData flinkRow =
- flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
+ RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRows.add(flinkRow);
}
@@ -145,13 +146,6 @@ public class FlinkLookupFunction extends LookupFunction {
}
}
- private InternalRow maybeProject(InternalRow row) {
- if (projectedRow == null) {
- return row;
- }
- return projectedRow.replaceRow(row);
- }
-
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
index 6364532bb..639d52513 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
@@ -40,8 +40,6 @@ import org.apache.flink.api.connector.source.SourceReaderContext;
import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
-import javax.annotation.Nullable;
-
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
@@ -57,7 +55,6 @@ public class FlinkSourceReader<OUT>
TablePath tablePath,
RowType sourceOutputType,
SourceReaderContext context,
- @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
FlinkRecordEmitter<OUT> recordEmitter,
LakeSource<LakeSplit> lakeSource) {
@@ -70,7 +67,6 @@ public class FlinkSourceReader<OUT>
flussConfig,
tablePath,
sourceOutputType,
- projectedFields,
flinkSourceReaderMetrics,
lakeSource),
(ignore) -> {}),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
index fa0cb9e1b..10997e91e 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
@@ -17,6 +17,7 @@
package org.apache.fluss.flink.source.reader;
+import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.client.Connection;
import org.apache.fluss.client.ConnectionFactory;
import org.apache.fluss.client.table.Table;
@@ -57,7 +58,6 @@ import java.io.IOException;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -119,7 +119,6 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
Configuration flussConf,
TablePath tablePath,
RowType sourceOutputType,
- @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
@Nullable LakeSource<LakeSplit> lakeSource) {
this.flinkMetricRegistry =
@@ -129,13 +128,17 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
this.sourceOutputType = sourceOutputType;
this.boundedSplits = new ArrayDeque<>();
this.subscribedBuckets = new HashMap<>();
- this.projectedFields = projectedFields;
this.flinkSourceReaderMetrics = flinkSourceReaderMetrics;
- sanityCheck(table.getTableInfo().getRowType(), projectedFields);
+ this.projectedFields =
+ reCalculateProjectedFields(sourceOutputType, table.getTableInfo().getRowType());
this.logScanner = table.newScan().project(projectedFields).createLogScanner();
this.stoppingOffsets = new HashMap<>();
this.emptyLogSplits = new HashSet<>();
this.lakeSource = lakeSource;
+ LOG.info(
+ "fluss table schema: {}, flink table output type:{}",
+ table.getTableInfo().getSchema(),
+ sourceOutputType);
}
@Override
@@ -213,7 +216,8 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
LakeSnapshotAndFlussLogSplit lakeSnapshotAndFlussLogSplit =
(LakeSnapshotAndFlussLogSplit) sourceSplitBase;
if (lakeSnapshotAndFlussLogSplit.isStreaming()) {
- // is streaming split which has no stopping offset, we need also subscribe
+ // is streaming split which has no stopping offset, we need also
+ // subscribe
// change log
subscribeLog(
lakeSnapshotAndFlussLogSplit,
@@ -560,35 +564,33 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
flinkMetricRegistry.close();
}
- private void sanityCheck(RowType flussTableRowType, @Nullable int[] projectedFields) {
- RowType tableRowType =
- projectedFields != null
- ? flussTableRowType.project(projectedFields)
- : flussTableRowType;
- if (!sourceOutputType.copy(false).equals(tableRowType.copy(false))) {
- // The default nullability of Flink row type and Fluss row type might be not the same,
- // thus we need to compare the row type without nullability here.
-
- final String flussSchemaMsg;
- if (projectedFields == null) {
- flussSchemaMsg = "\nFluss table schema: " + tableRowType;
- } else {
- flussSchemaMsg =
- "\nFluss table schema: "
- + tableRowType
- + " (projection "
- + Arrays.toString(projectedFields)
- + ")";
+ /**
+ * The projected fields for the fluss table from the source output types. Mapping based on
+ * column name rather thn column id.
+ */
+ private static int[] reCalculateProjectedFields(
+ RowType sourceOutputType, RowType flussRowType) {
+ if (sourceOutputType.copy(false).equals(flussRowType.copy(false))) {
+ return null;
+ }
+
+ List<String> fieldNames = sourceOutputType.getFieldNames();
+ int[] projectedFlussFields = new int[fieldNames.size()];
+ for (int i = 0; i < fieldNames.size(); i++) {
+ int fieldIndex = flussRowType.getFieldIndex(fieldNames.get(i));
+ if (fieldIndex == -1) {
+ throw new ValidationException(
+ String.format(
+ "The field %s is not found in the fluss table.",
+ fieldNames.get(i)));
}
- // Throw exception if the schema is the not same, this should rarely happen because we
- // only allow fluss tables derived from fluss catalog. But this can happen if an ALTER
- // TABLE command executed on the fluss table, after the job is submitted but before the
- // SinkFunction is opened.
- throw new ValidationException(
- "The Flink query schema is not matched to Fluss table schema. "
- + "\nFlink query schema: "
- + sourceOutputType
- + flussSchemaMsg);
+ projectedFlussFields[i] = fieldIndex;
}
+ return projectedFlussFields;
+ }
+
+ @VisibleForTesting
+ public int[] getProjectedFields() {
+ return projectedFields;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
index 3256b3f52..74375e24c 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
@@ -58,12 +58,14 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
import java.util.stream.Collectors;
import static org.apache.flink.table.factories.FactoryUtil.CONNECTOR;
import static org.apache.flink.table.utils.EncodingUtils.decodeBase64ToBytes;
import static org.apache.flink.table.utils.EncodingUtils.encodeBytesToBase64;
import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.apache.flink.util.Preconditions.checkState;
import static org.apache.fluss.config.FlussConfigUtils.isTableStorageConfig;
import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_KEY;
import static org.apache.fluss.flink.FlinkConnectorOptions.BUCKET_NUMBER;
@@ -84,6 +86,18 @@ public class FlinkConversions {
private FlinkConversions() {}
+ public static int[] toFlinkRowTypeIndexMapping(
+ RowType sourceFlussRowType,
+ org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
+ int[] indexMapping = new int[targetFlinkRowType.getFieldCount()];
+ for (int i = 0; i < targetFlinkRowType.getFieldCount(); i++) {
+ String fieldName = targetFlinkRowType.getFieldNames().get(i);
+ int index = sourceFlussRowType.getFieldIndex(fieldName);
+ indexMapping[i] = index;
+ }
+ return indexMapping;
+ }
+
/** Convert Fluss's type to Flink's type. */
@VisibleForTesting
public static org.apache.flink.table.types.DataType toFlinkType(DataType flussDataType) {
@@ -394,6 +408,44 @@ public class FlinkConversions {
return Collections.singletonList(
convertSetOption(
(org.apache.flink.table.catalog.TableChange.SetOption) tableChange));
+ } else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.AddColumn) {
+ org.apache.flink.table.catalog.TableChange.AddColumn addColumn =
+ (org.apache.flink.table.catalog.TableChange.AddColumn) tableChange;
+ Column column = addColumn.getColumn();
+ return Collections.singletonList(
+ TableChange.addColumn(
+ column.getName(),
+ toFlussType(column.getDataType()),
+ column.getComment().orElse(null),
+ toFlussColumnPosition(addColumn.getPosition())));
+ } else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.DropColumn) {
+ return Collections.singletonList(
+ TableChange.dropColumn(
+ ((org.apache.flink.table.catalog.TableChange.DropColumn) tableChange)
+ .getColumnName()));
+ } else if (tableChange
+ instanceof org.apache.flink.table.catalog.TableChange.ModifyColumnName) {
+ org.apache.flink.table.catalog.TableChange.ModifyColumnName renameColumn =
+ (org.apache.flink.table.catalog.TableChange.ModifyColumnName) tableChange;
+ return Collections.singletonList(
+ TableChange.renameColumn(
+ renameColumn.getOldColumn().getName(),
+ renameColumn.getNewColumnName()));
+ } else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.ModifyColumn) {
+ org.apache.flink.table.catalog.TableChange.ModifyColumn modifyColumn =
+ (org.apache.flink.table.catalog.TableChange.ModifyColumn) tableChange;
+ checkState(
+ Objects.equals(
+ modifyColumn.getNewColumn().getName(),
+ modifyColumn.getOldColumn().getName()),
+ "Can only modify columns with the same name.");
+ Column newColumn = modifyColumn.getNewColumn();
+ return Collections.singletonList(
+ TableChange.modifyColumn(
+ newColumn.getName(),
+ toFlussType(newColumn.getDataType()),
+ newColumn.getComment().orElse(null),
+ toFlussColumnPosition(modifyColumn.getNewPosition())));
} else if (tableChange instanceof org.apache.flink.table.catalog.TableChange.ResetOption) {
return Collections.singletonList(
convertResetOption(
@@ -408,6 +460,21 @@ public class FlinkConversions {
}
}
+ private static TableChange.ColumnPosition toFlussColumnPosition(
+ org.apache.flink.table.catalog.TableChange.ColumnPosition columnPosition) {
+ if (columnPosition == null) {
+ return TableChange.ColumnPosition.last();
+ } else if (columnPosition instanceof org.apache.flink.table.catalog.TableChange.First) {
+ return TableChange.ColumnPosition.first();
+ } else if (columnPosition instanceof org.apache.flink.table.catalog.TableChange.After) {
+ return TableChange.ColumnPosition.after(
+ ((org.apache.flink.table.catalog.TableChange.After) columnPosition).column());
+ } else {
+ throw new UnsupportedOperationException(
+ String.format("Unsupported column position: %s.", columnPosition));
+ }
+ }
+
private static TableChange.SetOption convertSetOption(
org.apache.flink.table.catalog.TableChange.SetOption flinkSetOption) {
return TableChange.set(flinkSetOption.getKey(), flinkSetOption.getValue());
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
index e52a83a82..4dc5287b0 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
@@ -39,18 +39,32 @@ import org.apache.flink.types.RowKind;
import java.io.Serializable;
import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowKind;
+import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
/** A converter to convert Fluss's {@link InternalRow} to Flink's {@link RowData}. */
public class FlussRowToFlinkRowConverter {
private final FlussDeserializationConverter[] toFlinkFieldConverters;
private final InternalRow.FieldGetter[] flussFieldGetters;
+ private final int[] indexMapping;
+ // todo: remove
public FlussRowToFlinkRowConverter(RowType rowType) {
- this.toFlinkFieldConverters = new FlussDeserializationConverter[rowType.getFieldCount()];
- this.flussFieldGetters = new InternalRow.FieldGetter[rowType.getFieldCount()];
- for (int i = 0; i < rowType.getFieldCount(); i++) {
- toFlinkFieldConverters[i] = createNullableInternalConverter(rowType.getTypeAt(i));
- flussFieldGetters[i] = InternalRow.createFieldGetter(rowType.getTypeAt(i), i);
+ this(rowType, toFlinkRowType(rowType));
+ }
+
+ public FlussRowToFlinkRowConverter(
+ RowType sourceFlussRowType,
+ org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
+ this.indexMapping =
+ FlinkConversions.toFlinkRowTypeIndexMapping(sourceFlussRowType, targetFlinkRowType);
+ this.toFlinkFieldConverters =
+ new FlussDeserializationConverter[sourceFlussRowType.getFieldCount()];
+ this.flussFieldGetters = new InternalRow.FieldGetter[sourceFlussRowType.getFieldCount()];
+ for (int i = 0; i < sourceFlussRowType.getFieldCount(); i++) {
+ toFlinkFieldConverters[i] =
+ createNullableInternalConverter(sourceFlussRowType.getTypeAt(i));
+ flussFieldGetters[i] =
+ InternalRow.createFieldGetter(sourceFlussRowType.getTypeAt(i), i);
}
}
@@ -63,11 +77,18 @@ public class FlussRowToFlinkRowConverter {
}
private RowData toFlinkRowData(InternalRow flussRow, RowKind rowKind) {
- GenericRowData genericRowData = new GenericRowData(toFlinkFieldConverters.length);
+ int targetLength = indexMapping.length;
+ GenericRowData genericRowData = new GenericRowData(targetLength);
genericRowData.setRowKind(rowKind);
- for (int i = 0; i < toFlinkFieldConverters.length; i++) {
- Object flussField = flussFieldGetters[i].getFieldOrNull(flussRow);
- genericRowData.setField(i, toFlinkFieldConverters[i].deserialize(flussField));
+ for (int i = 0; i < targetLength; i++) {
+ int flussIndex = indexMapping[i];
+ if (flussIndex == -1) {
+ genericRowData.setField(i, null);
+ } else {
+ Object flussField = flussFieldGetters[flussIndex].getFieldOrNull(flussRow);
+ genericRowData.setField(
+ i, toFlinkFieldConverters[flussIndex].deserialize(flussField));
+ }
}
return genericRowData;
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java
index 88e16e996..570d7121a 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/catalog/FlinkCatalogITCase.java
@@ -203,7 +203,7 @@ abstract class FlinkCatalogITCase {
}
@Test
- void testAlterTable() throws Exception {
+ void testAlterTableConfig() throws Exception {
String ddl =
"create table test_alter_table_append_only ("
+ "a string, "
@@ -286,6 +286,25 @@ abstract class FlinkCatalogITCase {
"Property 'paimon.file.format' is not supported to alter which is for datalake table.");
}
+ @Test
+ void testAlterTableSchema() throws Exception {
+ ObjectPath objectPath = new ObjectPath(DEFAULT_DB, "append_only_table");
+ tEnv.executeSql(
+ "create table append_only_table(a int, b STRING) with ('bucket.num' = '10')")
+ .await();
+ tEnv.executeSql("alter table append_only_table add c int").await();
+ CatalogTable table = (CatalogTable) catalog.getTable(objectPath);
+
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ Schema expectedSchema =
+ schemaBuilder
+ .column("a", DataTypes.INT())
+ .column("b", DataTypes.STRING())
+ .column("c", DataTypes.INT())
+ .build();
+ assertThat(table.getUnresolvedSchema()).isEqualTo(expectedSchema);
+ }
+
@Test
void testCreateUnSupportedTable() {
// test invalid property
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
index 73ba044d8..1f147751a 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
@@ -31,6 +31,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
+import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.offset;
@@ -59,7 +60,7 @@ class FlinkAsFlussRowTest {
TimestampData.fromEpochMillis(1672531200000L, 3),
new byte[] {1, 2, 3},
null);
- row = new FlinkAsFlussRow().replace(flinkRow);
+ row = new FlinkAsFlussRow(IntStream.range(0, 14).toArray()).replace(flinkRow);
}
@Test
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java
index 0ea8ed94f..a22316d70 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/FlinkTableSinkITCase.java
@@ -165,6 +165,52 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
assertResultsIgnoreOrder(rowIter, expectedRows, true);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testAppendLogDuringAddColumn(boolean compressed) throws Exception {
+ String compressedProperties =
+ compressed
+ ? ",'table.log.format' = 'arrow', 'table.log.arrow.compression.type' = 'zstd'"
+ : "";
+ tEnv.executeSql(
+ "create table sink_test (a int not null, b bigint, c string) with "
+ + "('bucket.num' = '3'"
+ + compressedProperties
+ + ")");
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (1, 3501, 'Tim'), "
+ + "(2, 3502, 'Fabian'), "
+ + "(3, 3503, 'coco') ")
+ .await();
+
+ CloseableIterator<Row> rowIter = tEnv.executeSql("select * from sink_test").collect();
+ // add new column
+ tEnv.executeSql("alter table sink_test add add_column int").await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, "sink_test"), 2);
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (4, 3504, 'jerry', 4), "
+ + "(5, 3505, 'piggy', 5), "
+ + "(6, 3506, 'stave', 6)")
+ .await();
+ List<String> expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim]", "+I[2, 3502, Fabian]",
+ "+I[3, 3503, coco]", "+I[4, 3504, jerry]",
+ "+I[5, 3505, piggy]", "+I[6, 3506, stave]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+
+ // read with new schema.
+ rowIter = tEnv.executeSql("select * from sink_test").collect();
+ expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim, null]", "+I[2, 3502, Fabian, null]",
+ "+I[3, 3503, coco, null]", "+I[4, 3504, jerry, 4]",
+ "+I[5, 3505, piggy, 5]", "+I[6, 3506, stave, 6]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+ }
+
@ParameterizedTest
@ValueSource(booleans = {true, false})
void testAppendLogWithBucketKey(boolean sinkBucketShuffle) throws Exception {
@@ -356,6 +402,45 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
assertResultsIgnoreOrder(rowIter, expectedRows, true);
}
+ @Test
+ void testPutDuringAddColumn() throws Exception {
+ tEnv.executeSql(
+ "create table sink_test (a int not null primary key not enforced, b bigint, c string)");
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (1, 3501, 'Tim'), "
+ + "(2, 3502, 'Fabian'), "
+ + "(3, 3503, 'coco') ")
+ .await();
+
+ CloseableIterator<Row> rowIter = tEnv.executeSql("select * from sink_test").collect();
+ // add new column
+ tEnv.executeSql("alter table sink_test add add_column int").await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, "sink_test"), 2);
+ tEnv.executeSql(
+ "INSERT INTO sink_test "
+ + "VALUES (4, 3504, 'jerry', 4), "
+ + "(5, 3505, 'piggy', 5), "
+ + "(6, 3506, 'stave', 6)")
+ .await();
+ List<String> expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim]", "+I[2, 3502, Fabian]",
+ "+I[3, 3503, coco]", "+I[4, 3504, jerry]",
+ "+I[5, 3505, piggy]", "+I[6, 3506, stave]");
+ // read with old schema
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+
+ // read with new schema.
+ rowIter = tEnv.executeSql("select * from sink_test").collect();
+ expectedRows =
+ Arrays.asList(
+ "+I[1, 3501, Tim, null]", "+I[2, 3502, Fabian, null]",
+ "+I[3, 3503, coco, null]", "+I[4, 3504, jerry, 4]",
+ "+I[5, 3505, piggy, 5]", "+I[6, 3506, stave, 6]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+ }
+
@Test
void testPartialUpsert() throws Exception {
tEnv.executeSql(
@@ -407,6 +492,58 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
assertResultsIgnoreOrder(rowIter, expectedRows, true);
}
+ @Test
+ void testPartialUpsertDuringAddColumn() throws Exception {
+ tEnv.executeSql(
+ "create table sink_test (a int not null primary key not enforced, b bigint, c string) with('bucket.num' = '3')");
+
+ // partial insert
+ tEnv.executeSql("INSERT INTO sink_test(a, b) VALUES (1, 111), (2, 222)").await();
+ tEnv.executeSql("INSERT INTO sink_test(c, a) VALUES ('c1', 1), ('c2', 2)").await();
+
+ CloseableIterator<Row> rowIter = tEnv.executeSql("select * from sink_test").collect();
+ // add new column
+ tEnv.executeSql("alter table sink_test add add_column string").await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, "sink_test"), 2);
+ tEnv.executeSql(
+ "INSERT INTO sink_test(add_column, a ) VALUES ('new_value', 1), ('new_value', 2)")
+ .await();
+
+ // read with old schema
+ List<String> expectedRows =
+ Arrays.asList(
+ "+I[1, 111, null]",
+ "+I[2, 222, null]",
+ "-U[1, 111, null]",
+ "+U[1, 111, c1]",
+ "-U[2, 222, null]",
+ "+U[2, 222, c2]",
+ "-U[1, 111, c1]",
+ "+U[1, 111, c1]",
+ "-U[2, 222, c2]",
+ "+U[2, 222, c2]");
+ assertResultsIgnoreOrder(rowIter, expectedRows, true);
+
+ // read with new schema
+ CloseableIterator<Row> newSchemaRowIter =
+ tEnv.executeSql(
+ "select * from sink_test /*+ OPTIONS('scan.startup.mode' = 'earliest') */ ")
+ .collect();
+ expectedRows =
+ Arrays.asList(
+ "+I[1, 111, null, null]",
+ "+I[2, 222, null, null]",
+ "-U[1, 111, null, null]",
+ "+U[1, 111, c1, null]",
+ "-U[2, 222, null, null]",
+ "+U[2, 222, c2, null]",
+ "-U[1, 111, c1, null]",
+ "+U[1, 111, c1, new_value]",
+ "-U[2, 222, c2, null]",
+ "+U[2, 222, c2, new_value]");
+ assertResultsIgnoreOrder(newSchemaRowIter, expectedRows, true);
+ }
+
@Test
void testFirstRowMergeEngine() throws Exception {
tEnv.executeSql(
@@ -676,6 +813,21 @@ abstract class FlinkTableSinkITCase extends AbstractTestBase {
"-U[4, 3504, Seattle]",
"+U[4, 3504, New York]");
assertResultsIgnoreOrder(changelogIter, expectedRows, true);
+
+ // test schema evolution.
+ tBatchEnv
+ .executeSql(String.format("alter table %s add new_added_column int", tableName))
+ .await();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(TablePath.of(DEFAULT_DB, tableName), 2);
+ tBatchEnv
+ .executeSql("UPDATE " + tableName + " SET new_added_column = 2 WHERE a = 4")
+ .await();
+ CloseableIterator<Row> row5 =
+ tBatchEnv
+ .executeSql(String.format("select * from %s WHERE a = 4", tableName))
+ .collect();
+ expected = Collections.singletonList("+I[4, 3504, New York, 2]");
+ assertResultsIgnoreOrder(row5, expected, true);
}
@Test
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java
index f65da98e6..9ac81ba6d 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriterTest.java
@@ -24,13 +24,14 @@ import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.NetworkException;
import org.apache.fluss.flink.sink.serializer.RowDataSerializationSchema;
-import org.apache.fluss.flink.sink.serializer.SerializerInitContextImpl;
import org.apache.fluss.flink.utils.FlinkTestBase;
import org.apache.fluss.metadata.DatabaseDescriptor;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.server.testutils.FlussClusterExtension;
+import org.apache.fluss.types.DataTypes;
import org.apache.flink.api.common.operators.MailboxExecutor;
import org.apache.flink.api.connector.sink2.SinkWriter;
@@ -50,9 +51,9 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
+import java.util.Collections;
import java.util.function.BiConsumer;
-import static org.apache.fluss.flink.utils.FlinkConversions.toFlussRowType;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
@@ -106,6 +107,52 @@ public class FlinkSinkWriterTest extends FlinkTestBase {
flinkSinkWriter.close();
}
+ @Test
+ void testFlussSchemaChange() throws Exception {
+ admin.createDatabase(
+ DEFAULT_SINK_TABLE_PATH.getDatabaseName(), DatabaseDescriptor.EMPTY, true);
+ createTable(DEFAULT_SINK_TABLE_PATH, TABLE_DESCRIPTOR);
+ admin.alterTable(
+ DEFAULT_SINK_TABLE_PATH,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+
+ Configuration clientConfig = FLUSS_CLUSTER_EXTENSION.getClientConfig();
+
+ MockWriterInitContext mockWriterInitContext =
+ new MockWriterInitContext(new InterceptingOperatorMetricGroup());
+ try (FlinkSinkWriter<RowData> writer =
+ createSinkWriter(clientConfig, mockWriterInitContext.getMailboxExecutor())) {
+ writer.initialize(mockWriterInitContext.metricGroup());
+ // case1: write data which is matched flink schema.
+ writer.write(
+ GenericRowData.of(1, StringData.fromString("a")), new MockSinkWriterContext());
+ writer.flush(false);
+
+ // case2: write data which lacks the last column of flink schema
+ writer.write(GenericRowData.of(1), new MockSinkWriterContext());
+ writer.flush(false);
+
+ // case3: write data which has reorder column of flink schema
+ assertThatThrownBy(
+ () ->
+ writer.write(
+ GenericRowData.of(StringData.fromString("a"), 1),
+ new MockSinkWriterContext()))
+ .rootCause()
+ .isExactlyInstanceOf(ClassCastException.class)
+ .hasMessageContaining(
+ "class org.apache.flink.table.data.binary.BinaryStringData cannot be cast to class java.lang.Integer");
+ writer.flush(false);
+ }
+ }
+
@Test
void testWriteExceptionWhenFlussUnavailable() throws Exception {
testExceptionWhenFlussUnavailable(
@@ -210,7 +257,6 @@ public class FlinkSinkWriterTest extends FlinkTestBase {
new String[] {"id", "name"});
RowDataSerializationSchema serializationSchema =
new RowDataSerializationSchema(true, false);
- serializationSchema.open(new SerializerInitContextImpl(toFlussRowType(tableRowType)));
return new AppendSinkWriter<>(
DEFAULT_SINK_TABLE_PATH,
configuration,
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java
index f235afc56..8a5f3b81d 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/deserializer/FlussDeserializationSchemaTest.java
@@ -37,7 +37,9 @@ import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.time.LocalDate;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import static org.assertj.core.api.Assertions.assertThat;
@@ -125,7 +127,7 @@ public class FlussDeserializationSchemaTest {
@Test
public void testJsonStringDeserialize() throws Exception {
- List<DataField> fields =
+ List<DataField> sourceFields =
Arrays.asList(
new DataField("char", DataTypes.CHAR(64)),
new DataField("string", DataTypes.STRING()),
@@ -145,7 +147,12 @@ public class FlussDeserializationSchemaTest {
new DataField("timestampWithLocalTimeZone", DataTypes.TIMESTAMP_LTZ()),
new DataField("nullVal", DataTypes.STRING()));
- RowType rowType = new RowType(fields);
+ RowType sourceRowType = new RowType(sourceFields);
+
+ List<DataField> targetFields = new ArrayList<>(sourceFields);
+ targetFields.add(new DataField("new_column", DataTypes.STRING()));
+ Collections.reverse(targetFields);
+ RowType targetRowType = new RowType(targetFields);
// Mon Apr 21 2025 10:00:00 GMT+0000
long testTimestampInSeconds = 1745229600;
@@ -174,7 +181,7 @@ public class FlussDeserializationSchemaTest {
// Create deserializer
JsonStringDeserializationSchema deserializer = new JsonStringDeserializationSchema();
// Test deserialization
- deserializer.open(new DeserializerInitContextImpl(null, null, rowType));
+ deserializer.open(new DeserializerInitContextImpl(null, null, sourceRowType));
String result = deserializer.deserialize(scanRecord);
String rowJson =
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
index 297c6a16d..596780302 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
@@ -21,13 +21,19 @@ import org.apache.fluss.client.table.Table;
import org.apache.fluss.client.table.writer.UpsertWriter;
import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkTestBase;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableDescriptor;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.types.DataTypes;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.functions.AsyncLookupFunction;
import org.apache.flink.table.types.logical.RowType;
import org.apache.flink.util.Collector;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
import java.util.ArrayList;
import java.util.Collection;
@@ -44,10 +50,11 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link FlinkLookupFunction} and {@link FlinkAsyncLookupFunction}. */
class FlinkLookupFunctionTest extends FlinkTestBase {
- @Test
- void testSyncLookupEval() throws Exception {
- TablePath tablePath = TablePath.of(DEFAULT_DB, "sync-lookup-table");
- prepareData(tablePath, 3);
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testSyncLookupEval(boolean schemaNotMatch) throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "sync-lookup-table-" + schemaNotMatch);
+ prepareData(tablePath, 3, schemaNotMatch);
RowType flinkRowType =
FlinkConversions.toFlinkRowType(DEFAULT_PK_TABLE_SCHEMA.getRowType());
@@ -80,11 +87,12 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
lookupFunction.close();
}
- @Test
- void testAsyncLookupEval() throws Exception {
- TablePath tablePath = TablePath.of(DEFAULT_DB, "async-lookup-table");
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void testAsyncLookupEval(boolean schemaNotMatch) throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "async-lookup-table-" + schemaNotMatch);
int rows = 3;
- prepareData(tablePath, rows);
+ prepareData(tablePath, rows, schemaNotMatch);
RowType flinkRowType =
FlinkConversions.toFlinkRowType(DEFAULT_PK_TABLE_SCHEMA.getRowType());
@@ -133,14 +141,107 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
"4: null");
}
- private void prepareData(TablePath tablePath, int rows) throws Exception {
- createTable(tablePath, DEFAULT_PK_TABLE_DESCRIPTOR);
+ @Test
+ void testSchemaChange() throws Exception {
+ TablePath tablePath = TablePath.of(DEFAULT_DB, "sync-add-column");
+ prepareData(tablePath, 3, false);
+
+ RowType flinkRowType =
+ FlinkConversions.toFlinkRowType(DEFAULT_PK_TABLE_SCHEMA.getRowType());
+ FlinkLookupFunction lookupFunction =
+ new FlinkLookupFunction(
+ clientConf,
+ tablePath,
+ flinkRowType,
+ createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ null);
+
+ ListOutputCollector collector = new ListOutputCollector();
+ lookupFunction.setCollector(collector);
+ lookupFunction.open(null);
+
+ // alter table after lookup function is created.
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "new_column",
+ DataTypes.INT(),
+ null,
+ TableChange.ColumnPosition.last())),
+ false)
+ .get();
+ FLUSS_CLUSTER_EXTENSION.waitAllSchemaSync(tablePath, 2);
+
+ try (Table table = conn.getTable(tablePath)) {
+ UpsertWriter upsertWriter = table.newUpsert().createWriter();
+ upsertWriter.upsert(row(3, "name3", 3));
+ upsertWriter.flush();
+ }
+
+ // look up
+ for (int i = 0; i < 5; i++) {
+ lookupFunction.eval(i);
+ }
+
+ // collect the result and check
+ List<String> result =
+ collector.getOutputs().stream()
+ .map(RowData::toString)
+ .sorted()
+ .collect(Collectors.toList());
+ assertThat(result)
+ .containsExactly("+I(0,name0)", "+I(1,name1)", "+I(2,name2)", "+I(3,name3)");
+ lookupFunction.close();
+
+ // start lookup job after schema change.
+ lookupFunction =
+ new FlinkLookupFunction(
+ clientConf,
+ tablePath,
+ flinkRowType,
+ createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ null);
+ collector = new ListOutputCollector();
+ lookupFunction.setCollector(collector);
+ lookupFunction.open(null);
+ for (int i = 0; i < 5; i++) {
+ lookupFunction.eval(i);
+ }
+ result =
+ collector.getOutputs().stream()
+ .map(RowData::toString)
+ .sorted()
+ .collect(Collectors.toList());
+ assertThat(result)
+ .containsExactly("+I(0,name0)", "+I(1,name1)", "+I(2,name2)", "+I(3,name3)");
+ lookupFunction.close();
+ }
+
+ private void prepareData(TablePath tablePath, int rows, boolean schemaNotMatch)
+ throws Exception {
+ TableDescriptor descriptor = DEFAULT_PK_TABLE_DESCRIPTOR;
+ if (schemaNotMatch) {
+ descriptor =
+ TableDescriptor.builder()
+ .schema(
+ Schema.newBuilder()
+ .primaryKey("id")
+ .column("name", DataTypes.STRING())
+ .column("id", DataTypes.INT())
+ .column("age", DataTypes.INT())
+ .build())
+ .distributedBy(DEFAULT_BUCKET_NUM, "id")
+ .build();
+ }
+
+ createTable(tablePath, descriptor);
// first write some data to the table
try (Table table = conn.getTable(tablePath)) {
UpsertWriter upsertWriter = table.newUpsert().createWriter();
for (int i = 0; i < rows; i++) {
- upsertWriter.upsert(row(i, "name" + i));
+ upsertWriter.upsert(schemaNotMatch ? row("name" + i, i, i) : row(i, "name" + i));
}
upsertWriter.flush();
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
index 6634236aa..88e2cb539 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
@@ -178,7 +178,6 @@ class FlinkSourceReaderTest extends FlinkTestBase {
tablePath,
sourceOutputType,
context,
- null,
new FlinkSourceReaderMetrics(context.metricGroup()),
recordEmitter,
null);
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
index 19aa0f7a9..c30c9cf72 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
@@ -85,35 +85,23 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
clientConf,
tablePath1,
DataTypes.ROW(
- DataTypes.FIELD("id", DataTypes.BIGINT()),
- DataTypes.FIELD("name", DataTypes.STRING()),
- DataTypes.FIELD("age", DataTypes.INT())),
- null,
+ DataTypes.FIELD("name2", DataTypes.STRING()),
+ DataTypes.FIELD("id", DataTypes.BIGINT())),
createMockSourceReaderMetrics(),
null))
.isInstanceOf(ValidationException.class)
- .hasMessage(
- "The Flink query schema is not matched to Fluss table schema. \n"
- + "Flink query schema: ROW<`id` BIGINT, `name` STRING, `age` INT>\n"
- + "Fluss table schema: ROW<`id` BIGINT NOT NULL, `name` STRING, `age` INT>");
-
- assertThatThrownBy(
- () ->
- new FlinkSourceSplitReader(
- clientConf,
- tablePath1,
- DataTypes.ROW(
- DataTypes.FIELD(
- "id", DataTypes.BIGINT().copy(false)),
- DataTypes.FIELD("name", DataTypes.STRING())),
- new int[] {1, 0},
- createMockSourceReaderMetrics(),
- null))
- .isInstanceOf(ValidationException.class)
- .hasMessage(
- "The Flink query schema is not matched to Fluss table schema. \n"
- + "Flink query schema: ROW<`id` BIGINT NOT NULL, `name` STRING>\n"
- + "Fluss table schema: ROW<`name` STRING, `id` BIGINT NOT NULL> (projection [1, 0])");
+ .hasMessage("The field name2 is not found in the fluss table.");
+
+ FlinkSourceSplitReader flinkSourceSplitReader =
+ new FlinkSourceSplitReader(
+ clientConf,
+ tablePath1,
+ DataTypes.ROW(
+ DataTypes.FIELD("name", DataTypes.STRING()),
+ DataTypes.FIELD("id", DataTypes.BIGINT().copy(false))),
+ createMockSourceReaderMetrics(),
+ null);
+ assertThat(flinkSourceSplitReader.getProjectedFields()).isEqualTo(new int[] {1, 0});
}
@Test
@@ -396,7 +384,7 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
private FlinkSourceSplitReader createSplitReader(TablePath tablePath, RowType rowType) {
return new FlinkSourceSplitReader(
- clientConf, tablePath, rowType, null, createMockSourceReaderMetrics(), null);
+ clientConf, tablePath, rowType, createMockSourceReaderMetrics(), null);
}
private FlinkSourceReaderMetrics createMockSourceReaderMetrics() {
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java
index 6f0ab20f4..08dc96468 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/tiering/committer/TieringCommitOperatorTest.java
@@ -132,7 +132,7 @@ class TieringCommitOperatorTest extends FlinkTestBase {
tablePath2, t2b1, 2, 22, 32L, numberOfWriteResults));
verifyNoLakeSnapshot(tablePath2);
- // add table1, bucket2
+ // addColumn table1, bucket2
TableBucket t1b2 = new TableBucket(tableId1, 2);
committerOperator.processElement(
createTableBucketWriteResultStreamRecord(
@@ -148,7 +148,7 @@ class TieringCommitOperatorTest extends FlinkTestBase {
expectedMaxTimestamps.put(t1b2, 23L);
verifyLakeSnapshot(tablePath1, tableId1, 1, expectedLogEndOffsets, expectedMaxTimestamps);
- // add table2, bucket2
+ // addColumn table2, bucket2
TableBucket t2b2 = new TableBucket(tableId2, 2);
committerOperator.processElement(
createTableBucketWriteResultStreamRecord(
diff --git a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
index 5aad1d5bf..9c86a92c1 100644
--- a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
+++ b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
@@ -79,7 +79,8 @@ public class HadoopUtils {
}
// Approach 3: Fluss configuration
- // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop conf
+ // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop
+ // conf
for (String key : flussConfiguration.keySet()) {
for (String prefix : FLUSS_CONFIG_PREFIXES) {
if (key.startsWith(prefix)) {
@@ -108,7 +109,7 @@ public class HadoopUtils {
}
/**
- * Search Hadoop configuration files in the given path, and add them to the configuration if
+ * Search Hadoop configuration files in the given path, andadd them to the configuration if
* found.
*/
private static boolean addHadoopConfIfFound(
diff --git a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
index 4b6d9de71..133c23322 100644
--- a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
+++ b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
@@ -147,7 +147,7 @@ class IcebergRewriteITCase extends FlinkIcebergTieringTestBase {
// add pos-delete and trigger compaction
rows = Arrays.asList(row(4, "v1"), row(4, "v2"));
flussRows.add(writeIcebergTableRecords(t1, t1Bucket, 6, false, rows).get(1));
- // rewritten files should fail to commit due to conflict, add check here
+ // rewritten files should fail to commit due to conflict,add check here
checkRecords(getIcebergRecords(t1), flussRows);
// 4 data file and 1 delete file
checkFileStatusInIcebergTable(t1, 4, true);
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
index a09a50c09..7f4d6e84e 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
@@ -69,7 +69,8 @@ final class ServerConnection {
private final ServerNode node;
- // TODO: add max inflight requests limit like Kafka's "max.in.flight.requests.per.connection"
+ // TODO: add max inflight requests limit like Kafka's
+ // "max.in.flight.requests.per.connection"
private final Map<Integer, InflightRequest> inflightRequests = MapUtils.newConcurrentHashMap();
private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
private final ConnectionMetrics connectionMetrics;
diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto
index ca41ebcb0..9feb38dc1 100644
--- a/fluss-rpc/src/main/proto/FlussApi.proto
+++ b/fluss-rpc/src/main/proto/FlussApi.proto
@@ -113,9 +113,12 @@ message AlterTableRequest {
required PbTablePath table_path = 1;
required bool ignore_if_not_exists = 2;
repeated PbAlterConfig config_changes = 3;
+ repeated PbAddColumn add_columns = 4;
+ repeated PbDropColumn drop_columns = 5;
+ repeated PbRenameColumn rename_columns = 6;
+ repeated PbModifyColumn modify_columns = 7;
}
-
message AlterTableResponse {
}
@@ -915,6 +918,32 @@ message PbAlterConfig {
required int32 op_type = 3; // SET=0, DELETE=1, APPEND=2, SUBTRACT=3
}
+message PbAddColumn{
+ required string column_name = 1;
+ required bytes data_type_json = 2;
+ optional string comment = 3;
+ required int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
+}
+
+message PbDropColumn{
+ required string column_name = 1;
+}
+
+message PbRenameColumn{
+ required string old_column_name = 1;
+ required string new_column_name = 2;
+}
+
+
+message PbModifyColumn{
+ required string column_name = 1;
+ required bytes data_type_json = 2;
+ optional string comment = 3;
+ optional int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
+}
+
+
+
message PbDescribeConfig{
required string config_key = 1;
optional string config_value = 2;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
index 82f3d5ec2..ededf510f 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
@@ -384,7 +384,7 @@ public abstract class RpcServiceBase extends RpcGatewayService implements AdminR
@Override
public CompletableFuture<GetFileSystemSecurityTokenResponse> getFileSystemSecurityToken(
GetFileSystemSecurityTokenRequest request) {
- // TODO: add ACL for per-table in https://github.com/apache/fluss/issues/752
+ // TODO:add ACL for per-table in https://github.com/apache/fluss/issues/752
try {
// In order to avoid repeatedly obtaining security token, cache it for a while.
long currentTimeMs = System.currentTimeMillis();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
index b7c92289b..def57231d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
@@ -148,7 +148,7 @@ public class CompletedSnapshotStoreManager {
() -> getAllSnapshotSize(bucket));
} else {
LOG.warn(
- "Failed to add bucketMetricGroup for tableBucket {} when creating completed snapshot.",
+ "Failed toadd bucketMetricGroup for tableBucket {} when creating completed snapshot.",
bucket);
}
return snapshotStore;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index b3fed108b..fbe0986b1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -31,6 +31,7 @@ import org.apache.fluss.exception.InvalidUpdateVersionException;
import org.apache.fluss.exception.TabletServerNotAvailableException;
import org.apache.fluss.exception.UnknownTableOrBucketException;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableBucketReplica;
import org.apache.fluss.metadata.TableInfo;
@@ -62,6 +63,7 @@ import org.apache.fluss.server.coordinator.event.FencedCoordinatorEvent;
import org.apache.fluss.server.coordinator.event.NewTabletServerEvent;
import org.apache.fluss.server.coordinator.event.NotifyKvSnapshotOffsetEvent;
import org.apache.fluss.server.coordinator.event.NotifyLeaderAndIsrResponseReceivedEvent;
+import org.apache.fluss.server.coordinator.event.SchemaChangeEvent;
import org.apache.fluss.server.coordinator.event.watcher.TableChangeWatcher;
import org.apache.fluss.server.coordinator.event.watcher.TabletServerChangeWatcher;
import org.apache.fluss.server.coordinator.statemachine.ReplicaStateMachine;
@@ -499,6 +501,9 @@ public class CoordinatorEventProcessor implements EventProcessor {
processDropTable((DropTableEvent) event);
} else if (event instanceof DropPartitionEvent) {
processDropPartition((DropPartitionEvent) event);
+ } else if (event instanceof SchemaChangeEvent) {
+ SchemaChangeEvent schemaChangeEvent = (SchemaChangeEvent) event;
+ processSchemaChange(schemaChangeEvent);
} else if (event instanceof NotifyLeaderAndIsrResponseReceivedEvent) {
processNotifyLeaderAndIsrResponseReceivedEvent(
(NotifyLeaderAndIsrResponseReceivedEvent) event);
@@ -551,6 +556,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
private void processCreateTable(CreateTableEvent createTableEvent) {
long tableId = createTableEvent.getTableInfo().getTableId();
+ int schemaId = createTableEvent.getTableInfo().getSchemaId();
// skip the table if it already exists
if (coordinatorContext.containsTableId(tableId)) {
return;
@@ -595,6 +601,44 @@ public class CoordinatorEventProcessor implements EventProcessor {
}
}
+ private void processSchemaChange(SchemaChangeEvent schemaChangeEvent) {
+ TablePath tablePath = schemaChangeEvent.getTablePath();
+ SchemaInfo schemaInfo = schemaChangeEvent.getSchemaInfo();
+ long tableId = coordinatorContext.getTableIdByPath(tablePath);
+
+ // if table is created, will create schema first. In this case, schema id will be included
+ // in CreateTableEvent.
+ if (tableId == TableInfo.UNKNOWN_TABLE_ID) {
+ return;
+ }
+
+ TableInfo oldTableInfo = coordinatorContext.getTableInfoById(tableId);
+ if (oldTableInfo.getSchemaId() == schemaInfo.getSchemaId()) {
+ return;
+ }
+
+ coordinatorContext.putTableInfo(
+ new TableInfo(
+ tablePath,
+ tableId,
+ schemaInfo.getSchemaId(),
+ schemaInfo.getSchema(),
+ oldTableInfo.getBucketKeys(),
+ oldTableInfo.getPartitionKeys(),
+ oldTableInfo.getNumBuckets(),
+ oldTableInfo.getProperties(),
+ oldTableInfo.getCustomProperties(),
+ oldTableInfo.getComment().orElse(null),
+ oldTableInfo.getCreatedTime(),
+ System.currentTimeMillis()));
+
+ updateTabletServerMetadataCache(
+ new HashSet<>(coordinatorContext.getLiveTabletServers().values()),
+ tableId,
+ null,
+ null);
+ }
+
private void processCreatePartition(CreatePartitionEvent createPartitionEvent) {
long partitionId = createPartitionEvent.getPartitionId();
// skip the partition if it already exists
@@ -1310,6 +1354,7 @@ public class CoordinatorEventProcessor implements EventProcessor {
aliveTabletServers.stream().map(ServerInfo::id).collect(Collectors.toSet());
Set<Long> tablesToBeDeleted = coordinatorContext.getTablesToBeDeleted();
+
tablesToBeDeleted.forEach(
tableId ->
coordinatorRequestBatch.addUpdateMetadataRequestForTabletServers(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java
index de6778d35..982995d21 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorRequestBatch.java
@@ -279,8 +279,10 @@ public class CoordinatorRequestBatch {
* <li>case7: Leader and isr is changed for these input tableBuckets
* <li>case8: One newly tabletServer added into cluster
* <li>case9: One tabletServer is removed from cluster
+ * <li>case10: schemaId is changed after table is created.
* </ol>
*/
+ // todo: improve this with different phase enum.
public void addUpdateMetadataRequestForTabletServers(
Set<Integer> tabletServers,
@Nullable Long tableId,
@@ -298,7 +300,7 @@ public class CoordinatorRequestBatch {
.computeIfAbsent(tableId, k -> new HashMap<>())
.put(partitionId, Collections.emptyList());
} else {
- // case3, case4
+ // case3, case4, case10
updateMetadataRequestBucketMap.put(tableId, Collections.emptyList());
}
} else {
@@ -306,6 +308,7 @@ public class CoordinatorRequestBatch {
for (TableBucket tableBucket : tableBuckets) {
long currentTableId = tableBucket.getTableId();
Long currentPartitionId = tableBucket.getPartitionId();
+ // case1, case2
Optional<LeaderAndIsr> bucketLeaderAndIsr =
coordinatorContext.getBucketLeaderAndIsr(tableBucket);
Integer leaderEpoch =
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java
index 54480d663..73a760337 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorService.java
@@ -135,7 +135,8 @@ import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getCommitRemot
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.getPartitionSpec;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeCreateAclsResponse;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.makeDropAclsResponse;
-import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toTableChanges;
+import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toAlterTableConfigChanges;
+import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toAlterTableSchemaChanges;
import static org.apache.fluss.server.utils.ServerRpcMessageUtils.toTablePath;
import static org.apache.fluss.server.utils.TableAssignmentUtils.generateAssignment;
import static org.apache.fluss.utils.PartitionUtils.validatePartitionSpec;
@@ -312,19 +313,35 @@ public final class CoordinatorService extends RpcServiceBase implements Coordina
authorizer.authorize(currentSession(), OperationType.ALTER, Resource.table(tablePath));
}
- List<TableChange> tableChanges = toTableChanges(request.getConfigChangesList());
- TablePropertyChanges tablePropertyChanges = toTablePropertyChanges(tableChanges);
+ List<TableChange> alterTableConfigChanges =
+ toAlterTableConfigChanges(request.getConfigChangesList());
+ TablePropertyChanges tablePropertyChanges = toTablePropertyChanges(alterTableConfigChanges);
+ List<TableChange> alterSchemaChanges = toAlterTableSchemaChanges(request);
- LakeCatalogDynamicLoader.LakeCatalogContainer lakeCatalogContainer =
- lakeCatalogDynamicLoader.getLakeCatalogContainer();
- metadataManager.alterTableProperties(
- tablePath,
- tableChanges,
- tablePropertyChanges,
- request.isIgnoreIfNotExists(),
- lakeCatalogContainer.getLakeCatalog(),
- lakeTableTieringManager,
- new DefaultLakeCatalogContext(false, currentSession().getPrincipal()));
+ if (!alterSchemaChanges.isEmpty() && !alterTableConfigChanges.isEmpty()) {
+ // Only support one of alterTableConfigChanges and alterSchemaChanges for atomic change.
+ throw new InvalidAlterTableException(
+ "Table alteration can only be applied to one of the following: "
+ + "table properties or table schema.");
+ }
+
+ if (!alterSchemaChanges.isEmpty()) {
+ metadataManager.alterTableSchema(
+ tablePath, alterSchemaChanges, request.isIgnoreIfNotExists());
+ }
+
+ if (!alterTableConfigChanges.isEmpty()) {
+ LakeCatalogDynamicLoader.LakeCatalogContainer lakeCatalogContainer =
+ lakeCatalogDynamicLoader.getLakeCatalogContainer();
+ metadataManager.alterTableProperties(
+ tablePath,
+ alterTableConfigChanges,
+ tablePropertyChanges,
+ request.isIgnoreIfNotExists(),
+ lakeCatalogContainer.getLakeCatalog(),
+ lakeTableTieringManager,
+ new DefaultLakeCatalogContext(false, currentSession().getPrincipal()));
+ }
return CompletableFuture.completedFuture(new AlterTableResponse());
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
index ed544a8d1..19b13aaa9 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
@@ -38,6 +38,7 @@ import org.apache.fluss.lake.lakestorage.LakeCatalog;
import org.apache.fluss.metadata.DatabaseDescriptor;
import org.apache.fluss.metadata.DatabaseInfo;
import org.apache.fluss.metadata.ResolvedPartitionSpec;
+import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
@@ -320,6 +321,37 @@ public class MetadataManager {
"Fail to create table " + tablePath);
}
+ public void alterTableSchema(
+ TablePath tablePath, List<TableChange> schemaChanges, boolean ignoreIfNotExists)
+ throws TableNotExistException, TableNotPartitionedException {
+ try {
+
+ TableInfo table = getTable(tablePath);
+
+ // validate the table column changes
+ if (!schemaChanges.isEmpty()) {
+ UpdateSchema schemaUpdate = new SchemaUpdate(table);
+ for (TableChange schemaChange : schemaChanges) {
+ schemaUpdate = schemaUpdate.applySchemaChange(schemaChange);
+ }
+ Schema newSchema = schemaUpdate.getSchema();
+ // update the schema
+ zookeeperClient.registerSchema(tablePath, newSchema);
+ }
+ } catch (Exception e) {
+ if (e instanceof TableNotExistException) {
+ if (ignoreIfNotExists) {
+ return;
+ }
+ throw (TableNotExistException) e;
+ } else if (e instanceof RuntimeException) {
+ throw (RuntimeException) e;
+ } else {
+ throw new FlussRuntimeException("Failed to alter table schema: " + tablePath, e);
+ }
+ }
+ }
+
public void alterTableProperties(
TablePath tablePath,
List<TableChange> tableChanges,
@@ -386,7 +418,8 @@ public class MetadataManager {
} else if (e instanceof RuntimeException) {
throw (RuntimeException) e;
} else {
- throw new FlussRuntimeException("Failed to alter table: " + tablePath, e);
+ throw new FlussRuntimeException(
+ "Failed to alter table properties: " + tablePath, e);
}
}
}
@@ -452,7 +485,7 @@ public class MetadataManager {
if (toEnableDataLake) {
TableInfo newTableInfo = newTableRegistration.toTableInfo(tablePath, schemaInfo);
- // if the table is lake table, we need to add it to lake table tiering manager
+ // if the table is lake table, we need toadd it to lake table tiering manager
lakeTableTieringManager.addNewLakeTable(newTableInfo);
} else if (toDisableDataLake) {
lakeTableTieringManager.removeLakeTable(newTableRegistration.tableId);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
new file mode 100644
index 000000000..1cdc702d7
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
@@ -0,0 +1,119 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.coordinator;
+
+import org.apache.fluss.exception.SchemaChangeException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
+import org.apache.fluss.metadata.TableInfo;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicInteger;
+
+/** Schema update. */
+public class SchemaUpdate implements UpdateSchema {
+ private final List<Schema.Column> columns;
+ private final AtomicInteger highestFieldId;
+ private final List<String> primaryKeys;
+ private final List<String> bucketKeys;
+ private final List<String> partitionKeys;
+ private final Map<String, Schema.Column> existedColumns;
+
+ public SchemaUpdate(TableInfo tableInfo) {
+ this.columns = new ArrayList<>();
+ this.existedColumns = new HashMap<>();
+ this.highestFieldId = new AtomicInteger(tableInfo.getSchema().getHighestFieldId());
+ this.primaryKeys = tableInfo.getPrimaryKeys();
+ this.bucketKeys = tableInfo.getBucketKeys();
+ this.partitionKeys = tableInfo.getPartitionKeys();
+ this.columns.addAll(tableInfo.getSchema().getColumns());
+ for (Schema.Column column : columns) {
+ existedColumns.put(column.getName(), column);
+ }
+ }
+
+ @Override
+ public Schema getSchema() {
+ Schema.Builder builder =
+ Schema.newBuilder()
+ .fromColumns(columns)
+ .highestFieldId((short) highestFieldId.get());
+ if (!primaryKeys.isEmpty()) {
+ builder.primaryKey(primaryKeys);
+ }
+
+ return builder.build();
+ }
+
+ @Override
+ public UpdateSchema applySchemaChange(TableChange columnChange) {
+ if (columnChange instanceof TableChange.AddColumn) {
+ return addColumn((TableChange.AddColumn) columnChange);
+ } else if (columnChange instanceof TableChange.ModifyColumn) {
+ return modifiedColumn((TableChange.ModifyColumn) columnChange);
+ } else if (columnChange instanceof TableChange.RenameColumn) {
+ return renameColumn((TableChange.RenameColumn) columnChange);
+ } else if (columnChange instanceof TableChange.DropColumn) {
+ return dropColumn((TableChange.DropColumn) columnChange);
+ }
+ throw new IllegalArgumentException(
+ "Unknown column change type " + columnChange.getClass().getName());
+ }
+
+ private UpdateSchema addColumn(TableChange.AddColumn addColumn) {
+ if (existedColumns.containsKey(addColumn.getName())) {
+ throw new IllegalArgumentException(
+ "Column " + addColumn.getName() + " already exists.");
+ }
+
+ TableChange.ColumnPosition position = addColumn.getPosition();
+ if (position != TableChange.ColumnPosition.last()) {
+ throw new IllegalArgumentException("Only support addColumn column at last now.");
+ }
+
+ if (!addColumn.getDataType().isNullable()) {
+ throw new IllegalArgumentException(
+ "Column " + addColumn.getName() + " must be nullable.");
+ }
+
+ Schema.Column newColumn =
+ new Schema.Column(
+ addColumn.getName(),
+ addColumn.getDataType(),
+ addColumn.getComment(),
+ (byte) highestFieldId.incrementAndGet());
+ columns.add(newColumn);
+ existedColumns.put(newColumn.getName(), newColumn);
+ return this;
+ }
+
+ private UpdateSchema dropColumn(TableChange.DropColumn dropColumn) {
+ throw new SchemaChangeException("Not support drop column now.");
+ }
+
+ private UpdateSchema modifiedColumn(TableChange.ModifyColumn modifyColumn) {
+ throw new SchemaChangeException("Not support modify column now.");
+ }
+
+ private UpdateSchema renameColumn(TableChange.RenameColumn renameColumn) {
+ throw new SchemaChangeException("Not support rename column now.");
+ }
+}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
similarity index 56%
copy from fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
copy to fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
index b2cea1382..1d51704ac 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
@@ -15,26 +15,14 @@
* limitations under the License.
*/
-package org.apache.fluss.flink.sink.serializer;
+package org.apache.fluss.server.coordinator;
-import org.apache.fluss.types.RowType;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.TableChange;
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
+/** Schema update. */
+public interface UpdateSchema {
+ Schema getSchema();
-/**
- * Contextual information that can be used during initialization of {@link
- * FlussSerializationSchema}.
- */
-public class SerializerInitContextImpl implements FlussSerializationSchema.InitializationContext {
-
- private final RowType rowSchema;
-
- public SerializerInitContextImpl(RowType rowSchema) {
- this.rowSchema = checkNotNull(rowSchema, "rowSchema");
- }
-
- @Override
- public RowType getRowSchema() {
- return rowSchema;
- }
+ UpdateSchema applySchemaChange(TableChange columnChange);
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/SchemaChangeEvent.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/SchemaChangeEvent.java
new file mode 100644
index 000000000..7f7f7362b
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/SchemaChangeEvent.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.coordinator.event;
+
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TablePath;
+
+import java.util.Objects;
+
+/** An event for schema change. */
+public class SchemaChangeEvent implements CoordinatorEvent {
+ private final TablePath tablePath;
+ private final SchemaInfo schemaInfo;
+
+ public SchemaChangeEvent(TablePath tablePath, SchemaInfo schemaInfo) {
+ this.tablePath = tablePath;
+ this.schemaInfo = schemaInfo;
+ }
+
+ public TablePath getTablePath() {
+ return tablePath;
+ }
+
+ public SchemaInfo getSchemaInfo() {
+ return schemaInfo;
+ }
+
+ @Override
+ public String toString() {
+ return "SchemaChangeEvent{" + "tablePath=" + tablePath + ", schemaInfo=" + schemaInfo + '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof SchemaChangeEvent)) {
+ return false;
+ }
+ SchemaChangeEvent that = (SchemaChangeEvent) o;
+ return Objects.equals(tablePath, that.tablePath)
+ && Objects.equals(schemaInfo, that.schemaInfo);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tablePath, schemaInfo);
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
index c8ee481f6..acfb7e689 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
@@ -29,16 +29,19 @@ import org.apache.fluss.server.coordinator.event.CreateTableEvent;
import org.apache.fluss.server.coordinator.event.DropPartitionEvent;
import org.apache.fluss.server.coordinator.event.DropTableEvent;
import org.apache.fluss.server.coordinator.event.EventManager;
+import org.apache.fluss.server.coordinator.event.SchemaChangeEvent;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.data.PartitionAssignment;
import org.apache.fluss.server.zk.data.TableAssignment;
import org.apache.fluss.server.zk.data.TableRegistration;
import org.apache.fluss.server.zk.data.ZkData.DatabasesZNode;
import org.apache.fluss.server.zk.data.ZkData.PartitionZNode;
+import org.apache.fluss.server.zk.data.ZkData.SchemaZNode;
import org.apache.fluss.server.zk.data.ZkData.TableZNode;
import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.ChildData;
import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCache;
import org.apache.fluss.shaded.curator5.org.apache.curator.framework.recipes.cache.CuratorCacheListener;
+import org.apache.fluss.utils.types.Tuple2;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -102,6 +105,14 @@ public class TableChangeWatcher {
physicalTablePath.getTablePath(),
physicalTablePath.getPartitionName(),
newData);
+ break;
+ }
+
+ Tuple2<TablePath, Integer> tablePathIntegerTuple2 =
+ SchemaZNode.parsePath(newData.getPath());
+ if (tablePathIntegerTuple2 != null) {
+ processSchemaChange(
+ tablePathIntegerTuple2.f0, tablePathIntegerTuple2.f1);
}
}
break;
@@ -231,4 +242,31 @@ public class TableChangeWatcher {
tablePath, tableId, partitionId, partitionName, partitionAssignment));
}
}
+
+ private void processSchemaChange(TablePath tablePath, int schemaId) {
+
+ try {
+ int currentSchemaId = zooKeeperClient.getCurrentSchemaId(tablePath);
+ SchemaInfo schemaInfo;
+ if (schemaId != currentSchemaId) {
+ LOG.warn(
+ "Schema id {} is not equal to current schema id {}. Skipping schema change processing.",
+ schemaId,
+ currentSchemaId);
+ return;
+ }
+
+ Optional<SchemaInfo> optSchema = zooKeeperClient.getSchemaById(tablePath, schemaId);
+ if (!optSchema.isPresent()) {
+ LOG.error("No schema for table {} in zookeeper.", tablePath);
+ return;
+ } else {
+ schemaInfo = optSchema.get();
+ }
+
+ eventManager.put(new SchemaChangeEvent(tablePath, schemaInfo));
+ } catch (Exception e) {
+ LOG.error("Fail to get current schema id for table {}.", tablePath, e);
+ }
+ }
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
index 460562bc1..fad96a596 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
@@ -55,7 +55,7 @@ public class BatchRegisterLeadAndIsr {
tableBucket, leaderAndIsr, partitionName, liveReplicas));
} else {
throw new IllegalArgumentException(
- "Try to add a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "Try toadd a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "batch tableId="
+ tableId
+ " batch partitionId:"
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java
index 2d2764499..d1e24f6bc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvManager.java
@@ -28,7 +28,7 @@ import org.apache.fluss.memory.LazyMemorySegmentPool;
import org.apache.fluss.memory.MemorySegmentPool;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
-import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -161,7 +161,7 @@ public final class KvManager extends TabletManagerBase {
TableBucket tableBucket,
LogTablet logTablet,
KvFormat kvFormat,
- Schema schema,
+ SchemaGetter schemaGetter,
TableConfig tableConfig,
ArrowCompressionInfo arrowCompressionInfo)
throws Exception {
@@ -173,8 +173,7 @@ public final class KvManager extends TabletManagerBase {
}
File tabletDir = getOrCreateTabletDir(tablePath, tableBucket);
-
- RowMerger merger = RowMerger.create(tableConfig, schema, kvFormat);
+ RowMerger merger = RowMerger.create(tableConfig, kvFormat);
KvTablet tablet =
KvTablet.create(
logTablet,
@@ -184,9 +183,9 @@ public final class KvManager extends TabletManagerBase {
arrowBufferAllocator,
memorySegmentPool,
kvFormat,
- schema,
merger,
- arrowCompressionInfo);
+ arrowCompressionInfo,
+ schemaGetter);
currentKvs.put(tableBucket, tablet);
LOG.info(
@@ -256,7 +255,7 @@ public final class KvManager extends TabletManagerBase {
}
}
- public KvTablet loadKv(File tabletDir) throws Exception {
+ public KvTablet loadKv(File tabletDir, SchemaGetter schemaGetter) throws Exception {
Tuple2<PhysicalTablePath, TableBucket> pathAndBucket = FlussPaths.parseTabletDir(tabletDir);
PhysicalTablePath physicalTablePath = pathAndBucket.f0;
TableBucket tableBucket = pathAndBucket.f1;
@@ -277,11 +276,10 @@ public final class KvManager extends TabletManagerBase {
// TODO: we should support recover schema from disk to decouple put and schema.
TablePath tablePath = physicalTablePath.getTablePath();
TableInfo tableInfo = getTableInfo(zkClient, tablePath);
+
RowMerger rowMerger =
RowMerger.create(
- tableInfo.getTableConfig(),
- tableInfo.getSchema(),
- tableInfo.getTableConfig().getKvFormat());
+ tableInfo.getTableConfig(), tableInfo.getTableConfig().getKvFormat());
KvTablet kvTablet =
KvTablet.create(
physicalTablePath,
@@ -293,9 +291,9 @@ public final class KvManager extends TabletManagerBase {
arrowBufferAllocator,
memorySegmentPool,
tableInfo.getTableConfig().getKvFormat(),
- tableInfo.getSchema(),
rowMerger,
- tableInfo.getTableConfig().getArrowCompressionInfo());
+ tableInfo.getTableConfig().getArrowCompressionInfo(),
+ schemaGetter);
if (this.currentKvs.containsKey(tableBucket)) {
throw new IllegalStateException(
String.format(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
index 9057598a1..1f1c4192d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
@@ -17,9 +17,9 @@
package org.apache.fluss.server.kv;
-import org.apache.fluss.exception.KvStorageException;
import org.apache.fluss.metadata.DataLakeFormat;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -62,6 +62,7 @@ public class KvRecoverHelper {
private KeyEncoder keyEncoder;
private RowEncoder rowEncoder;
+ private final SchemaGetter schemaGetter;
private InternalRow.FieldGetter[] currentFieldGetters;
@@ -71,11 +72,22 @@ public class KvRecoverHelper {
long recoverPointOffset,
KvRecoverContext recoverContext,
KvFormat kvFormat) {
+ throw new UnsupportedOperationException();
+ }
+
+ public KvRecoverHelper(
+ KvTablet kvTablet,
+ LogTablet logTablet,
+ long recoverPointOffset,
+ KvRecoverContext recoverContext,
+ KvFormat kvFormat,
+ SchemaGetter schemaGetter) {
this.kvTablet = kvTablet;
this.logTablet = logTablet;
this.recoverPointOffset = recoverPointOffset;
this.recoverContext = recoverContext;
this.kvFormat = kvFormat;
+ this.schemaGetter = schemaGetter;
}
public void recover() throws Exception {
@@ -89,6 +101,8 @@ public class KvRecoverHelper {
// after the corresponding log offset is acked(when high watermark is advanced to the
// offset)
+ initSchema(schemaGetter.getLatestSchemaInfo().getSchemaId());
+
long nextLogOffset = recoverPointOffset;
// read to high watermark
try (KvBatchWriter kvBatchWriter = kvTablet.createKvBatchWriter()) {
@@ -138,20 +152,24 @@ public class KvRecoverHelper {
}
for (LogRecordBatch logRecordBatch : logRecords.batches()) {
- short schemaId = logRecordBatch.schemaId();
- if (currentSchemaId == null) {
- initSchema(schemaId);
- } else if (currentSchemaId != schemaId) {
- throw new KvStorageException(
- String.format(
- "Can't recover kv tablet for table bucket from log %s since the schema changes from schema id %d to schema id %d. "
- + "Currently, schema change is not supported.",
- recoverContext.tableBucket, currentSchemaId, schemaId));
- }
+ // short schemaId = logRecordBatch.schemaId();
+ // if (currentSchemaId == null) {
+ // initSchema(schemaId);
+ // } else if (currentSchemaId != schemaId) {
+ // throw new KvStorageException(
+ // String.format(
+ // "Can't recover kv tablet for table bucket from
+ // log %s since the schema changes from schema id %d to schema id %d. "
+ // + "Currently, schema change is not
+ // supported.",
+ // recoverContext.tableBucket, currentSchemaId,
+ // schemaId));
+ // }
+ // todo: currentRowType和currentSchemaId无法对齐
try (LogRecordReadContext readContext =
LogRecordReadContext.createArrowReadContext(
- currentRowType, currentSchemaId);
+ currentRowType, currentSchemaId, schemaGetter);
CloseableIterator<LogRecord> logRecordIter =
logRecordBatch.records(readContext)) {
while (logRecordIter.hasNext()) {
@@ -164,7 +182,8 @@ public class KvRecoverHelper {
// the log row format may not compatible with kv row format,
// e.g, arrow vs. compacted, thus needs a conversion here.
BinaryRow row = toKvRow(logRecord.getRow());
- value = ValueEncoder.encodeValue(schemaId, row);
+ // todo: short value是否会有问题,感觉可以先check一下
+ value = ValueEncoder.encodeValue(currentSchemaId.shortValue(), row);
}
resumeRecordConsumer.accept(
new KeyValueAndLogOffset(key, value, logRecord.logOffset()));
@@ -203,7 +222,7 @@ public class KvRecoverHelper {
// kv tablet's table id or not. If not equal, it means other table with same
// table path has been created, so the kv tablet's table is consider to be
// deleted. We can ignore the restore operation
- currentRowType = tableInfo.getRowType();
+ currentRowType = schemaGetter.getSchema(schemaId).getRowType();
DataType[] dataTypes = currentRowType.getChildren().toArray(new DataType[0]);
currentSchemaId = schemaId;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
index f91d5cc59..176e749f5 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
@@ -23,19 +23,23 @@ import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.DeletionDisabledException;
import org.apache.fluss.exception.KvStorageException;
+import org.apache.fluss.exception.SchemaNotExistException;
import org.apache.fluss.memory.MemorySegmentPool;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.KvRecord;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.record.KvRecordReadContext;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.arrow.ArrowWriterPool;
import org.apache.fluss.row.arrow.ArrowWriterProvider;
import org.apache.fluss.row.encode.ValueDecoder;
@@ -57,7 +61,6 @@ import org.apache.fluss.server.log.LogTablet;
import org.apache.fluss.server.metrics.group.TabletServerMetricGroup;
import org.apache.fluss.server.utils.FatalErrorHandler;
import org.apache.fluss.shaded.arrow.org.apache.arrow.memory.BufferAllocator;
-import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.BytesUtils;
import org.apache.fluss.utils.FileUtils;
@@ -80,6 +83,7 @@ import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import static org.apache.fluss.utils.SchemaUtil.getTargetColumns;
import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock;
import static org.apache.fluss.utils.concurrent.LockUtils.inWriteLock;
@@ -104,11 +108,12 @@ public final class KvTablet {
private final ReadWriteLock kvLock = new ReentrantReadWriteLock();
private final LogFormat logFormat;
private final KvFormat kvFormat;
- private final Schema schema;
// defines how to merge rows on the same primary key
private final RowMerger rowMerger;
private final ArrowCompressionInfo arrowCompressionInfo;
+ private final SchemaGetter schemaGetter;
+
/**
* The kv data in pre-write buffer whose log offset is less than the flushedLogOffset has been
* flushed into kv.
@@ -130,9 +135,9 @@ public final class KvTablet {
BufferAllocator arrowBufferAllocator,
MemorySegmentPool memorySegmentPool,
KvFormat kvFormat,
- Schema schema,
RowMerger rowMerger,
- ArrowCompressionInfo arrowCompressionInfo) {
+ ArrowCompressionInfo arrowCompressionInfo,
+ SchemaGetter schemaGetter) {
this.physicalPath = physicalPath;
this.tableBucket = tableBucket;
this.logTablet = logTablet;
@@ -144,9 +149,9 @@ public final class KvTablet {
this.arrowWriterProvider = new ArrowWriterPool(arrowBufferAllocator);
this.memorySegmentPool = memorySegmentPool;
this.kvFormat = kvFormat;
- this.schema = schema;
this.rowMerger = rowMerger;
this.arrowCompressionInfo = arrowCompressionInfo;
+ this.schemaGetter = schemaGetter;
}
public static KvTablet create(
@@ -157,9 +162,9 @@ public final class KvTablet {
BufferAllocator arrowBufferAllocator,
MemorySegmentPool memorySegmentPool,
KvFormat kvFormat,
- Schema schema,
RowMerger rowMerger,
- ArrowCompressionInfo arrowCompressionInfo)
+ ArrowCompressionInfo arrowCompressionInfo,
+ SchemaGetter schemaGetter)
throws IOException {
Tuple2<PhysicalTablePath, TableBucket> tablePathAndBucket =
FlussPaths.parseTabletDir(kvTabletDir);
@@ -173,9 +178,9 @@ public final class KvTablet {
arrowBufferAllocator,
memorySegmentPool,
kvFormat,
- schema,
rowMerger,
- arrowCompressionInfo);
+ arrowCompressionInfo,
+ schemaGetter);
}
public static KvTablet create(
@@ -188,9 +193,9 @@ public final class KvTablet {
BufferAllocator arrowBufferAllocator,
MemorySegmentPool memorySegmentPool,
KvFormat kvFormat,
- Schema schema,
RowMerger rowMerger,
- ArrowCompressionInfo arrowCompressionInfo)
+ ArrowCompressionInfo arrowCompressionInfo,
+ SchemaGetter schemaGetter)
throws IOException {
RocksDBKv kv = buildRocksDBKv(serverConf, kvTabletDir);
return new KvTablet(
@@ -205,9 +210,9 @@ public final class KvTablet {
arrowBufferAllocator,
memorySegmentPool,
kvFormat,
- schema,
rowMerger,
- arrowCompressionInfo);
+ arrowCompressionInfo,
+ schemaGetter);
}
private static RocksDBKv buildRocksDBKv(Configuration configuration, File kvDir)
@@ -259,26 +264,58 @@ public final class KvTablet {
kvLock,
() -> {
rocksDBKv.checkIfRocksDBClosed();
- short schemaId = kvRecords.schemaId();
- RowMerger currentMerger = rowMerger.configureTargetColumns(targetColumns);
- RowType rowType = schema.getRowType();
- WalBuilder walBuilder = createWalBuilder(schemaId, rowType);
+
+ SchemaInfo schemaInfo = schemaGetter.getLatestSchemaInfo();
+ Schema latestSchema = schemaInfo.getSchema();
+ // In the following scenario, we need to support schema migration:
+ // schema1: old data's schema -> valueDecoder
+ // schema2: new data's schema
+ // schema3: current schema (also the latest schema)
+ // schema1 <= schema2 <= current schema id
+ short latestSchemaId = (short) schemaInfo.getSchemaId();
+ short schemaIdOfNewData = kvRecords.schemaId();
+ if (schemaIdOfNewData > latestSchemaId || schemaIdOfNewData < 0) {
+ throw new SchemaNotExistException(
+ "Invalid schema id: "
+ + schemaIdOfNewData
+ + ", latest schema id: "
+ + latestSchemaId);
+ }
+ Schema schemaOfNewData = schemaGetter.getSchema(schemaIdOfNewData);
+ RowMerger currentMerger =
+ rowMerger.configureTargetColumns(
+ schemaIdOfNewData == latestSchemaId
+ ? targetColumns
+ : getTargetColumns(
+ targetColumns, schemaOfNewData, latestSchema),
+ latestSchemaId,
+ latestSchema);
+ RowType currentRowType = latestSchema.getRowType();
+ WalBuilder walBuilder = createWalBuilder(latestSchemaId, currentRowType);
walBuilder.setWriterState(kvRecords.writerId(), kvRecords.batchSequence());
// get offset to track the offset corresponded to the kv record
long logEndOffsetOfPrevBatch = logTablet.localLogEndOffset();
- DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
try {
long logOffset = logEndOffsetOfPrevBatch;
// TODO: reuse the read context and decoder
KvRecordBatch.ReadContext readContext =
- KvRecordReadContext.createReadContext(kvFormat, fieldTypes);
- ValueDecoder valueDecoder =
- new ValueDecoder(readContext.getRowDecoder(schemaId));
+ KvRecordReadContext.createReadContext(kvFormat, schemaGetter);
+
+ // replace new row with current schema if the schema id is older than now
+ ProjectedRow projectedRow =
+ ProjectedRow.from(schemaOfNewData, latestSchema);
+ ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat);
for (KvRecord kvRecord : kvRecords.records(readContext)) {
+
byte[] keyBytes = BytesUtils.toArray(kvRecord.getKey());
KvPreWriteBuffer.Key key = KvPreWriteBuffer.Key.of(keyBytes);
- if (kvRecord.getRow() == null) {
+ InternalRow row = kvRecord.getRow();
+ if (row != null && schemaIdOfNewData < latestSchemaId) {
+ row = projectedRow.replaceRow(row);
+ }
+
+ if (row == null) {
DeleteBehavior deleteBehavior = rowMerger.deleteBehavior();
if (deleteBehavior == DeleteBehavior.IGNORE) {
// skip delete rows if the merger doesn't support yet
@@ -290,14 +327,26 @@ public final class KvTablet {
}
// it's for deletion
byte[] oldValue = getFromBufferOrKv(key);
+
if (oldValue == null) {
// there might be large amount of such deletion, so we don't log
LOG.debug(
"The specific key can't be found in kv tablet although the kv record is for deletion, "
+ "ignore it directly as it doesn't exist in the kv tablet yet.");
} else {
- BinaryRow oldRow = valueDecoder.decodeValue(oldValue).row;
- BinaryRow newRow = currentMerger.delete(oldRow);
+ ValueDecoder.Value oldRowAndSchemaId =
+ valueDecoder.decodeValue(oldValue);
+ InternalRow oldRow = oldRowAndSchemaId.row;
+ if (oldRow != null
+ && oldRowAndSchemaId.schemaId != latestSchemaId) {
+ // todo: 后续想办法复用对应的projected row.
+ Schema schemaOfOldData =
+ schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
+ oldRow =
+ ProjectedRow.from(schemaOfOldData, latestSchema)
+ .replaceRow(oldRow);
+ }
+ InternalRow newRow = currentMerger.delete(oldRow);
// if newRow is null, it means the row should be deleted
if (newRow == null) {
walBuilder.append(ChangeType.DELETE, oldRow);
@@ -308,7 +357,11 @@ public final class KvTablet {
walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
kvPreWriteBuffer.put(
key,
- ValueEncoder.encodeValue(schemaId, newRow),
+ ValueEncoder.encodeRow(
+ latestSchemaId,
+ kvFormat,
+ currentRowType,
+ newRow),
logOffset + 1);
logOffset += 2;
}
@@ -318,32 +371,53 @@ public final class KvTablet {
byte[] oldValue = getFromBufferOrKv(key);
// it's update
if (oldValue != null) {
- BinaryRow oldRow = valueDecoder.decodeValue(oldValue).row;
- BinaryRow newRow =
- currentMerger.merge(oldRow, kvRecord.getRow());
+ ValueDecoder.Value oldRowAndSchemaId =
+ valueDecoder.decodeValue(oldValue);
+ InternalRow oldRow = oldRowAndSchemaId.row;
+ if (oldRow != null
+ && oldRowAndSchemaId.schemaId != latestSchemaId) {
+ // todo: 后续想办法复用对应的projected row.
+ Schema schemaOfOldData =
+ schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
+ oldRow =
+ ProjectedRow.from(schemaOfOldData, latestSchema)
+ .replaceRow(oldRow);
+ }
+
+ InternalRow newRow = currentMerger.merge(oldRow, row);
if (newRow == oldRow) {
// newRow is the same to oldRow, means nothing
// happens (no update/delete), and input should be ignored
continue;
}
+
walBuilder.append(ChangeType.UPDATE_BEFORE, oldRow);
walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
// logOffset is for -U, logOffset + 1 is for +U, we need to use
// the log offset for +U
kvPreWriteBuffer.put(
key,
- ValueEncoder.encodeValue(schemaId, newRow),
+ ValueEncoder.encodeRow(
+ latestSchemaId,
+ kvFormat,
+ currentRowType,
+ newRow),
logOffset + 1);
logOffset += 2;
} else {
// it's insert
- // TODO: we should add guarantees that all non-specified columns
+ // TODO: we shouldadd guarantees that all non-specified
+ // columns
// of the input row are set to null.
- BinaryRow newRow = kvRecord.getRow();
+ InternalRow newRow = row;
walBuilder.append(ChangeType.INSERT, newRow);
kvPreWriteBuffer.put(
key,
- ValueEncoder.encodeValue(schemaId, newRow),
+ ValueEncoder.encodeRow(
+ latestSchemaId,
+ kvFormat,
+ currentRowType,
+ newRow),
logOffset++);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
index 70063afc8..3ce22854c 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
@@ -42,13 +42,13 @@ public class PartialUpdaterCache {
// TODO: extend to tableId and schemaId when the cache is shared across all tables
public PartialUpdater getOrCreatePartialUpdater(
- KvFormat kvFormat, Schema schema, int[] targetColumns) {
+ KvFormat kvFormat, short schemaId, Schema schema, int[] targetColumns) {
return rowPartialUpdaters.get(
- getPartialUpdaterKey(targetColumns),
+ getPartialUpdaterKey(targetColumns, schemaId),
k -> new PartialUpdater(kvFormat, schema, targetColumns));
}
- private String getPartialUpdaterKey(int[] targetColumns) {
- return Arrays.toString(targetColumns);
+ private String getPartialUpdaterKey(int[] targetColumns, int schemaId) {
+ return schemaId + "_" + Arrays.toString(targetColumns);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
index 049d2859a..af3500c59 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
import org.apache.fluss.server.kv.partialupdate.PartialUpdater;
import org.apache.fluss.server.kv.partialupdate.PartialUpdaterCache;
@@ -34,12 +34,9 @@ public class DefaultRowMerger implements RowMerger {
private final PartialUpdaterCache partialUpdaterCache;
private final KvFormat kvFormat;
- private final Schema schema;
private final DeleteBehavior deleteBehavior;
- public DefaultRowMerger(
- Schema schema, KvFormat kvFormat, @Nullable DeleteBehavior deleteBehavior) {
- this.schema = schema;
+ public DefaultRowMerger(KvFormat kvFormat, @Nullable DeleteBehavior deleteBehavior) {
this.kvFormat = kvFormat;
// for compatibility, default to ALLOW if not specified
this.deleteBehavior = deleteBehavior != null ? deleteBehavior : DeleteBehavior.ALLOW;
@@ -49,14 +46,14 @@ public class DefaultRowMerger implements RowMerger {
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
// always retain the new row (latest row)
return newRow;
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
// returns null to indicate the row is deleted
return null;
}
@@ -67,13 +64,15 @@ public class DefaultRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(@Nullable int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ @Nullable int[] targetColumns, short schemaId, Schema schema) {
if (targetColumns == null) {
return this;
} else {
// this also sanity checks the validity of the partial update
PartialUpdater partialUpdater =
- partialUpdaterCache.getOrCreatePartialUpdater(kvFormat, schema, targetColumns);
+ partialUpdaterCache.getOrCreatePartialUpdater(
+ kvFormat, schemaId, schema, targetColumns);
return new PartialUpdateRowMerger(partialUpdater, deleteBehavior);
}
}
@@ -91,20 +90,21 @@ public class DefaultRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ int[] targetColumns, short schemaId, Schema schema) {
throw new IllegalStateException(
"PartialUpdateRowMerger does not support reconfigure target merge columns.");
}
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
return partialUpdater.updateRow(oldRow, newRow);
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
return partialUpdater.deleteRow(oldRow);
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
index 3271f0920..79592e177 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
@@ -19,7 +19,8 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.row.InternalRow;
import javax.annotation.Nullable;
@@ -43,14 +44,14 @@ public class FirstRowRowMerger implements RowMerger {
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
// always retain the old row (first row)
return oldRow;
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the first_row merge engine.");
}
@@ -61,7 +62,8 @@ public class FirstRowRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(@Nullable int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ @Nullable int[] targetColumns, short schemaId, Schema schema) {
if (targetColumns == null) {
return this;
} else {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
index b942cf181..5f3414574 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
@@ -23,7 +23,7 @@ import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.InternalRow;
import javax.annotation.Nullable;
@@ -40,7 +40,7 @@ public interface RowMerger {
* @return the merged row, if the returned row is the same to the old row, then nothing happens
* to the row (no update, no delete).
*/
- BinaryRow merge(BinaryRow oldRow, BinaryRow newRow);
+ InternalRow merge(InternalRow oldRow, InternalRow newRow);
/**
* Merge the old row with a delete row.
@@ -52,7 +52,7 @@ public interface RowMerger {
* @return the merged row, or null if the row is deleted.
*/
@Nullable
- BinaryRow delete(BinaryRow oldRow);
+ InternalRow delete(InternalRow oldRow);
/**
* The behavior of delete operations on primary key tables.
@@ -62,10 +62,10 @@ public interface RowMerger {
DeleteBehavior deleteBehavior();
/** Dynamically configure the target columns to merge and return the effective merger. */
- RowMerger configureTargetColumns(@Nullable int[] targetColumns);
+ RowMerger configureTargetColumns(@Nullable int[] targetColumns, short schemaId, Schema schema);
/** Create a row merger based on the given configuration. */
- static RowMerger create(TableConfig tableConf, Schema schema, KvFormat kvFormat) {
+ static RowMerger create(TableConfig tableConf, KvFormat kvFormat) {
Optional<MergeEngineType> mergeEngineType = tableConf.getMergeEngineType();
@Nullable DeleteBehavior deleteBehavior = tableConf.getDeleteBehavior().orElse(null);
@@ -81,14 +81,13 @@ public interface RowMerger {
"'%s' must be set for versioned merge engine.",
ConfigOptions.TABLE_MERGE_ENGINE_VERSION_COLUMN.key()));
}
- return new VersionedRowMerger(
- schema.getRowType(), versionColumn.get(), deleteBehavior);
+ return new VersionedRowMerger(versionColumn.get(), deleteBehavior);
default:
throw new IllegalArgumentException(
"Unsupported merge engine type: " + mergeEngineType.get());
}
} else {
- return new DefaultRowMerger(schema, kvFormat, deleteBehavior);
+ return new DefaultRowMerger(kvFormat, deleteBehavior);
}
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
index 11901b34b..9706230b5 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
@@ -19,7 +19,8 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
import org.apache.fluss.types.DataType;
@@ -45,31 +46,32 @@ public class VersionedRowMerger implements RowMerger {
private static final TimestampLtz MIN_TIMESTAMP_LTZ =
TimestampLtz.fromEpochMillis(Long.MIN_VALUE, 0);
- private final Comparator<BinaryRow> versionComparator;
-
+ private final String versionColumnName;
private final DeleteBehavior deleteBehavior;
- public VersionedRowMerger(
- RowType schema, String versionColumnName, @Nullable DeleteBehavior deleteBehavior) {
- this.versionComparator = createVersionComparator(schema, versionColumnName);
+ private int schemaId;
+ private Comparator<InternalRow> versionComparator;
+
+ public VersionedRowMerger(String versionColumnName, @Nullable DeleteBehavior deleteBehavior) {
if (deleteBehavior == DeleteBehavior.ALLOW) {
throw new IllegalArgumentException(
"DELETE is not supported for the versioned merge engine.");
}
// for compatibility, default to IGNORE if not specified
this.deleteBehavior = deleteBehavior != null ? deleteBehavior : DeleteBehavior.IGNORE;
+ this.versionColumnName = versionColumnName;
}
@Nullable
@Override
- public BinaryRow merge(BinaryRow oldRow, BinaryRow newRow) {
+ public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
// return newRow if newRow's version is larger or equal than oldRow's version
return versionComparator.compare(oldRow, newRow) <= 0 ? newRow : oldRow;
}
@Nullable
@Override
- public BinaryRow delete(BinaryRow oldRow) {
+ public InternalRow delete(InternalRow oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the versioned merge engine.");
}
@@ -80,8 +82,14 @@ public class VersionedRowMerger implements RowMerger {
}
@Override
- public RowMerger configureTargetColumns(@Nullable int[] targetColumns) {
+ public RowMerger configureTargetColumns(
+ @Nullable int[] targetColumns, short schemaId, Schema schema) {
if (targetColumns == null) {
+ if (schemaId != this.schemaId) {
+ this.schemaId = schemaId;
+ this.versionComparator =
+ createVersionComparator(schema.getRowType(), versionColumnName);
+ }
return this;
} else {
throw new UnsupportedOperationException(
@@ -90,7 +98,7 @@ public class VersionedRowMerger implements RowMerger {
}
/** Create a comparator for version column. */
- public static Comparator<BinaryRow> createVersionComparator(
+ public static Comparator<InternalRow> createVersionComparator(
RowType schema, String versionColumnName) {
int columnIndex = schema.getFieldIndex(versionColumnName);
checkArgument(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
index 3e93cc46a..b8d941221 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
@@ -99,7 +99,7 @@ public class CompletedSnapshotStore {
* Synchronously writes the new snapshots to snapshot handle store and asynchronously removes
* older ones.
*
- * @param snapshot Completed snapshot to add.
+ * @param snapshot Completed snapshot toadd.
*/
@VisibleForTesting
CompletedSnapshot addSnapshotAndSubsumeOldestOne(
@@ -116,7 +116,7 @@ public class CompletedSnapshotStore {
completedSnapshotHandleStore.add(
snapshot.getTableBucket(), snapshot.getSnapshotID(), completedSnapshotHandle);
- // Now add the new one. If it fails, we don't want to lose existing data.
+ // Nowadd the new one. If it fails, we don't want to lose existing data.
completedSnapshots.addLast(snapshot);
// Remove completed snapshot from queue and snapshotStateHandleStore, not discard.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
index fe517f268..e09c93b22 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
@@ -328,7 +328,7 @@ public class SnapshotLocation {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: may add entropy injection?
+ // todo: mayadd entropy injection?
this.kvFilePath = createFilePath();
this.outStream = fs.create(kvFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
index 2af972ae7..847bfa883 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
@@ -19,9 +19,9 @@ package org.apache.fluss.server.log;
import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.compression.ArrowCompressionInfo;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.rpc.messages.FetchLogRequest;
-import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -66,7 +66,7 @@ public final class FetchParams {
private final int minFetchBytes;
private final long maxWaitMs;
- // TODO: add more params like epoch etc.
+ // TODO:add more params like epoch etc.
public FetchParams(int replicaId, int maxFetchBytes) {
this(replicaId, true, maxFetchBytes, DEFAULT_MIN_FETCH_BYTES, DEFAULT_MAX_WAIT_MS);
@@ -97,7 +97,7 @@ public final class FetchParams {
long tableId,
long fetchOffset,
int maxFetchBytes,
- RowType schema,
+ SchemaGetter schemaGetter,
ArrowCompressionInfo compressionInfo,
@Nullable int[] projectedFields) {
this.fetchOffset = fetchOffset;
@@ -107,8 +107,9 @@ public final class FetchParams {
if (fileLogProjection == null) {
fileLogProjection = new FileLogProjection();
}
+
fileLogProjection.setCurrentProjection(
- tableId, schema, compressionInfo, projectedFields);
+ tableId, schemaGetter, compressionInfo, projectedFields);
} else {
projectionEnabled = false;
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
index 130b3855d..086e23b0e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
@@ -405,6 +405,8 @@ public final class LocalLog {
(maxOffsetMetadata.getSegmentBaseOffset() == segment.getBaseOffset())
? maxOffsetMetadata.getRelativePositionInSegment()
: segment.getSizeInBytes();
+
+ // segment需要查询
fetchDataInfo =
segment.read(readOffset, maxLength, maxPosition, minOneMessage, projection);
if (fetchDataInfo == null) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
index 835a94ff6..05233e0ff 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
@@ -203,7 +203,7 @@ public class FsRemoteLogOutputStream extends FSDataOutputStream {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: may add entropy injection?
+ // todo: mayadd entropy injection?
this.outStream = fs.create(remoteLogFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
} catch (Exception e) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
index 73aee521e..86a6bbf9e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
@@ -40,7 +40,7 @@ public class LogSegmentFiles {
private final Path offsetIndex;
private final Path timeIndex;
private final @Nullable Path writerIdIndex;
- // TODO add leader epoch index after introduce leader epoch.
+ // TODOadd leader epoch index after introduce leader epoch.
public LogSegmentFiles(
Path logSegment, Path offsetIndex, Path timeIndex, @Nullable Path writerIdIndex) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
index cdde3842d..0b425b8fc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
@@ -227,7 +227,7 @@ public class LogTieringTask implements Runnable {
}
/**
- * Copy the given log segments to remote and add the successfully copied segment to the {@code
+ * Copy the given log segments to remote andadd the successfully copied segment to the {@code
* copiedSegments} parameter.
*
* @return the end offset of the last segment copied to remote.
@@ -347,7 +347,7 @@ public class LogTieringTask implements Runnable {
// the commit failed, it means the commit snapshot is invalid or register zk
// failed, we will revert this commit and delete the remote log manifest
// file.
- // TODO: add the fail reason in the future.
+ // TODO:add the fail reason in the future.
LOG.error(
"Commit remote log manifest failed for table bucket {}. We will delete the"
+ " written remote log manifest file",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
new file mode 100644
index 000000000..f56429bf1
--- /dev/null
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.server.metadata;
+
+import org.apache.fluss.annotation.VisibleForTesting;
+import org.apache.fluss.exception.SchemaNotExistException;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.server.coordinator.MetadataManager;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.Cache;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.CacheBuilder;
+import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.UncheckedExecutionException;
+import org.apache.fluss.utils.MapUtils;
+
+import java.util.Map;
+import java.util.Objects;
+import java.util.concurrent.ExecutionException;
+
+/**
+ * Caches and manages schema metadata for tables, providing methods to update and query schema
+ * information. Unlike other metadata cached all in {@link ServerMetadataCache}, only required
+ * schemas are cached here due to {@link Schema} being a large object. Including two kind of schema:
+ * 1. latest schema of each subscribed table, updated by UpdateMetadata request. 2. history schemas
+ * of each table, updated by lookup from tablet server.
+ */
+public class SchemaMetadataManager {
+
+ private MetadataManager metadataManager;
+ private final Map<TablePath, SchemaInfo> latestSchemaByTablePath;
+ private final Map<TablePath, Integer> subscriberCounters;
+ private final Cache<TableSchemaKey, Schema> schemaCache;
+
+ public SchemaMetadataManager(MetadataManager metadataManager) {
+ this.metadataManager = metadataManager;
+ // thread safe is guaranteed by subscriberCounters.
+ this.subscriberCounters = MapUtils.newConcurrentHashMap();
+ this.latestSchemaByTablePath = MapUtils.newConcurrentHashMap();
+ this.schemaCache = CacheBuilder.newBuilder().maximumSize(100).build();
+ }
+
+ public SchemaGetter subscribeWithInitialSchema(
+ TablePath tablePath, int initialSchemaId, Schema initialSchema) {
+ subscriberCounters.compute(
+ tablePath,
+ (key, oldValue) -> {
+ updateSchema(tablePath, initialSchemaId, initialSchema);
+ if (oldValue == null) {
+ return 1;
+ } else {
+ return oldValue + 1;
+ }
+ });
+ return new SchemaMetadataSubscriberImpl(tablePath);
+ }
+
+ public void updateLatestSchema(TablePath tablePath, short schemaId, Schema schema) {
+ // only update if tablePath is subscribed.
+ subscriberCounters.computeIfPresent(
+ tablePath,
+ (key, oldValue) -> {
+ updateSchema(tablePath, schemaId, schema);
+ return oldValue;
+ });
+ }
+
+ private void unsubscribe(TablePath tablePath) {
+ subscriberCounters.compute(
+ tablePath,
+ (key, oldValue) -> {
+ if (oldValue != null && oldValue > 1) {
+ return oldValue - 1;
+ }
+ latestSchemaByTablePath.remove(tablePath);
+ return null;
+ });
+ }
+
+ private Schema getFlussSchema(TablePath tablePath, int schemaId) throws ExecutionException {
+ return schemaCache.get(
+ new TableSchemaKey(tablePath, schemaId),
+ () -> {
+ SchemaInfo schemaInfo = latestSchemaByTablePath.get(tablePath);
+ if (schemaInfo != null && schemaInfo.getSchemaId() == schemaId) {
+ return latestSchemaByTablePath.get(tablePath).getSchema();
+ } else {
+ return metadataManager.getSchemaById(tablePath, schemaId).getSchema();
+ }
+ });
+ }
+
+ private void updateSchema(TablePath tablePath, int schemaId, Schema schema) {
+ latestSchemaByTablePath.compute(
+ tablePath,
+ (key, oldValue) -> {
+ if (oldValue == null || oldValue.getSchemaId() < schemaId) {
+ return new SchemaInfo(schema, schemaId);
+ } else {
+ return oldValue;
+ }
+ });
+
+ schemaCache.put(new TableSchemaKey(tablePath, schemaId), schema);
+ }
+
+ @VisibleForTesting
+ public Map<TablePath, SchemaInfo> getLatestSchemaByTablePath() {
+ return latestSchemaByTablePath;
+ }
+
+ @VisibleForTesting
+ Map<TablePath, Integer> getSubscriberCounters() {
+ return subscriberCounters;
+ }
+
+ private static class TableSchemaKey {
+ private final TablePath tablePath;
+ private final int schemaId;
+
+ TableSchemaKey(TablePath tablePath, int schemaId) {
+ this.tablePath = tablePath;
+ this.schemaId = schemaId;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof TableSchemaKey)) {
+ return false;
+ }
+ TableSchemaKey that = (TableSchemaKey) o;
+ return schemaId == that.schemaId && Objects.equals(tablePath, that.tablePath);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tablePath, schemaId);
+ }
+ }
+
+ private class SchemaMetadataSubscriberImpl implements SchemaGetter {
+ private final TablePath tablePath;
+ private volatile boolean released;
+
+ public SchemaMetadataSubscriberImpl(TablePath tablePath) {
+ this.tablePath = tablePath;
+ this.released = false;
+ }
+
+ @Override
+ public SchemaInfo getLatestSchemaInfo() {
+ return latestSchemaByTablePath.get(tablePath);
+ }
+
+ @Override
+ public Schema getSchema(int schemaId) {
+ // todo: support with get table.
+ try {
+ return getFlussSchema(tablePath, schemaId);
+ } catch (ExecutionException | UncheckedExecutionException executionException) {
+ Throwable cause = executionException.getCause();
+ if (cause instanceof SchemaNotExistException) {
+ throw (SchemaNotExistException) cause;
+ } else {
+ throw new RuntimeException(cause);
+ }
+ }
+ }
+
+ @Override
+ public void release() {
+ if (!released) {
+ unsubscribe(tablePath);
+ released = true;
+ }
+ }
+ }
+}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
index 79a22621b..6a11a53ae 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
@@ -21,6 +21,9 @@ import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.cluster.ServerNode;
import org.apache.fluss.cluster.TabletServerInfo;
import org.apache.fluss.metadata.PhysicalTablePath;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -65,9 +68,13 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
private final MetadataManager metadataManager;
+ private final SchemaMetadataManager schemaMetadataManager;
+
+ // todo: replace this in test with schemaMetadataManager.
public TabletServerMetadataCache(MetadataManager metadataManager) {
this.serverMetadataSnapshot = ServerMetadataSnapshot.empty();
this.metadataManager = metadataManager;
+ this.schemaMetadataManager = new SchemaMetadataManager(metadataManager);
}
@Override
@@ -133,6 +140,17 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
}
}
+ public SchemaGetter subscribeWithInitialSchema(
+ TablePath tablePath, int initialSchemaId, Schema initialSchema) {
+ return schemaMetadataManager.subscribeWithInitialSchema(
+ tablePath, initialSchemaId, initialSchema);
+ }
+
+ public void updateLatestSchema(TablePath tablePath, SchemaInfo schemaInfo) {
+ schemaMetadataManager.updateLatestSchema(
+ tablePath, (short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+
public Optional<PartitionMetadata> getPartitionMetadata(PhysicalTablePath partitionPath) {
TablePath tablePath = partitionPath.getTablePath();
String partitionName = partitionPath.getPartitionName();
@@ -185,6 +203,13 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
TableInfo tableInfo = tableMetadata.getTableInfo();
TablePath tablePath = tableInfo.getTablePath();
long tableId = tableInfo.getTableId();
+ // Update schema metadata.
+ // todo: apply schema id and schema info if needs
+ int schemaId = tableInfo.getSchemaId();
+ Schema schema = tableInfo.getSchema();
+ schemaMetadataManager.updateLatestSchema(
+ tablePath, (short) schemaId, schema);
+
if (tableId == DELETED_TABLE_ID) {
Long removedTableId = tableIdByPath.remove(tablePath);
if (removedTableId != null) {
@@ -410,4 +435,9 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
bucketMetadataMapForPartitions);
});
}
+
+ @VisibleForTesting
+ public SchemaMetadataManager getSchemaMetadataManager() {
+ return schemaMetadataManager;
+ }
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
index d3b5d63c9..251d670b6 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
@@ -77,7 +77,7 @@ public class AdjustIsrManager {
public CompletableFuture<LeaderAndIsr> submit(
TableBucket tableBucket, LeaderAndIsr leaderAndIsr) {
- // TODO add coordinatorEpoch.
+ // TODOadd coordinatorEpoch.
CompletableFuture<LeaderAndIsr> future = new CompletableFuture<>();
AdjustIsrItem adjustIsrItem = new AdjustIsrItem(tableBucket, leaderAndIsr, future);
boolean enqueued = unsentAdjustIsrMap.putIfAbsent(tableBucket, adjustIsrItem) == null;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index 3b7ac5b7f..ad2dcb24d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -33,7 +33,7 @@ import org.apache.fluss.exception.NotLeaderOrFollowerException;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
-import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -74,6 +74,7 @@ import org.apache.fluss.server.log.LogTablet;
import org.apache.fluss.server.log.checkpoint.OffsetCheckpointFile;
import org.apache.fluss.server.log.remote.RemoteLogManager;
import org.apache.fluss.server.metadata.ServerMetadataCache;
+import org.apache.fluss.server.metadata.TabletServerMetadataCache;
import org.apache.fluss.server.metrics.group.BucketMetricGroup;
import org.apache.fluss.server.metrics.group.TableMetricGroup;
import org.apache.fluss.server.metrics.group.TabletServerMetricGroup;
@@ -86,7 +87,6 @@ import org.apache.fluss.server.zk.ZkSequenceIDCounter;
import org.apache.fluss.server.zk.ZooKeeperClient;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
import org.apache.fluss.server.zk.data.ZkData;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.FlussPaths;
import org.apache.fluss.utils.IOUtils;
@@ -105,7 +105,6 @@ import java.io.File;
import java.io.IOException;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
@@ -167,7 +166,7 @@ public final class Replica {
/** The manger to manger the isr expand and shrink. */
private final AdjustIsrManager adjustIsrManager;
- private final Schema schema;
+ private final SchemaGetter schemaGetter;
private final TableConfig tableConfig;
// logFormat and arrowCompressionInfo are used in hot-path, so cache them here.
private final LogFormat logFormat;
@@ -216,7 +215,7 @@ public final class Replica {
DelayedOperationManager<DelayedFetchLog> delayedFetchLogManager,
AdjustIsrManager adjustIsrManager,
SnapshotContext snapshotContext,
- ServerMetadataCache metadataCache,
+ TabletServerMetadataCache metadataCache,
FatalErrorHandler fatalErrorHandler,
BucketMetricGroup bucketMetricGroup,
TableInfo tableInfo,
@@ -235,7 +234,11 @@ public final class Replica {
this.adjustIsrManager = adjustIsrManager;
this.fatalErrorHandler = fatalErrorHandler;
this.bucketMetricGroup = bucketMetricGroup;
- this.schema = tableInfo.getSchema();
+ this.schemaGetter =
+ metadataCache.subscribeWithInitialSchema(
+ physicalPath.getTablePath(),
+ tableInfo.getSchemaId(),
+ tableInfo.getSchema());
this.tableConfig = tableInfo.getTableConfig();
this.logFormat = tableConfig.getLogFormat();
this.arrowCompressionInfo = tableConfig.getArrowCompressionInfo();
@@ -286,10 +289,6 @@ public final class Replica {
return kvManager != null;
}
- public RowType getRowType() {
- return schema.getRowType();
- }
-
public ArrowCompressionInfo getArrowCompressionInfo() {
return arrowCompressionInfo;
}
@@ -374,6 +373,10 @@ public final class Replica {
return bucketMetricGroup.getTableMetricGroup();
}
+ public LogFormat getLogFormat() {
+ return logFormat;
+ }
+
public void makeLeader(NotifyLeaderAndIsrData data) throws IOException {
boolean leaderHWIncremented =
inWriteLock(
@@ -491,39 +494,11 @@ public final class Replica {
// drop log then
logManager.dropLog(tableBucket);
// close the closeable registry
+ IOUtils.closeQuietly(schemaGetter::release);
IOUtils.closeQuietly(closeableRegistry);
});
}
- public void checkProjection(@Nullable int[] projectedFields) {
- if (projectedFields != null) {
- if (logFormat != LogFormat.ARROW) {
- throw new InvalidColumnProjectionException(
- String.format(
- "Column projection is only supported for ARROW format, but the table %s is %s format.",
- physicalPath.getTablePath(), logFormat));
- }
- int fieldCount = schema.getColumns().size();
- int prev = -1;
- for (int i : projectedFields) {
- if (i <= prev) {
- throw new InvalidColumnProjectionException(
- "The projection indexes should be in field order, but is "
- + Arrays.toString(projectedFields));
- }
- if (i >= fieldCount) {
- throw new InvalidColumnProjectionException(
- "Projected fields "
- + Arrays.toString(projectedFields)
- + " is out of bound for schema with "
- + fieldCount
- + " fields.");
- }
- prev = i;
- }
- }
- }
-
public LogOffsetSnapshot fetchOffsetSnapshot(boolean fetchOnlyFromLeader) throws IOException {
return inReadLock(
leaderIsrUpdateLock,
@@ -675,7 +650,7 @@ public final class Replica {
downloadKvSnapshots(completedSnapshot, tabletDir.toPath());
// as we have downloaded kv files into the tablet dir, now, we can load it
- kvTablet = kvManager.loadKv(tabletDir);
+ kvTablet = kvManager.loadKv(tabletDir, schemaGetter);
checkNotNull(kvTablet, "kv tablet should not be null.");
restoreStartOffset = completedSnapshot.getLogOffset();
@@ -684,6 +659,7 @@ public final class Replica {
"No snapshot found for {} of {}, restore from log.",
tableBucket,
physicalPath);
+
// actually, kv manager always create a kv tablet since we will drop the kv
// if it exists before init kv tablet
kvTablet =
@@ -692,7 +668,7 @@ public final class Replica {
tableBucket,
logTablet,
tableConfig.getKvFormat(),
- schema,
+ schemaGetter,
tableConfig,
arrowCompressionInfo);
}
@@ -774,7 +750,8 @@ public final class Replica {
logTablet,
startRecoverLogOffset,
recoverContext,
- tableConfig.getKvFormat());
+ tableConfig.getKvFormat(),
+ schemaGetter);
kvRecoverHelper.recover();
} catch (Exception e) {
throw new KvStorageException(
@@ -1540,7 +1517,7 @@ public final class Replica {
List<Integer> isrToSend = new ArrayList<>(isrState.isr());
isrToSend.add(newInSyncReplicaId);
- // TODO add server epoch to isr.
+ // TODOadd server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
@@ -1562,7 +1539,7 @@ public final class Replica {
// erroneously advance the HW if the `AdjustIsr` were to fail. Hence, the "maximal ISR"
// for `PendingShrinkIsr` is the current ISR.
- // TODO add server epoch to isr.
+ // TODOadd server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
@@ -1937,4 +1914,9 @@ public final class Replica {
public List<Integer> getIsr() {
return isrState.isr();
}
+
+ @VisibleForTesting
+ public SchemaGetter getSchemaGetter() {
+ return schemaGetter;
+ }
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index 8a5f85615..9f6e4d8ad 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -21,6 +21,7 @@ import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.FencedLeaderEpochException;
+import org.apache.fluss.exception.InvalidColumnProjectionException;
import org.apache.fluss.exception.InvalidCoordinatorException;
import org.apache.fluss.exception.InvalidRequiredAcksException;
import org.apache.fluss.exception.LogOffsetOutOfRangeException;
@@ -29,6 +30,7 @@ import org.apache.fluss.exception.NotLeaderOrFollowerException;
import org.apache.fluss.exception.StorageException;
import org.apache.fluss.exception.UnknownTableOrBucketException;
import org.apache.fluss.fs.FsPath;
+import org.apache.fluss.metadata.LogFormat;
import org.apache.fluss.metadata.PhysicalTablePath;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
@@ -839,7 +841,7 @@ public class ReplicaManager {
* 2. Stop fetchers for these replicas so that no more data can be added by the replica fetcher threads.
* 3. Truncate the log and checkpoint offsets for these replicas.
* 4. Clear the delayed produce in the purgatory.
- * 5. If the server is not shutting down, add the fetcher to the new leaders.
+ * 5. If the server is not shutting down,add the fetcher to the new leaders.
* </pre>
*/
private void makeFollowers(
@@ -1066,12 +1068,20 @@ public class ReplicaManager {
"Fetching log record for replica {}, offset {}",
tb,
fetchReqInfo.getFetchOffset());
- replica.checkProjection(fetchReqInfo.getProjectFields());
+ // todo: change here to modified project fields.
+ if (fetchReqInfo.getProjectFields() != null
+ && replica.getLogFormat() != LogFormat.ARROW) {
+ throw new InvalidColumnProjectionException(
+ String.format(
+ "Column projection is only supported for ARROW format, but the table %s is %s format.",
+ replica.getTablePath(), replica.getLogFormat()));
+ }
+
fetchParams.setCurrentFetch(
tb.getTableId(),
fetchOffset,
adjustedMaxBytes,
- replica.getRowType(),
+ replica.getSchemaGetter(),
replica.getArrowCompressionInfo(),
fetchReqInfo.getProjectFields());
LogReadInfo readInfo = replica.fetchRecords(fetchParams);
@@ -1522,6 +1532,7 @@ public class ReplicaManager {
PhysicalTablePath physicalTablePath = data.getPhysicalTablePath();
TablePath tablePath = physicalTablePath.getTablePath();
TableInfo tableInfo = getTableInfo(zkClient, tablePath);
+
boolean isKvTable = tableInfo.hasPrimaryKey();
BucketMetricGroup bucketMetricGroup =
serverMetricGroup.addTableBucketMetricGroup(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
index 9e4479b3e..06e94e9ae 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
@@ -225,7 +225,7 @@ public class ReplicaFetcherManager {
fetcherThread.getLeader().leaderServerId(),
initialFetchStatusMap);
} catch (InterruptedException e) {
- LOG.error("Interrupted while add buckets to fetcher threads.", e);
+ LOG.error("Interrupted whileadd buckets to fetcher threads.", e);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
index 398bfb6ff..76195a0df 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
@@ -80,7 +80,7 @@ final class ReplicaFetcherThread extends ShutdownableThread {
private final LeaderEndpoint leader;
private final int fetchBackOffMs;
- // manually add timout logic in here, todo remove this timeout logic if
+ // manuallyadd timout logic in here, todo remove this timeout logic if
// we support global request timeout in #279
private final int timeoutSeconds = 30;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
index c89fca6f7..062eec73d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
@@ -65,7 +65,7 @@ public class RpcGatewayManager<T extends RpcGateway> implements AutoCloseable {
}
/**
- * Add a server to the manager. It'll create a new gateway for the server and add it to the
+ * Add a server to the manager. It'll create a new gateway for the server andadd it to the
* manager. If the server has already existed, it'll remove the already existing server before
* adding the new one.
*/
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
index 78c1f587b..165fd23e9 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
@@ -22,6 +22,7 @@ import org.apache.fluss.cluster.ServerNode;
import org.apache.fluss.cluster.ServerType;
import org.apache.fluss.config.ConfigOptions;
import org.apache.fluss.config.cluster.AlterConfigOpType;
+import org.apache.fluss.config.cluster.ColumnPositionType;
import org.apache.fluss.config.cluster.ConfigEntry;
import org.apache.fluss.fs.FsPath;
import org.apache.fluss.fs.token.ObtainedSecurityToken;
@@ -52,6 +53,7 @@ import org.apache.fluss.rpc.entity.ProduceLogResultForBucket;
import org.apache.fluss.rpc.entity.PutKvResultForBucket;
import org.apache.fluss.rpc.messages.AdjustIsrRequest;
import org.apache.fluss.rpc.messages.AdjustIsrResponse;
+import org.apache.fluss.rpc.messages.AlterTableRequest;
import org.apache.fluss.rpc.messages.CommitKvSnapshotRequest;
import org.apache.fluss.rpc.messages.CommitLakeTableSnapshotRequest;
import org.apache.fluss.rpc.messages.CommitRemoteLogManifestRequest;
@@ -79,6 +81,7 @@ import org.apache.fluss.rpc.messages.NotifyLeaderAndIsrRequest;
import org.apache.fluss.rpc.messages.NotifyLeaderAndIsrResponse;
import org.apache.fluss.rpc.messages.NotifyRemoteLogOffsetsRequest;
import org.apache.fluss.rpc.messages.PbAclInfo;
+import org.apache.fluss.rpc.messages.PbAddColumn;
import org.apache.fluss.rpc.messages.PbAdjustIsrReqForBucket;
import org.apache.fluss.rpc.messages.PbAdjustIsrReqForTable;
import org.apache.fluss.rpc.messages.PbAdjustIsrRespForBucket;
@@ -89,6 +92,7 @@ import org.apache.fluss.rpc.messages.PbCreateAclRespInfo;
import org.apache.fluss.rpc.messages.PbDescribeConfig;
import org.apache.fluss.rpc.messages.PbDropAclsFilterResult;
import org.apache.fluss.rpc.messages.PbDropAclsMatchingAcl;
+import org.apache.fluss.rpc.messages.PbDropColumn;
import org.apache.fluss.rpc.messages.PbFetchLogReqForBucket;
import org.apache.fluss.rpc.messages.PbFetchLogReqForTable;
import org.apache.fluss.rpc.messages.PbFetchLogRespForBucket;
@@ -101,6 +105,7 @@ import org.apache.fluss.rpc.messages.PbLakeTableSnapshotInfo;
import org.apache.fluss.rpc.messages.PbListOffsetsRespForBucket;
import org.apache.fluss.rpc.messages.PbLookupReqForBucket;
import org.apache.fluss.rpc.messages.PbLookupRespForBucket;
+import org.apache.fluss.rpc.messages.PbModifyColumn;
import org.apache.fluss.rpc.messages.PbNotifyLakeTableOffsetReqForBucket;
import org.apache.fluss.rpc.messages.PbNotifyLeaderAndIsrReqForBucket;
import org.apache.fluss.rpc.messages.PbNotifyLeaderAndIsrRespForBucket;
@@ -115,6 +120,7 @@ import org.apache.fluss.rpc.messages.PbPutKvReqForBucket;
import org.apache.fluss.rpc.messages.PbPutKvRespForBucket;
import org.apache.fluss.rpc.messages.PbRemoteLogSegment;
import org.apache.fluss.rpc.messages.PbRemotePathAndLocalFile;
+import org.apache.fluss.rpc.messages.PbRenameColumn;
import org.apache.fluss.rpc.messages.PbServerNode;
import org.apache.fluss.rpc.messages.PbStopReplicaReqForBucket;
import org.apache.fluss.rpc.messages.PbStopReplicaRespForBucket;
@@ -160,6 +166,8 @@ import org.apache.fluss.server.metadata.TableMetadata;
import org.apache.fluss.server.zk.data.BucketSnapshot;
import org.apache.fluss.server.zk.data.LakeTableSnapshot;
import org.apache.fluss.server.zk.data.LeaderAndIsr;
+import org.apache.fluss.utils.json.DataTypeJsonSerde;
+import org.apache.fluss.utils.json.JsonSerdeUtils;
import javax.annotation.Nullable;
@@ -260,13 +268,87 @@ public class ServerRpcMessageUtils {
}
}
- public static List<TableChange> toTableChanges(List<PbAlterConfig> alterConfigs) {
+ public static List<TableChange> toAlterTableConfigChanges(List<PbAlterConfig> alterConfigs) {
return alterConfigs.stream()
.filter(Objects::nonNull)
.map(ServerRpcMessageUtils::toTableChange)
.collect(Collectors.toList());
}
+ public static List<TableChange> toAlterTableSchemaChanges(AlterTableRequest request) {
+ List<TableChange> alterTableSchemaChanges = new ArrayList<>();
+ alterTableSchemaChanges.addAll(toAddColumns(request.getAddColumnsList()));
+ alterTableSchemaChanges.addAll(toDropColumns(request.getDropColumnsList()));
+ alterTableSchemaChanges.addAll(toRenameColumns(request.getRenameColumnsList()));
+ alterTableSchemaChanges.addAll(toModifyColumns(request.getModifyColumnsList()));
+ return alterTableSchemaChanges;
+ }
+
+ public static List<TableChange> toAddColumns(List<PbAddColumn> addColumns) {
+ return addColumns.stream()
+ .filter(Objects::nonNull)
+ .map(
+ pbAddColumn ->
+ TableChange.addColumn(
+ pbAddColumn.getColumnName(),
+ JsonSerdeUtils.readValue(
+ pbAddColumn.getDataTypeJson(),
+ DataTypeJsonSerde.INSTANCE),
+ pbAddColumn.hasComment() ? pbAddColumn.getComment() : null,
+ toColumnPosition(pbAddColumn.getColumnPositionType())))
+ .collect(Collectors.toList());
+ }
+
+ public static List<TableChange.SchemaChange> toDropColumns(List<PbDropColumn> dropColumns) {
+ return dropColumns.stream()
+ .filter(Objects::nonNull)
+ .map(pbDropColumn -> TableChange.dropColumn(pbDropColumn.getColumnName()))
+ .collect(Collectors.toList());
+ }
+
+ public static List<TableChange.SchemaChange> toRenameColumns(
+ List<PbRenameColumn> alterColumns) {
+ return alterColumns.stream()
+ .filter(Objects::nonNull)
+ .map(
+ pbRenameColumn ->
+ TableChange.renameColumn(
+ pbRenameColumn.getOldColumnName(),
+ pbRenameColumn.getNewColumnName()))
+ .collect(Collectors.toList());
+ }
+
+ public static List<TableChange.SchemaChange> toModifyColumns(
+ List<PbModifyColumn> modifyColumns) {
+ return modifyColumns.stream()
+ .filter(Objects::nonNull)
+ .map(
+ pbModifyColumn ->
+ TableChange.modifyColumn(
+ pbModifyColumn.getColumnName(),
+ JsonSerdeUtils.readValue(
+ pbModifyColumn.getDataTypeJson(),
+ DataTypeJsonSerde.INSTANCE),
+ pbModifyColumn.hasComment()
+ ? pbModifyColumn.getComment()
+ : null,
+ pbModifyColumn.hasColumnPositionType()
+ ? toColumnPosition(
+ pbModifyColumn.getColumnPositionType())
+ : null))
+ .collect(Collectors.toList());
+ }
+
+ private static @Nullable TableChange.ColumnPosition toColumnPosition(int columnPositionType) {
+ ColumnPositionType opType = ColumnPositionType.from(columnPositionType);
+ switch (opType) {
+ case LAST:
+ return TableChange.ColumnPosition.last();
+ default:
+ throw new IllegalArgumentException("Unsupported column position type " + opType);
+ }
+ }
+
public static MetadataResponse buildMetadataResponse(
@Nullable ServerNode coordinatorServer,
Set<ServerNode> aliveTabletServers,
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
index ea6aead7b..270d334ee 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
@@ -35,7 +35,7 @@ public interface Timer {
* Add a new task to this executor. It will be executed after the task's delay (beginning from
* the time of submission).
*
- * @param timerTask the task to add
+ * @param timerTask the task toadd
*/
void add(TimerTask timerTask);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
index 1dab9f229..934777c93 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
@@ -108,8 +108,8 @@ import java.util.concurrent.atomic.AtomicInteger;
* while priority queue based timers takes O(log N) for both insert and delete where N is the number
* of items in the queue.
*
- * <p>This class is not thread-safe. There should not be any add calls while advanceClock is
- * executing. It is caller's responsibility to enforce it. Simultaneous add calls are thread-safe.
+ * <p>This class is not thread-safe. There should not be anyadd calls while advanceClock is
+ * executing. It is caller's responsibility to enforce it. Simultaneousadd calls are thread-safe.
*/
@NotThreadSafe
final class TimingWheel {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
index 1c7401b8a..e27ffd9c3 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
@@ -810,16 +810,18 @@ public class ZooKeeperClient implements AutoCloseable {
/** Register schema to ZK metadata and return the schema id. */
public int registerSchema(TablePath tablePath, Schema schema) throws Exception {
- int currentSchemaId = getCurrentSchemaId(tablePath);
+ return registerSchema(tablePath, schema, getCurrentSchemaId(tablePath) + 1);
+ }
+
+ public int registerSchema(TablePath tablePath, Schema schema, int schemaId) throws Exception {
// increase schema id.
- currentSchemaId++;
- String path = SchemaZNode.path(tablePath, currentSchemaId);
+ String path = SchemaZNode.path(tablePath, schemaId);
zkClient.create()
.creatingParentsIfNeeded()
.withMode(CreateMode.PERSISTENT)
.forPath(path, SchemaZNode.encode(schema));
- LOG.info("Registered new schema version {} for table {}.", currentSchemaId, tablePath);
- return currentSchemaId;
+ LOG.info("Registered new schema version {} for table {}.", schemaId, tablePath);
+ return schemaId;
}
/** Get the specific schema by schema id in ZK metadata. */
@@ -1130,7 +1132,7 @@ public class ZooKeeperClient implements AutoCloseable {
.forPath(
AclChangeNotificationNode.pathPrefix(),
AclChangeNotificationNode.encode(resource));
- LOG.info("add acl change notification for resource {} ", resource);
+ LOG.info("addColumn acl change notification for resource {} ", resource);
}
public Map<String, String> fetchEntityConfig() throws Exception {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java
index 024707d8f..702400184 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/data/ZkData.java
@@ -25,6 +25,7 @@ import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.security.acl.Resource;
import org.apache.fluss.security.acl.ResourceType;
import org.apache.fluss.utils.json.JsonSerdeUtils;
+import org.apache.fluss.utils.types.Tuple2;
import javax.annotation.Nullable;
@@ -130,6 +131,7 @@ public final class ZkData {
* <p>/metadata/databases/[databaseName]/tables/[tableName]/schemas
*/
public static final class SchemasZNode {
+
public static String path(TablePath tablePath) {
return TableZNode.path(tablePath) + "/schemas";
}
@@ -142,6 +144,32 @@ public final class ZkData {
* <p>/metadata/databases/[databaseName]/tables/[tableName]/schemas/[schemaId]
*/
public static final class SchemaZNode {
+
+ @Nullable
+ public static Tuple2<TablePath, Integer> parsePath(String zkPath) {
+ String splitter = "/schemas/";
+ if (!zkPath.contains(splitter)) {
+ return null;
+ }
+
+ String[] split = zkPath.split(splitter);
+ if (split.length != 2) {
+ return null;
+ }
+ TablePath tablePath = TableZNode.parsePath(split[0]);
+ if (tablePath == null) {
+ return null;
+ }
+
+ int schemaId;
+ try {
+ schemaId = Integer.parseInt(split[1]);
+ return Tuple2.of(tablePath, schemaId);
+ } catch (NumberFormatException e) {
+ return null;
+ }
+ }
+
public static String path(TablePath tablePath, int schemaId) {
return SchemasZNode.path(tablePath) + "/" + schemaId;
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
index af69caffe..abc575fd5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
@@ -487,7 +487,7 @@ public class DefaultAuthorizerTest {
Set<AccessControlEntry> expectedAcls = new HashSet<>();
for (int i = 0; i < 50; i++) {
- // each task represent that add acl to different user on same resource.
+ // each task represent that addColumn acl to different user on same resource.
final AccessControlEntry accessControlEntry =
createAclEntry(String.valueOf(i), "host-1", READ);
int finalI = i;
@@ -550,7 +550,7 @@ public class DefaultAuthorizerTest {
// generate 50 concurrent acl operation tasks.
List<Runnable> concurrentTasks = new ArrayList<>();
for (int i = 0; i < 50; i++) {
- // each task represent that add acl to same user on same resource.
+ // each task represent that addColumn acl to same user on same resource.
Runnable runnable =
() -> {
addAcls(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
index b8fec59ca..4460cfc26 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
@@ -99,13 +99,13 @@ class CompletedSnapshotStoreManagerTest {
CompletedSnapshotStoreManager completedSnapshotStoreManager =
createCompletedSnapshotStoreManager(maxNumberOfSnapshotsToRetain);
- // add snapshots for a series of buckets
+ // addColumn snapshots for a series of buckets
Set<TableBucket> tableBuckets = createTableBuckets(2, 3);
int snapshotNum = 3;
Map<TableBucket, CompletedSnapshot> tableBucketLatestCompletedSnapshots = new HashMap<>();
for (TableBucket tableBucket : tableBuckets) {
- // add some snapshots
+ // addColumn some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
@@ -137,7 +137,7 @@ class CompletedSnapshotStoreManagerTest {
assertThat(completedSnapshot)
.isEqualTo(tableBucketLatestCompletedSnapshots.get(tableBucket));
- // add a new snapshot
+ // addColumn a new snapshot
long snapshotId = completedSnapshot.getSnapshotID() + 1;
completedSnapshot = KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshotId);
addCompletedSnapshot(completedSnapshotStoreManager, completedSnapshot);
@@ -169,7 +169,7 @@ class CompletedSnapshotStoreManagerTest {
Set<TableBucket> tableBuckets = createTableBuckets(1, 2);
int snapshotNum = 3;
for (TableBucket tableBucket : tableBuckets) {
- // add some snapshots
+ // addColumn some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
index bd4467c0a..581edafcc 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
@@ -69,14 +69,14 @@ class CoordinatorChannelManagerTest {
// now, shouldn't send as we already remove the tablet server
checkSendRequest(coordinatorChannelManager, server0.id(), false);
- // test add tablet server
- // before add, shouldn't send
+ // test addColumn tablet server
+ // before addColumn, shouldn't send
ServerNode server1 = tabletServersNode.get(1);
checkSendRequest(coordinatorChannelManager, server1.id(), false);
coordinatorChannelManager.addTabletServer(server1);
- // after add the tablet server, should send
+ // after addColumn the tablet server, should send
// try to send message
checkSendRequest(coordinatorChannelManager, server1.id(), true);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
index adbb14768..2549c597e 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessorTest.java
@@ -28,14 +28,19 @@ import org.apache.fluss.metadata.DatabaseDescriptor;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableBucketReplica;
+import org.apache.fluss.metadata.TableChange;
import org.apache.fluss.metadata.TableDescriptor;
+import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePartition;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.rpc.messages.AdjustIsrResponse;
... 2654 lines suppressed ...