You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@fluss.apache.org by ja...@apache.org on 2025/12/02 04:17:05 UTC
(fluss) 02/02: [server][client] Improve the design and performance of ADD COLUMN operation
This is an automated email from the ASF dual-hosted git repository.
jark pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/fluss.git
commit 07721fe6ecf033fb5be550d23cc82ce7fdc267e7
Author: Jark Wu <ja...@apache.org>
AuthorDate: Sun Nov 30 16:32:04 2025 +0800
[server][client] Improve the design and performance of ADD COLUMN operation
1. Make lookup requests async.
2. Introduce ServerProjectionCache to share ProjectionInfo in TabletServer level.
3. Revert FlinkAsFlussRow changes, and introduce PaddingRow for the padding null columns for schema changes.
4. Make PutKv zero-copy to persist kv records.
5. Rename SchemaMetadataManager to ServerSchemaCache and use TableId instead of TablePath to track schemas which is safer.
6. Revert source related changes to still include the projectedFieldIndexes parameter.
---
.../org/apache/fluss/client/admin/FlussAdmin.java | 22 +--
.../fluss/client/lookup/AbstractLookuper.java | 180 +++++++++++++++++++++
.../fluss/client/lookup/PrefixKeyLookuper.java | 66 ++------
.../apache/fluss/client/lookup/PrefixLookup.java | 5 +-
.../fluss/client/lookup/PrimaryKeyLookuper.java | 61 ++-----
.../fluss/client/metadata/ClientSchemaGetter.java | 69 ++++++--
.../org/apache/fluss/client/table/FlussTable.java | 6 +-
.../client/table/scanner/batch/BatchScanUtils.java | 3 +-
.../table/scanner/batch/LimitBatchScanner.java | 22 ++-
.../table/scanner/batch/SnapshotFilesReader.java | 22 ++-
.../client/table/scanner/log/BucketScanStatus.java | 3 +-
.../fluss/client/table/scanner/log/LogFetcher.java | 3 +-
.../client/table/writer/AbstractTableWriter.java | 2 -
.../fluss/client/utils/ClientRpcMessageUtils.java | 65 ++++----
.../fluss/client/write/RecordAccumulator.java | 3 +-
.../java/org/apache/fluss/client/write/Sender.java | 6 +-
.../fluss/client/admin/FlussAdminITCase.java | 37 ++++-
.../batch/KvSnapshotBatchScannerITCase.java | 4 +-
.../scanner/log/DefaultCompletedFetchTest.java | 3 +-
.../table/scanner/log/LogFetchCollectorTest.java | 3 +-
.../client/table/scanner/log/LogFetcherTest.java | 1 +
.../fluss/client/write/RecordAccumulatorTest.java | 1 -
.../org/apache/fluss/exception/ApiException.java | 16 +-
.../fluss/exception/SchemaChangeException.java | 4 +
.../java/org/apache/fluss/metadata/Schema.java | 15 +-
.../org/apache/fluss/metadata/SchemaGetter.java | 50 +++++-
.../org/apache/fluss/metadata/TableChange.java | 2 +-
.../java/org/apache/fluss/metadata/TableInfo.java | 5 +
.../java/org/apache/fluss/record/BinaryValue.java | 62 +++++++
.../apache/fluss/record/DefaultLogRecordBatch.java | 96 ++++-------
.../org/apache/fluss/record/FileLogProjection.java | 108 +++++--------
.../apache/fluss/record/LogRecordReadContext.java | 57 +++----
.../fluss/record/ProjectionPushdownCache.java | 90 +++++++++++
.../main/java/org/apache/fluss/row/PaddingRow.java | 133 +++++++++++++++
.../java/org/apache/fluss/row/ProjectedRow.java | 6 +-
.../fluss/row/decode/FixedSchemaDecoder.java | 95 +++++++++++
.../org/apache/fluss/row/decode/RowDecoder.java | 5 +-
.../org/apache/fluss/row/encode/ValueDecoder.java | 16 +-
.../org/apache/fluss/row/encode/ValueEncoder.java | 25 +--
.../java/org/apache/fluss/utils/Projection.java | 6 +-
.../java/org/apache/fluss/utils/SchemaUtil.java | 46 ++----
.../apache/fluss/record/FileLogProjectionTest.java | 55 ++-----
.../java/org/apache/fluss/record/LogTestBase.java | 1 -
.../apache/fluss/record/TestingSchemaGetter.java | 12 +-
.../java/org/apache/fluss/row/PaddingRowTest.java | 79 +++++++++
.../org/apache/fluss/row/ProjectedRowTest.java | 1 +
.../src/test/resources/log4j2-test.properties | 2 +-
.../apache/fluss/flink/row/FlinkAsFlussRow.java | 100 ++----------
.../sink/serializer/FlussSerializationSchema.java | 7 +-
.../serializer/RowDataSerializationSchema.java | 76 ++++++++-
.../sink/serializer/SerializerInitContextImpl.java | 2 +-
.../fluss/flink/sink/writer/FlinkSinkWriter.java | 21 +--
.../org/apache/fluss/flink/source/FlinkSource.java | 7 +-
.../fluss/flink/source/FlinkTableSource.java | 1 +
.../org/apache/fluss/flink/source/FlussSource.java | 4 +
.../fluss/flink/source/FlussSourceBuilder.java | 1 +
.../source/lookup/FlinkAsyncLookupFunction.java | 29 +++-
.../flink/source/lookup/FlinkLookupFunction.java | 27 +++-
.../flink/source/reader/FlinkSourceReader.java | 4 +
.../source/reader/FlinkSourceSplitReader.java | 67 ++++----
.../apache/fluss/flink/utils/FlinkConversions.java | 12 --
.../flink/utils/FlussRowToFlinkRowConverter.java | 39 ++---
.../fluss/flink/row/FlinkAsFlussRowTest.java | 3 +-
.../source/lookup/FlinkLookupFunctionTest.java | 12 +-
.../flink/source/reader/FlinkSourceReaderTest.java | 1 +
.../source/reader/FlinkSourceSplitReaderTest.java | 28 +++-
.../fluss/lake/iceberg/conf/HadoopUtils.java | 5 +-
.../iceberg/maintenance/IcebergRewriteITCase.java | 2 +-
.../fluss/rpc/netty/client/ServerConnection.java | 3 +-
fluss-rpc/src/main/proto/FlussApi.proto | 12 +-
.../org/apache/fluss/server/RpcServiceBase.java | 2 +-
.../coordinator/CompletedSnapshotStoreManager.java | 2 +-
.../coordinator/CoordinatorEventProcessor.java | 1 -
.../fluss/server/coordinator/MetadataManager.java | 16 +-
.../fluss/server/coordinator/SchemaUpdate.java | 28 ++--
.../fluss/server/coordinator/UpdateSchema.java | 28 ----
.../event/watcher/TableChangeWatcher.java | 9 --
.../server/entity/BatchRegisterLeadAndIsr.java | 2 +-
.../apache/fluss/server/kv/KvRecoverHelper.java | 99 +++++-------
.../java/org/apache/fluss/server/kv/KvTablet.java | 160 ++++++++----------
.../server/kv/partialupdate/PartialUpdater.java | 45 +++---
.../kv/partialupdate/PartialUpdaterCache.java | 9 +-
.../server/kv/rowmerger/DefaultRowMerger.java | 14 +-
.../server/kv/rowmerger/FirstRowRowMerger.java | 8 +-
.../fluss/server/kv/rowmerger/RowMerger.java | 24 +--
.../server/kv/rowmerger/VersionedRowMerger.java | 7 +-
.../server/kv/snapshot/CompletedSnapshotStore.java | 4 +-
.../fluss/server/kv/snapshot/SnapshotLocation.java | 2 +-
.../org/apache/fluss/server/log/FetchParams.java | 8 +-
.../java/org/apache/fluss/server/log/LocalLog.java | 2 -
.../server/log/remote/FsRemoteLogOutputStream.java | 2 +-
.../fluss/server/log/remote/LogSegmentFiles.java | 2 +-
.../fluss/server/log/remote/LogTieringTask.java | 4 +-
...MetadataManager.java => ServerSchemaCache.java} | 87 +++++-----
.../server/metadata/TabletServerMetadataCache.java | 24 ++-
.../fluss/server/replica/AdjustIsrManager.java | 2 +-
.../org/apache/fluss/server/replica/Replica.java | 5 +-
.../fluss/server/replica/ReplicaManager.java | 5 +-
.../replica/fetcher/ReplicaFetcherManager.java | 2 +-
.../replica/fetcher/ReplicaFetcherThread.java | 2 +-
.../fluss/server/utils/RpcGatewayManager.java | 2 +-
.../fluss/server/utils/ServerRpcMessageUtils.java | 2 +-
.../org/apache/fluss/server/utils/timer/Timer.java | 2 +-
.../fluss/server/utils/timer/TimingWheel.java | 4 +-
.../apache/fluss/server/zk/ZooKeeperClient.java | 52 ++++--
.../server/authorizer/DefaultAuthorizerTest.java | 4 +-
.../CompletedSnapshotStoreManagerTest.java | 8 +-
.../coordinator/CoordinatorChannelManagerTest.java | 6 +-
.../fluss/server/coordinator/TableManagerTest.java | 2 -
.../org/apache/fluss/server/kv/KvTabletTest.java | 11 +-
.../server/kv/prewrite/KvPreWriteBufferTest.java | 4 +-
.../server/kv/rowmerger/DefaultRowMergerTest.java | 57 ++++---
.../kv/rowmerger/VersionedRowMergerTest.java | 41 ++---
.../kv/snapshot/CompletedSnapshotStoreTest.java | 2 +-
.../kv/snapshot/KvTabletSnapshotTargetTest.java | 2 +-
.../ZooKeeperCompletedSnapshotStoreTest.java | 4 +-
.../fluss/server/log/DroppedTableRecoveryTest.java | 2 +-
.../apache/fluss/server/log/FetchParamsTest.java | 14 +-
.../apache/fluss/server/log/LogManagerTest.java | 6 +-
.../org/apache/fluss/server/log/LogTabletTest.java | 4 +-
.../server/log/remote/RemoteLogTabletTest.java | 4 +-
.../server/metadata/MetadataUpdateITCase.java | 2 +-
...ManagerTest.java => ServerSchemaCacheTest.java} | 62 ++++---
.../fluss/server/replica/AdjustIsrITCase.java | 2 +-
.../fluss/server/replica/AdjustIsrManagerTest.java | 2 +-
.../replica/HighWatermarkPersistenceTest.java | 2 +-
.../replica/RemoveOfflineReplicaFromIsrITCase.java | 2 +-
.../fluss/server/replica/ReplicaManagerTest.java | 21 +--
.../apache/fluss/server/replica/ReplicaTest.java | 26 ++-
.../fluss/server/replica/ReplicaTestBase.java | 12 +-
.../replica/fetcher/ReplicaFetcherThreadTest.java | 2 +-
.../fluss/server/tablet/TabletServiceITCase.java | 6 +-
.../server/testutils/FlussClusterExtension.java | 20 +--
.../fluss/server/zk/ZooKeeperClientTest.java | 4 +-
.../src/test/resources/log4j2-test.properties | 2 +-
fluss-test-coverage/pom.xml | 4 +-
136 files changed, 1820 insertions(+), 1224 deletions(-)
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
index 25bfcfbd8..69070e01c 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/admin/FlussAdmin.java
@@ -88,9 +88,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
-import java.util.stream.Collectors;
-import static org.apache.fluss.client.utils.ClientRpcMessageUtils.addPbAlterSchemas;
+import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeAlterTableRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeCreatePartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeDropPartitionRequest;
import static org.apache.fluss.client.utils.ClientRpcMessageUtils.makeListOffsetsRequest;
@@ -249,23 +248,8 @@ public class FlussAdmin implements Admin {
public CompletableFuture<Void> alterTable(
TablePath tablePath, List<TableChange> tableChanges, boolean ignoreIfNotExists) {
tablePath.validate();
- AlterTableRequest request = new AlterTableRequest();
- List<PbAlterConfig> pbFlussTableChanges =
- tableChanges.stream()
- .filter(tableChange -> !(tableChange instanceof TableChange.SchemaChange))
- .map(ClientRpcMessageUtils::toPbAlterConfigs)
- .collect(Collectors.toList());
-
- List<TableChange> schemaChanges =
- tableChanges.stream()
- .filter(tableChange -> tableChange instanceof TableChange.SchemaChange)
- .collect(Collectors.toList());
- addPbAlterSchemas(request, schemaChanges);
- request.addAllConfigChanges(pbFlussTableChanges)
- .setIgnoreIfNotExists(ignoreIfNotExists)
- .setTablePath()
- .setDatabaseName(tablePath.getDatabaseName())
- .setTableName(tablePath.getTableName());
+ AlterTableRequest request =
+ makeAlterTableRequest(tablePath, tableChanges, ignoreIfNotExists);
return gateway.alterTable(request).thenApply(r -> null);
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java
new file mode 100644
index 000000000..6cd93adf7
--- /dev/null
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/AbstractLookuper.java
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.client.lookup;
+
+import org.apache.fluss.client.metadata.MetadataUpdater;
+import org.apache.fluss.memory.MemorySegment;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.metadata.SchemaInfo;
+import org.apache.fluss.metadata.TableInfo;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.decode.FixedSchemaDecoder;
+import org.apache.fluss.utils.CopyOnWriteMap;
+import org.apache.fluss.utils.concurrent.FutureUtils;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+
+import static org.apache.fluss.utils.Preconditions.checkArgument;
+
+/** Abstract lookuper implementation for common methods. */
+abstract class AbstractLookuper implements Lookuper {
+ protected final TableInfo tableInfo;
+
+ protected final MetadataUpdater metadataUpdater;
+
+ protected final LookupClient lookupClient;
+
+ protected final short targetSchemaId;
+
+ private final SchemaGetter schemaGetter;
+
+ /**
+ * Cache for row decoders for different schema ids. Use CopyOnWriteMap for fast access, as it is
+ * not frequently updated.
+ */
+ private final CopyOnWriteMap<Short, FixedSchemaDecoder> decoders;
+
+ AbstractLookuper(
+ TableInfo tableInfo,
+ MetadataUpdater metadataUpdater,
+ LookupClient lookupClient,
+ SchemaGetter schemaGetter) {
+ this.tableInfo = tableInfo;
+ this.metadataUpdater = metadataUpdater;
+ this.lookupClient = lookupClient;
+ this.targetSchemaId = (short) tableInfo.getSchemaId();
+ this.schemaGetter = schemaGetter;
+ this.decoders = new CopyOnWriteMap<>();
+ // initialize the decoder for the same schema
+ this.decoders.put(
+ targetSchemaId,
+ new FixedSchemaDecoder(
+ tableInfo.getTableConfig().getKvFormat(), tableInfo.getSchema()));
+ }
+
+ protected void handleLookupResponse(
+ List<byte[]> result, CompletableFuture<LookupResult> lookupFuture) {
+ List<MemorySegment> valueList = new ArrayList<>(result.size());
+ Set<Short> schemaIdsToRequest = new HashSet<>();
+ boolean allTargetSchema = true;
+ for (byte[] valueBytes : result) {
+ if (valueBytes == null) {
+ continue;
+ }
+ MemorySegment memorySegment = MemorySegment.wrap(valueBytes);
+ short schemaId = memorySegment.getShort(0);
+ if (targetSchemaId != schemaId) {
+ allTargetSchema = false;
+ if (!decoders.containsKey(schemaId)) {
+ schemaIdsToRequest.add(schemaId);
+ }
+ }
+ valueList.add(memorySegment);
+ }
+ // all schema ids are the target schema id, fast path
+ if (allTargetSchema) {
+ lookupFuture.complete(processAllTargetSchemaRows(valueList));
+ return;
+ }
+
+ // all schema ids and decoders are cached in local
+ if (schemaIdsToRequest.isEmpty()) {
+ lookupFuture.complete(processSchemaMismatchedRows(valueList));
+ return;
+ }
+
+ // need to fetch schema infos for schema ids
+ List<CompletableFuture<SchemaInfo>> schemaFutures =
+ new ArrayList<>(schemaIdsToRequest.size());
+ for (short schemaId : schemaIdsToRequest) {
+ CompletableFuture<SchemaInfo> schemaFuture = schemaGetter.getSchemaInfoAsync(schemaId);
+ schemaFutures.add(schemaFuture);
+ }
+ FutureUtils.ConjunctFuture<Collection<SchemaInfo>> allFutures =
+ FutureUtils.combineAll(schemaFutures);
+ // normal async path
+ allFutures.whenComplete(
+ (schemas, error) -> {
+ if (error != null) {
+ lookupFuture.completeExceptionally(
+ new RuntimeException(
+ "Failed to get schema infos for lookup", error));
+ } else {
+ LookupResult lookupResult = processSchemaRequestedRows(schemas, valueList);
+ lookupFuture.complete(lookupResult);
+ }
+ });
+ }
+
+ protected LookupResult processAllTargetSchemaRows(List<MemorySegment> valueList) {
+ FixedSchemaDecoder decoder = decoders.get(targetSchemaId);
+ List<InternalRow> rowList = new ArrayList<>(valueList.size());
+ for (MemorySegment value : valueList) {
+ rowList.add(decoder.decode(value));
+ }
+ return new LookupResult(rowList);
+ }
+
+ protected LookupResult processSchemaMismatchedRows(List<MemorySegment> valueList) {
+ List<InternalRow> rowList = new ArrayList<>(valueList.size());
+ for (MemorySegment value : valueList) {
+ short schemaId = value.getShort(0);
+ FixedSchemaDecoder decoder = decoders.get(schemaId);
+ checkArgument(decoder != null, "Decoder for schema id %s not found", schemaId);
+ InternalRow row = decoder.decode(value);
+ rowList.add(row);
+ }
+ return new LookupResult(rowList);
+ }
+
+ protected LookupResult processSchemaRequestedRows(
+ Collection<SchemaInfo> schemaInfos, List<MemorySegment> valueList) {
+ // build a map from schema id to the target schema decoder
+ Map<Short, Schema> schemaMap = new HashMap<>();
+ for (SchemaInfo schemaInfo : schemaInfos) {
+ schemaMap.put((short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ }
+
+ // process the value list to convert to target schema
+ List<InternalRow> rowList = new ArrayList<>(valueList.size());
+ for (MemorySegment value : valueList) {
+ short schemaId = value.getShort(0);
+ FixedSchemaDecoder decoder =
+ decoders.computeIfAbsent(
+ schemaId,
+ (id) -> {
+ Schema sourceSchema = schemaMap.get(id);
+ return new FixedSchemaDecoder(
+ tableInfo.getTableConfig().getKvFormat(),
+ sourceSchema,
+ tableInfo.getSchema());
+ });
+ InternalRow row = decoder.decode(value);
+ rowList.add(row);
+ }
+ return new LookupResult(rowList);
+ }
+}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
index c9f6308f8..cf658ce80 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixKeyLookuper.java
@@ -22,14 +22,11 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
-import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
-import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -49,13 +46,7 @@ import static org.apache.fluss.client.utils.ClientUtils.getPartitionId;
* of the primary key.
*/
@NotThreadSafe
-class PrefixKeyLookuper implements Lookuper {
-
- private final TableInfo tableInfo;
-
- private final MetadataUpdater metadataUpdater;
-
- private final LookupClient lookupClient;
+class PrefixKeyLookuper extends AbstractLookuper {
/** Extract bucket key from prefix lookup key row. */
private final KeyEncoder bucketKeyEncoder;
@@ -68,24 +59,16 @@ class PrefixKeyLookuper implements Lookuper {
*/
private @Nullable final PartitionGetter partitionGetter;
- /** Decode the lookup bytes to result row. */
- private final ValueDecoder kvValueDecoder;
-
- private final SchemaGetter schemaGetter;
-
public PrefixKeyLookuper(
TableInfo tableInfo,
SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient,
List<String> lookupColumnNames) {
+ super(tableInfo, metadataUpdater, lookupClient, schemaGetter);
// sanity check
validatePrefixLookup(tableInfo, lookupColumnNames);
- // initialization
- this.tableInfo = tableInfo;
this.numBuckets = tableInfo.getNumBuckets();
- this.metadataUpdater = metadataUpdater;
- this.lookupClient = lookupClient;
// the row type of the input lookup row
RowType lookupRowType = tableInfo.getRowType().project(lookupColumnNames);
DataLakeFormat lakeFormat = tableInfo.getTableConfig().getDataLakeFormat().orElse(null);
@@ -96,9 +79,6 @@ class PrefixKeyLookuper implements Lookuper {
tableInfo.isPartitioned()
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
- this.kvValueDecoder =
- new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
- this.schemaGetter = schemaGetter;
}
private void validatePrefixLookup(TableInfo tableInfo, List<String> lookupColumns) {
@@ -168,36 +148,22 @@ class PrefixKeyLookuper implements Lookuper {
}
}
+ CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- CompletableFuture<LookupResult> future = new CompletableFuture<>();
-
- CompletableFuture.runAsync(
- () -> {
- try {
- List<byte[]> result =
- lookupClient.prefixLookup(tableBucket, bucketKeyBytes).get();
- List<InternalRow> rowList = new ArrayList<>(result.size());
- for (byte[] valueBytes : result) {
- if (valueBytes == null) {
- continue;
- }
- ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
- InternalRow row;
- if (value.schemaId == tableInfo.getSchemaId()) {
- row = value.row;
+ lookupClient
+ .prefixLookup(tableBucket, bucketKeyBytes)
+ .whenComplete(
+ (result, error) -> {
+ if (error != null) {
+ lookupFuture.completeExceptionally(
+ new RuntimeException(
+ "Failed to perform prefix lookup for table: "
+ + tableInfo.getTablePath(),
+ error));
} else {
- Schema schema = schemaGetter.getSchema(value.schemaId);
- row =
- ProjectedRow.from(schema, tableInfo.getSchema())
- .replaceRow(value.row);
+ handleLookupResponse(result, lookupFuture);
}
- rowList.add(row);
- }
- future.complete(new LookupResult(rowList));
- } catch (Exception e) {
- future.complete(new LookupResult(Collections.emptyList()));
- }
- });
- return future;
+ });
+ return lookupFuture;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java
index 18ee9a61b..4eadcd8f7 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrefixLookup.java
@@ -23,7 +23,8 @@ import org.apache.fluss.row.InternalRow;
import java.util.List;
/**
- * Used to describe the operation to prefix lookup by {@link PrefixLookuper} to a primary key table.
+ * Used to describe the operation to prefix lookup by {@link PrefixKeyLookuper} to a primary key
+ * table.
*
* @since 0.6
*/
@@ -37,7 +38,7 @@ public class PrefixLookup {
* <p>For partitioned table, the lookupColumnNames exclude partition fields should be a prefix
* of primary key exclude partition fields.
*
- * <p>See {@link PrefixLookuper#prefixLookup(InternalRow)} for more details.
+ * <p>See {@link PrefixKeyLookuper#lookup(InternalRow)} for more details.
*/
private final List<String> lookupColumnNames;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
index 091a8a38e..755e454df 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/lookup/PrimaryKeyLookuper.java
@@ -22,14 +22,11 @@ import org.apache.fluss.client.metadata.MetadataUpdater;
import org.apache.fluss.client.table.getter.PartitionGetter;
import org.apache.fluss.exception.PartitionNotExistException;
import org.apache.fluss.metadata.DataLakeFormat;
-import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.encode.KeyEncoder;
-import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.types.RowType;
import javax.annotation.Nullable;
@@ -40,17 +37,10 @@ import java.util.concurrent.CompletableFuture;
import static org.apache.fluss.client.utils.ClientUtils.getPartitionId;
import static org.apache.fluss.utils.Preconditions.checkArgument;
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** An implementation of {@link Lookuper} that lookups by primary key. */
@NotThreadSafe
-class PrimaryKeyLookuper implements Lookuper {
-
- private final TableInfo tableInfo;
-
- private final MetadataUpdater metadataUpdater;
-
- private final LookupClient lookupClient;
+class PrimaryKeyLookuper extends AbstractLookuper {
private final KeyEncoder primaryKeyEncoder;
@@ -66,24 +56,17 @@ class PrimaryKeyLookuper implements Lookuper {
/** a getter to extract partition from lookup key row, null when it's not a partitioned. */
private @Nullable final PartitionGetter partitionGetter;
- /** Decode the lookup bytes to result row. */
- private final ValueDecoder kvValueDecoder;
-
- private final SchemaGetter schemaGetter;
-
public PrimaryKeyLookuper(
TableInfo tableInfo,
SchemaGetter schemaGetter,
MetadataUpdater metadataUpdater,
LookupClient lookupClient) {
+ super(tableInfo, metadataUpdater, lookupClient, schemaGetter);
checkArgument(
tableInfo.hasPrimaryKey(),
"Log table %s doesn't support lookup",
tableInfo.getTablePath());
- this.tableInfo = tableInfo;
this.numBuckets = tableInfo.getNumBuckets();
- this.metadataUpdater = metadataUpdater;
- this.lookupClient = lookupClient;
// the row type of the input lookup row
RowType lookupRowType = tableInfo.getRowType().project(tableInfo.getPrimaryKeys());
@@ -102,9 +85,6 @@ class PrimaryKeyLookuper implements Lookuper {
tableInfo.isPartitioned()
? new PartitionGetter(lookupRowType, tableInfo.getPartitionKeys())
: null;
- this.kvValueDecoder =
- new ValueDecoder(schemaGetter, tableInfo.getTableConfig().getKvFormat());
- this.schemaGetter = schemaGetter;
}
@Override
@@ -132,30 +112,21 @@ class PrimaryKeyLookuper implements Lookuper {
int bucketId = bucketingFunction.bucketing(bkBytes, numBuckets);
TableBucket tableBucket = new TableBucket(tableInfo.getTableId(), partitionId, bucketId);
- CompletableFuture<LookupResult> future = new CompletableFuture<>();
-
- CompletableFuture.runAsync(
- () -> {
- try {
- byte[] valueBytes = lookupClient.lookup(tableBucket, pkBytes).get();
- InternalRow row = null;
- if (valueBytes != null) {
- ValueDecoder.Value value = kvValueDecoder.decodeValue(valueBytes);
- if (value.schemaId == tableInfo.getSchemaId()) {
- row = value.row;
+ CompletableFuture<LookupResult> lookupFuture = new CompletableFuture<>();
+ lookupClient
+ .lookup(tableBucket, pkBytes)
+ .whenComplete(
+ (result, error) -> {
+ if (error != null) {
+ lookupFuture.completeExceptionally(error);
} else {
- Schema schema = schemaGetter.getSchema(value.schemaId);
- checkNotNull(schema, "schema is null");
- row =
- ProjectedRow.from(schema, tableInfo.getSchema())
- .replaceRow(value.row);
+ handleLookupResponse(
+ result == null
+ ? Collections.emptyList()
+ : Collections.singletonList(result),
+ lookupFuture);
}
- }
- future.complete(new LookupResult(row));
- } catch (Exception e) {
- future.completeExceptionally(e);
- }
- });
- return future;
+ });
+ return lookupFuture;
}
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
index 181767781..6b6a11d77 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/metadata/ClientSchemaGetter.java
@@ -19,6 +19,8 @@ package org.apache.fluss.client.metadata;
import org.apache.fluss.annotation.Internal;
import org.apache.fluss.client.admin.Admin;
+import org.apache.fluss.exception.SchemaNotExistException;
+import org.apache.fluss.exception.TableNotExistException;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.SchemaInfo;
@@ -28,8 +30,10 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
+import static org.apache.fluss.utils.ExceptionUtils.stripExecutionException;
import static org.apache.fluss.utils.MapUtils.newConcurrentHashMap;
/** Schema getter for client. */
@@ -40,7 +44,7 @@ public class ClientSchemaGetter implements SchemaGetter {
private final TablePath tablePath;
private final Map<Integer, Schema> schemasById;
private final Admin admin;
- private SchemaInfo latestSchemaInfo;
+ private volatile SchemaInfo latestSchemaInfo;
public ClientSchemaGetter(TablePath tablePath, SchemaInfo latestSchemaInfo, Admin admin) {
this.tablePath = tablePath;
@@ -52,22 +56,55 @@ public class ClientSchemaGetter implements SchemaGetter {
@Override
public Schema getSchema(int schemaId) {
- return schemasById.computeIfAbsent(
- schemaId,
- (id) -> {
- try {
- SchemaInfo schemaInfo =
- admin.getTableSchema(tablePath, schemaId).get(1, TimeUnit.MINUTES);
- if (id > latestSchemaInfo.getSchemaId()) {
- latestSchemaInfo = schemaInfo;
- }
- return schemaInfo.getSchema();
+ Schema schema = schemasById.get(schemaId);
+ if (schema != null) {
+ return schema;
+ } else {
+ LOG.debug(
+ "Schema id {} not found in cache, fetching from cluster for table: {}",
+ schemaId,
+ tablePath);
+ try {
+ SchemaInfo schemaInfo =
+ admin.getTableSchema(tablePath, schemaId).get(1, TimeUnit.MINUTES);
+ if (schemaId > latestSchemaInfo.getSchemaId()) {
+ latestSchemaInfo = schemaInfo;
+ }
+ return schemaInfo.getSchema();
+ } catch (Exception e) {
+ Throwable strippedException = stripExecutionException(e);
+ if (strippedException instanceof SchemaNotExistException) {
+ throw (SchemaNotExistException) strippedException;
+ } else if (strippedException instanceof TableNotExistException) {
+ throw (TableNotExistException) strippedException;
+ } else {
+ LOG.warn("Failed to get schema for table: {}", tablePath);
+ throw new RuntimeException(e);
+ }
+ }
+ }
+ }
- } catch (Exception e) {
- LOG.warn("Failed to get schema for table: " + tablePath);
- throw new RuntimeException(e);
- }
- });
+ @Override
+ public CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId) {
+ Schema schema = schemasById.get(schemaId);
+ if (schema != null) {
+ return CompletableFuture.completedFuture(new SchemaInfo(schema, schemaId));
+ } else {
+ LOG.debug(
+ "Schema id {} not found in cache, fetching from cluster for table: {}",
+ schemaId,
+ tablePath);
+ return admin.getTableSchema(tablePath, schemaId)
+ .thenApply(
+ (schemaInfo) -> {
+ schemasById.put(schemaId, schemaInfo.getSchema());
+ if (schemaId > latestSchemaInfo.getSchemaId()) {
+ latestSchemaInfo = schemaInfo;
+ }
+ return schemaInfo;
+ });
+ }
}
@Override
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
index 629abe6ab..8532f2a85 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/FlussTable.java
@@ -29,7 +29,6 @@ import org.apache.fluss.client.table.writer.TableAppend;
import org.apache.fluss.client.table.writer.TableUpsert;
import org.apache.fluss.client.table.writer.Upsert;
import org.apache.fluss.metadata.SchemaGetter;
-import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
@@ -55,10 +54,7 @@ public class FlussTable implements Table {
this.tableInfo = tableInfo;
this.hasPrimaryKey = tableInfo.hasPrimaryKey();
this.schemaGetter =
- new ClientSchemaGetter(
- tableInfo.getTablePath(),
- new SchemaInfo(tableInfo.getSchema(), tableInfo.getSchemaId()),
- conn.getAdmin());
+ new ClientSchemaGetter(tablePath, tableInfo.getSchemaInfo(), conn.getAdmin());
}
@Override
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
index f4e150f30..d36bf9a7d 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/BatchScanUtils.java
@@ -71,8 +71,7 @@ public class BatchScanUtils {
// If the scanner has more data, add it back to the queue
scannerQueue.add(scanner);
} else {
- // Close the scanner if it has no more data, and not add it back to the
- // queue
+ // Close the scanner if it has no more data, and not add it back to the queue
scanner.close();
}
} catch (Exception e) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
index e6208f368..06c79fce8 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/LimitBatchScanner.java
@@ -39,6 +39,7 @@ import org.apache.fluss.rpc.messages.LimitScanRequest;
import org.apache.fluss.rpc.messages.LimitScanResponse;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.CloseableIterator;
+import org.apache.fluss.utils.SchemaUtil;
import javax.annotation.Nullable;
@@ -47,7 +48,9 @@ import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
@@ -64,6 +67,12 @@ public class LimitBatchScanner implements BatchScanner {
private final KvFormat kvFormat;
private final int targetSchemaId;
+ /**
+ * A cache for schema projection mapping from source schema to target. Use HashMap here, because
+ * LimitBatchScanner is used in single thread only.
+ */
+ private final Map<Short, int[]> schemaProjectionCache = new HashMap<>();
+
private boolean endOfInput;
public LimitBatchScanner(
@@ -142,11 +151,14 @@ public class LimitBatchScanner implements BatchScanner {
for (ValueRecord record : valueRecords.records(readContext)) {
InternalRow row = record.getRow();
if (targetSchemaId != record.schemaId()) {
- row =
- ProjectedRow.from(
- schemaGetter.getSchema(record.schemaId()),
- schemaGetter.getSchema(targetSchemaId))
- .replaceRow(row);
+ int[] indexMapping =
+ schemaProjectionCache.computeIfAbsent(
+ record.schemaId(),
+ sourceSchemaId ->
+ SchemaUtil.getIndexMapping(
+ schemaGetter.getSchema(sourceSchemaId),
+ schemaGetter.getSchema(targetSchemaId)));
+ row = ProjectedRow.from(indexMapping).replaceRow(row);
}
scanRows.add(maybeProject(row));
}
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
index 10e773609..8a52814f3 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/batch/SnapshotFilesReader.java
@@ -22,6 +22,7 @@ import org.apache.fluss.exception.FlussRuntimeException;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.rocksdb.RocksDBHandle;
import org.apache.fluss.rocksdb.RocksIteratorWrapper;
import org.apache.fluss.row.InternalRow;
@@ -30,6 +31,7 @@ import org.apache.fluss.row.encode.ValueDecoder;
import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.CloseableRegistry;
import org.apache.fluss.utils.IOUtils;
+import org.apache.fluss.utils.SchemaUtil;
import org.rocksdb.ColumnFamilyOptions;
import org.rocksdb.DBOptions;
@@ -43,6 +45,8 @@ import javax.annotation.concurrent.NotThreadSafe;
import java.io.IOException;
import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.Map;
/**
* A reader to read kv snapshot files to {@link ScanRecord}s. It will return the {@link ScanRecord}s
@@ -58,6 +62,12 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
@Nullable private final int[] projectedFields;
private RocksIteratorWrapper rocksIteratorWrapper;
+ /**
+ * A cache for schema projection mapping from source schema to target. Use HashMap here, because
+ * SnapshotFilesReader is used in single thread only.
+ */
+ private final Map<Short, int[]> schemaProjectionCache = new HashMap<>();
+
private Snapshot snapshot;
private RocksDBHandle rocksDBHandle;
private boolean isClose = false;
@@ -155,12 +165,16 @@ class SnapshotFilesReader implements CloseableIterator<InternalRow> {
byte[] value = rocksIteratorWrapper.value();
rocksIteratorWrapper.next();
- ValueDecoder.Value originValue = valueDecoder.decodeValue(value);
+ BinaryValue originValue = valueDecoder.decodeValue(value);
InternalRow originRow = originValue.row;
if (targetSchemaId != originValue.schemaId) {
- originRow =
- ProjectedRow.from(schemaGetter.getSchema(originValue.schemaId), targetSchema)
- .replaceRow(originRow);
+ int[] indexMapping =
+ schemaProjectionCache.computeIfAbsent(
+ originValue.schemaId,
+ sourceSchemaId ->
+ SchemaUtil.getIndexMapping(
+ schemaGetter.getSchema(sourceSchemaId), targetSchema));
+ originRow = ProjectedRow.from(indexMapping).replaceRow(originRow);
}
if (projectedFields != null) {
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
index bee0910e1..738e04a49 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/BucketScanStatus.java
@@ -24,8 +24,7 @@ import org.apache.fluss.annotation.Internal;
class BucketScanStatus {
private long offset; // last consumed position
private long highWatermark; // the high watermark from last fetch
- // TODO add
- // resetStrategy and nextAllowedRetryTimeMs.
+ // TODO add resetStrategy and nextAllowedRetryTimeMs.
public BucketScanStatus() {
this.offset = 0L;
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
index e8067dd4c..b19347772 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/scanner/log/LogFetcher.java
@@ -364,8 +364,7 @@ public class LogFetcher implements Closeable {
if (!MemoryLogRecords.EMPTY.equals(logRecords)
|| fetchResultForBucket.getErrorCode() != Errors.NONE.code()) {
// In oder to not signal notEmptyCondition, add completed
- // fetch to
- // buffer until log records is not empty.
+ // fetch to buffer until log records is not empty.
DefaultCompletedFetch completedFetch =
new DefaultCompletedFetch(
tb,
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
index f1811dfa2..98a93fdd2 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/table/writer/AbstractTableWriter.java
@@ -37,13 +37,11 @@ public abstract class AbstractTableWriter implements TableWriter {
protected final TablePath tablePath;
protected final WriterClient writerClient;
protected final int fieldCount;
- protected TableInfo tableInfo;
private final @Nullable PartitionGetter partitionFieldGetter;
protected AbstractTableWriter(
TablePath tablePath, TableInfo tableInfo, WriterClient writerClient) {
this.tablePath = tablePath;
- this.tableInfo = tableInfo;
this.writerClient = writerClient;
this.fieldCount = tableInfo.getRowType().getFieldCount();
this.partitionFieldGetter =
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
index 8b36b5cf8..58efb7f47 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/utils/ClientRpcMessageUtils.java
@@ -336,6 +336,44 @@ public class ClientRpcMessageUtils {
return dropPartitionRequest;
}
+ public static AlterTableRequest makeAlterTableRequest(
+ TablePath tablePath, List<TableChange> tableChanges, boolean ignoreIfNotExists) {
+ AlterTableRequest request = new AlterTableRequest();
+ request.setIgnoreIfNotExists(ignoreIfNotExists)
+ .setTablePath()
+ .setDatabaseName(tablePath.getDatabaseName())
+ .setTableName(tablePath.getTableName());
+
+ List<PbAddColumn> addColumns = new ArrayList<>();
+ List<PbDropColumn> dropColumns = new ArrayList<>();
+ List<PbRenameColumn> renameColumns = new ArrayList<>();
+ List<PbModifyColumn> modifyColumns = new ArrayList<>();
+ List<PbAlterConfig> alterConfigs = new ArrayList<>();
+ for (TableChange tableChange : tableChanges) {
+ if (tableChange instanceof TableChange.AddColumn) {
+ addColumns.add(toPbAddColumn((TableChange.AddColumn) tableChange));
+ } else if (tableChange instanceof TableChange.DropColumn) {
+ dropColumns.add(toPbDropColumn((TableChange.DropColumn) tableChange));
+ } else if (tableChange instanceof TableChange.RenameColumn) {
+ renameColumns.add(toPbRenameColumn((TableChange.RenameColumn) tableChange));
+ } else if (tableChange instanceof TableChange.ModifyColumn) {
+ modifyColumns.add(toPbModifyColumn((TableChange.ModifyColumn) tableChange));
+ } else if (tableChange instanceof TableChange.SetOption
+ || tableChange instanceof TableChange.ResetOption) {
+ alterConfigs.add(toPbAlterConfigs(tableChange));
+ } else {
+ throw new IllegalArgumentException(
+ "Unsupported table change: " + tableChange.getClass());
+ }
+ }
+ request.addAllConfigChanges(alterConfigs)
+ .addAllAddColumns(addColumns)
+ .addAllDropColumns(dropColumns)
+ .addAllRenameColumns(renameColumns)
+ .addAllModifyColumns(modifyColumns);
+ return request;
+ }
+
public static List<PartitionInfo> toPartitionInfos(ListPartitionInfosResponse response) {
return response.getPartitionsInfosList().stream()
.map(
@@ -380,33 +418,6 @@ public class ClientRpcMessageUtils {
return info;
}
- public static void addPbAlterSchemas(
- AlterTableRequest request, List<TableChange> tableChanges) {
- List<PbAddColumn> addColumns = new ArrayList<>();
- List<PbDropColumn> dropColumns = new ArrayList<>();
- List<PbRenameColumn> renameColumns = new ArrayList<>();
- List<PbModifyColumn> modifyColumns = new ArrayList<>();
- for (TableChange tableChange : tableChanges) {
- if (tableChange instanceof TableChange.AddColumn) {
- addColumns.add(toPbAddColumn((TableChange.AddColumn) tableChange));
- } else if (tableChange instanceof TableChange.DropColumn) {
- dropColumns.add(toPbDropColumn((TableChange.DropColumn) tableChange));
- } else if (tableChange instanceof TableChange.RenameColumn) {
- renameColumns.add(toPbRenameColumn((TableChange.RenameColumn) tableChange));
- } else if (tableChange instanceof TableChange.ModifyColumn) {
- modifyColumns.add(toPbModifyColumn((TableChange.ModifyColumn) tableChange));
- } else {
- throw new IllegalArgumentException(
- "Unsupported table change: " + tableChange.getClass());
- }
- }
-
- request.addAllAddColumns(addColumns)
- .addAllDropColumns(dropColumns)
- .addAllRenameColumns(renameColumns)
- .addAllModifyColumns(modifyColumns);
- }
-
public static PbAddColumn toPbAddColumn(TableChange.AddColumn addColumn) {
ColumnPositionType columnPositionType = ColumnPositionType.from(addColumn.getPosition());
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
index f7d7d3ef0..3a282ba92 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/RecordAccumulator.java
@@ -868,8 +868,7 @@ public final class RecordAccumulator {
// Either we have reached a point where there are batches without a sequence (i.e. never
// been drained and are hence in order by default), or the batch at the front of the
// queue has a sequence greater than the incoming batch. This is the right place to
- // add
- // the incoming batch.
+ // add the incoming batch.
deque.addFirst(batch);
// Now we have to re-insert the previously queued batches in the right order.
diff --git a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
index 1b9c79397..e9c285306 100644
--- a/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
+++ b/fluss-client/src/main/java/org/apache/fluss/client/write/Sender.java
@@ -130,7 +130,7 @@ public class Sender implements Runnable {
this.idempotenceManager = idempotenceManager;
this.writerMetricGroup = writerMetricGroup;
- // TODOadd retry logic while send failed. See FLUSS-56364375
+ // TODO add retry logic while send failed. See FLUSS-56364375
}
@VisibleForTesting
@@ -167,7 +167,7 @@ public class Sender implements Runnable {
}
}
- // TODO if force close failed,add logic to abort incomplete batches.
+ // TODO if force close failed, add logic to abort incomplete batches.
LOG.debug("Shutdown of Fluss write sender I/O thread has completed.");
}
@@ -245,7 +245,7 @@ public class Sender implements Runnable {
if (!batches.isEmpty()) {
addToInflightBatches(batches);
- // TODOadd logic for batch expire.
+ // TODO add logic for batch expire.
sendWriteRequests(batches);
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
index 05f196fdd..bd7954284 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/admin/FlussAdminITCase.java
@@ -354,13 +354,43 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
.get())
.hasMessageContaining("Column c1 must be nullable");
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING().copy(false),
+ null,
+ TableChange.ColumnPosition
+ .first())),
+ false)
+ .get())
+ .hasMessageContaining("Unsupported ColumnPositionType: FIRST");
+
+ assertThatThrownBy(
+ () ->
+ admin.alterTable(
+ tablePath,
+ Collections.singletonList(
+ TableChange.addColumn(
+ "c1",
+ DataTypes.STRING().copy(false),
+ null,
+ TableChange.ColumnPosition.after(
+ "name"))),
+ false)
+ .get())
+ .hasMessageContaining("Unsupported ColumnPositionType: AFTER(name)");
+
admin.alterTable(
tablePath,
Collections.singletonList(
TableChange.addColumn(
"c1",
DataTypes.STRING(),
- null,
+ "new column c1",
TableChange.ColumnPosition.last())),
false)
.get();
@@ -380,7 +410,10 @@ class FlussAdminITCase extends ClientToServerITCaseBase {
new Schema.Column(
"age", DataTypes.INT(), "person age", (short) 2),
new Schema.Column(
- "c1", DataTypes.STRING(), null, (short) 3)))
+ "c1",
+ DataTypes.STRING(),
+ "new column c1",
+ (short) 3)))
.build();
SchemaInfo schemaInfo = admin.getTableSchema(tablePath).get();
assertThat(schemaInfo).isEqualTo(new SchemaInfo(expectedSchema, 2));
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
index 527994777..62b5bdc13 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/batch/KvSnapshotBatchScannerITCase.java
@@ -134,7 +134,7 @@ class KvSnapshotBatchScannerITCase extends ClientToServerITCaseBase {
Collections.singletonList(
TableChange.addColumn(
"new_column",
- DataTypes.BIGINT().copy(false).copy(true),
+ DataTypes.BIGINT().copy(true),
null,
TableChange.ColumnPosition.last())),
false)
@@ -148,7 +148,7 @@ class KvSnapshotBatchScannerITCase extends ClientToServerITCaseBase {
.column("name", DataTypes.STRING())
.column("new_column", DataTypes.BIGINT())
.build();
- // put into values with old schema.
+ // put into values with new schema.
List<InternalRow> rows = new ArrayList<>();
for (int i = 10; i < 20; i++) {
InternalRow row =
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
index bfd686dc1..d487a2d46 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/DefaultCompletedFetchTest.java
@@ -29,6 +29,7 @@ import org.apache.fluss.record.FileLogProjection;
import org.apache.fluss.record.FileLogRecords;
import org.apache.fluss.record.LogRecordReadContext;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.rpc.entity.FetchLogResultForBucket;
@@ -265,7 +266,7 @@ public class DefaultCompletedFetchTest {
rowType, DEFAULT_SCHEMA_ID, 0L, 1000L, magic, objects, LogFormat.ARROW));
fileLogRecords.flush();
- FileLogProjection fileLogProjection = new FileLogProjection();
+ FileLogProjection fileLogProjection = new FileLogProjection(new ProjectionPushdownCache());
fileLogProjection.setCurrentProjection(
DATA2_TABLE_ID,
testingSchemaGetter,
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
index 83878a816..99a108e2f 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetchCollectorTest.java
@@ -93,8 +93,7 @@ public class LogFetchCollectorTest {
logFetchBuffer.add(completedFetch);
assertThat(logFetchBuffer.isEmpty()).isFalse();
- // Validate that the completed fetch isn't initialized just because we add it to the
- // buffer.
+ // Validate that the completed fetch isn't initialized just because we add it to the buffer
assertThat(completedFetch.isInitialized()).isFalse();
// Fetch the data and validate that we get all the records we want back.
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
index 03f908733..e5a8c8ff0 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/table/scanner/log/LogFetcherTest.java
@@ -199,6 +199,7 @@ public class LogFetcherTest extends ClientToServerITCaseBase {
scanRecords = records.get(tb0);
assertThat(scanRecords.stream().map(ScanRecord::getRow).collect(Collectors.toList()))
.isEqualTo(expectedRows);
+ newSchemaLogFetcher.close();
}
@Test
diff --git a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
index bc4e53b50..47a9c2414 100644
--- a/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
+++ b/fluss-client/src/test/java/org/apache/fluss/client/write/RecordAccumulatorTest.java
@@ -137,7 +137,6 @@ class RecordAccumulatorTest {
conf = new Configuration();
// init cluster.
cluster = updateCluster(Arrays.asList(bucket1, bucket2, bucket3));
- // todo: 从cluster中获取schema getter
schemaGetter = new TestingSchemaGetter(new SchemaInfo(DATA1_SCHEMA, (short) 1));
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java b/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java
index 14b0b7f96..b577b8633 100644
--- a/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/ApiException.java
@@ -25,21 +25,35 @@ public class ApiException extends FlussRuntimeException {
private static final long serialVersionUID = 1L;
+ private final boolean stackTraceEnabled;
+
+ public ApiException(String message, Throwable cause, boolean stackTraceEnabled) {
+ super(message, cause);
+ this.stackTraceEnabled = stackTraceEnabled;
+ }
+
public ApiException(String message, Throwable cause) {
super(message, cause);
+ this.stackTraceEnabled = false;
}
public ApiException(String message) {
super(message);
+ this.stackTraceEnabled = false;
}
public ApiException(Throwable cause) {
super(cause);
+ this.stackTraceEnabled = false;
}
/* avoid the expensive and useless stack trace for api exceptions */
@Override
public Throwable fillInStackTrace() {
- return this;
+ if (stackTraceEnabled) {
+ return super.fillInStackTrace();
+ } else {
+ return this;
+ }
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
index 8c9446682..77983e441 100644
--- a/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
+++ b/fluss-common/src/main/java/org/apache/fluss/exception/SchemaChangeException.java
@@ -27,4 +27,8 @@ public class SchemaChangeException extends ApiException {
public SchemaChangeException(String message) {
this(message, null);
}
+
+ public SchemaChangeException(String message, boolean stackTraceEnabled) {
+ super(message, null, stackTraceEnabled);
+ }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
index 367013363..0ae0c8b97 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/Schema.java
@@ -132,6 +132,7 @@ public final class Schema implements Serializable {
return columns.stream().map(Column::getName).collect(Collectors.toList());
}
+ /** Returns all column ids for top-level columns, the nested field ids are not included. */
public List<Integer> getColumnIds() {
return columns.stream().map(Column::getColumnId).collect(Collectors.toList());
}
@@ -154,6 +155,7 @@ public final class Schema implements Serializable {
return keyIndexes;
}
+ /** Returns the highest field ID in this schema. */
public int getHighestFieldId() {
return highestFieldId;
}
@@ -249,24 +251,21 @@ public final class Schema implements Serializable {
/** Adopts all columns from the given list. */
public Builder fromColumns(List<Column> inputColumns) {
- boolean nonMatchColumnId =
+ boolean nonSetColumnId =
inputColumns.stream()
.noneMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
- boolean allMatchColumnId =
+ boolean allSetColumnId =
inputColumns.stream()
.allMatch(column -> column.columnId != Column.UNKNOWN_COLUMN_ID);
checkState(
- nonMatchColumnId || allMatchColumnId,
+ nonSetColumnId || allSetColumnId,
"All columns must have columnId or none of them must have columnId.");
- if (allMatchColumnId) {
+ if (allSetColumnId) {
columns.addAll(inputColumns);
highestFieldId =
new AtomicInteger(
- inputColumns.stream()
- .mapToInt(Column::getColumnId)
- .max()
- .orElse(-1));
+ columns.stream().mapToInt(Column::getColumnId).max().orElse(-1));
} else {
// if all columnId is not set, this maybe from old version schema. Just use its
// position as columnId.
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
index 8cdc14670..e5b8e6699 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/SchemaGetter.java
@@ -17,11 +17,59 @@
package org.apache.fluss.metadata;
-/** A schema getter interface. */
+import org.apache.fluss.exception.SchemaNotExistException;
+
+import java.util.concurrent.CompletableFuture;
+
+/**
+ * Provides retrieval and lifecycle management for {@link Schema} instances identified by schema
+ * ids.
+ *
+ * <p>Implementations may expose both blocking and non-blocking retrieval paths. Clients can obtain
+ * a schema synchronously via {@link #getSchema(int)} or asynchronously via {@link
+ * #getSchemaInfoAsync(int)}.
+ */
public interface SchemaGetter {
+
+ /**
+ * Synchronously obtains the {@link Schema} associated with the given {@code schemaId}.
+ *
+ * <p>This method may block while fetching or deserializing the schema. The implementation may
+ * perform an RPC request to fetching schema metadata from the Cluster.
+ *
+ * @param schemaId numeric identifier of the requested schema
+ * @return the resolved {@code Schema}, or {@code null} if no schema is available for the id
+ * @throws SchemaNotExistException if the requested schema id is not exists.
+ */
Schema getSchema(int schemaId);
+ /**
+ * Asynchronously obtains the {@code Schema} associated with the given {@code schemaId}.
+ *
+ * <p>The returned {@link CompletableFuture} completes with the resolved {@link SchemaInfo} or
+ * completes exceptionally if an error occurs.
+ *
+ * <p>The following exceptions can be anticipated when calling {@code get()} on returned future.
+ *
+ * <ul>
+ * <li>{@link SchemaNotExistException} if the requested schema id is not exists.
+ * </ul>
+ *
+ * @param schemaId numeric identifier of the requested schema
+ * @return a {@link CompletableFuture} that will be completed with the {@link SchemaInfo} or
+ * completed exceptionally
+ */
+ CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId);
+
+ /** Returns metadata describing the latest known schema. */
SchemaInfo getLatestSchemaInfo();
+ /**
+ * Releases any resources held by this {@code SchemaGetter}.
+ *
+ * <p>After calling this method, the instance may no longer be usable. Implementations should
+ * free caches, close network connections, cancel outstanding requests, and stop background
+ * threads as appropriate.
+ */
void release();
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
index 36162f30c..e012a38f6 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableChange.java
@@ -402,7 +402,7 @@ public interface TableChange {
@Override
public String toString() {
- return String.format("AFTER %s", columnName);
+ return String.format("AFTER(%s)", columnName);
}
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
index af293f120..eba41e078 100644
--- a/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
+++ b/fluss-common/src/main/java/org/apache/fluss/metadata/TableInfo.java
@@ -137,6 +137,11 @@ public final class TableInfo {
return schema;
}
+ /** Returns the schema info of the table, including schema and schema id. */
+ public SchemaInfo getSchemaInfo() {
+ return new SchemaInfo(schema, schemaId);
+ }
+
/**
* Returns the row type of the table. The row type is the schema of the table, which defines the
* columns and types of the table.
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java b/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java
new file mode 100644
index 000000000..13906347d
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/record/BinaryValue.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.record;
+
+import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.encode.ValueEncoder;
+
+import java.util.Objects;
+
+/** A value of key-value pair that contains schema id and binary row. */
+public class BinaryValue {
+
+ public final short schemaId;
+ public final BinaryRow row;
+
+ public BinaryValue(short schemaId, BinaryRow row) {
+ this.schemaId = schemaId;
+ this.row = row;
+ }
+
+ /**
+ * Encode the value (consisted of {@code row} with a {@code schemaId}) to a byte array value to
+ * be expected persisted to kv store.
+ */
+ public byte[] encodeValue() {
+ return ValueEncoder.encodeValue(schemaId, row);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ BinaryValue that = (BinaryValue) o;
+ return schemaId == that.schemaId && Objects.equals(row, that.row);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(schemaId, row);
+ }
+
+ @Override
+ public String toString() {
+ return "BinaryValue{" + "schemaId=" + schemaId + ", row=" + row + '}';
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
index b6ed40eeb..8bf27e10e 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/DefaultLogRecordBatch.java
@@ -21,7 +21,6 @@ import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.exception.CorruptMessageException;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.LogFormat;
-import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.row.arrow.ArrowReader;
import org.apache.fluss.row.columnar.ColumnarRow;
@@ -34,13 +33,13 @@ import org.apache.fluss.utils.CloseableIterator;
import org.apache.fluss.utils.MurmurHashUtils;
import org.apache.fluss.utils.crc.Crc32C;
+import javax.annotation.Nullable;
+
import java.nio.ByteBuffer;
import java.util.NoSuchElementException;
-import java.util.function.Function;
import static org.apache.fluss.record.LogRecordBatchFormat.BASE_OFFSET_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.COMMIT_TIMESTAMP_OFFSET;
-import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_LENGTH;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_MAGIC_VALUE_V1;
import static org.apache.fluss.record.LogRecordBatchFormat.LOG_OVERHEAD;
@@ -213,7 +212,6 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
@Override
- @SuppressWarnings("unchecked")
public CloseableIterator<LogRecord> records(ReadContext context) {
if (getRecordCount() == 0) {
return CloseableIterator.emptyIterator();
@@ -223,26 +221,18 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
long timestamp = commitTimestamp();
LogFormat logFormat = context.getLogFormat();
RowType rowType = context.getRowType(schemaId);
- Function<InternalRow, InternalRow> outputRowProcessor =
- (row) -> {
- ProjectedRow projectedRow = context.getOutputProjectedRow(schemaId);
- if (projectedRow != null) {
- return projectedRow.replaceRow(row);
- } else {
- return row;
- }
- };
switch (logFormat) {
case ARROW:
return columnRecordIterator(
rowType,
- outputRowProcessor,
+ context.getOutputProjectedRow(schemaId),
context.getVectorSchemaRoot(schemaId),
context.getBufferAllocator(),
timestamp);
case INDEXED:
- return rowRecordIterator(rowType, outputRowProcessor, timestamp);
+ return rowRecordIterator(
+ rowType, context.getOutputProjectedRow(schemaId), timestamp);
default:
throw new IllegalArgumentException("Unsupported log format: " + logFormat);
}
@@ -270,9 +260,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
}
private CloseableIterator<LogRecord> rowRecordIterator(
- RowType rowType,
- Function<InternalRow, InternalRow> outputRowProcessor,
- long timestamp) {
+ RowType rowType, @Nullable ProjectedRow outputProjection, long timestamp) {
DataType[] fieldTypes = rowType.getChildren().toArray(new DataType[0]);
return new LogRecordIterator() {
int position = DefaultLogRecordBatch.this.position + recordBatchHeaderSize(magic);
@@ -281,16 +269,20 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
@Override
protected LogRecord readNext(long baseOffset) {
IndexedLogRecord logRecord =
- ProjectedIndexedLogRecord.readFrom(
- segment,
- position,
- baseOffset + rowId,
- timestamp,
- fieldTypes,
- outputRowProcessor);
+ IndexedLogRecord.readFrom(
+ segment, position, baseOffset + rowId, timestamp, fieldTypes);
rowId++;
position += logRecord.getSizeInBytes();
- return logRecord;
+ if (outputProjection == null) {
+ return logRecord;
+ } else {
+ // apply projection
+ return new GenericRecord(
+ logRecord.logOffset(),
+ logRecord.timestamp(),
+ logRecord.getChangeType(),
+ outputProjection.replaceRow(logRecord.getRow()));
+ }
}
@Override
@@ -305,7 +297,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
private CloseableIterator<LogRecord> columnRecordIterator(
RowType rowType,
- Function<InternalRow, InternalRow> outputRowProcessor,
+ @Nullable ProjectedRow outputProjection,
VectorSchemaRoot root,
BufferAllocator allocator,
long timestamp) {
@@ -319,7 +311,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputProjection) {
@Override
protected ChangeType getChangeType(int rowId) {
return ChangeType.APPEND_ONLY;
@@ -337,7 +329,7 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
ArrowReader reader =
ArrowUtils.createArrowReader(
segment, arrowOffset, arrowLength, root, allocator, rowType);
- return new ArrowLogRecordIterator(reader, timestamp, outputRowProcessor) {
+ return new ArrowLogRecordIterator(reader, timestamp, outputProjection) {
@Override
protected ChangeType getChangeType(int rowId) {
return changeTypeVector.getChangeType(rowId);
@@ -351,15 +343,13 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
private final ArrowReader reader;
private final long timestamp;
private int rowId = 0;
- private final Function<InternalRow, InternalRow> outputRowProcessor;
+ @Nullable private final ProjectedRow outputProjection;
private ArrowLogRecordIterator(
- ArrowReader reader,
- long timestamp,
- Function<InternalRow, InternalRow> outputRowProcessor) {
+ ArrowReader reader, long timestamp, @Nullable ProjectedRow outputProjection) {
this.reader = reader;
this.timestamp = timestamp;
- this.outputRowProcessor = outputRowProcessor;
+ this.outputProjection = outputProjection;
}
protected abstract ChangeType getChangeType(int rowId);
@@ -377,7 +367,9 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
baseOffset + rowId,
timestamp,
getChangeType(rowId),
- outputRowProcessor.apply(originalRow));
+ outputProjection == null
+ ? originalRow
+ : outputProjection.replaceRow(originalRow));
rowId++;
return record;
}
@@ -447,38 +439,4 @@ public class DefaultLogRecordBatch implements LogRecordBatch {
throw new UnsupportedOperationException();
}
}
-
- private static class ProjectedIndexedLogRecord extends IndexedLogRecord {
- private final Function<InternalRow, InternalRow> outputRowProcessor;
-
- ProjectedIndexedLogRecord(
- long logOffset,
- long timestamp,
- DataType[] fieldTypes,
- Function<InternalRow, InternalRow> outputRowProcessor) {
- super(logOffset, timestamp, fieldTypes);
- this.outputRowProcessor = outputRowProcessor;
- }
-
- public static IndexedLogRecord readFrom(
- MemorySegment segment,
- int position,
- long logOffset,
- long logTimestamp,
- DataType[] colTypes,
- Function<InternalRow, InternalRow> outputRowProcessor) {
- int sizeInBytes = segment.getInt(position);
- ProjectedIndexedLogRecord logRecord =
- new ProjectedIndexedLogRecord(
- logOffset, logTimestamp, colTypes, outputRowProcessor);
- logRecord.pointTo(segment, position, sizeInBytes + LENGTH_LENGTH);
- return logRecord;
- }
-
- @Override
- public InternalRow getRow() {
- InternalRow row = super.getRow();
- return outputRowProcessor.apply(row);
- }
- }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
index 11c50bb3d..9f227adb1 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/FileLogProjection.java
@@ -54,7 +54,6 @@ import java.util.BitSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import java.util.Objects;
import static org.apache.fluss.record.DefaultLogRecordBatch.APPEND_ONLY_FLAG_MASK;
import static org.apache.fluss.record.LogRecordBatchFormat.LENGTH_OFFSET;
@@ -71,7 +70,6 @@ import static org.apache.fluss.record.LogRecordBatchFormat.recordsCountOffset;
import static org.apache.fluss.record.LogRecordBatchFormat.schemaIdOffset;
import static org.apache.fluss.utils.FileUtils.readFully;
import static org.apache.fluss.utils.FileUtils.readFullyOrFail;
-import static org.apache.fluss.utils.Preconditions.checkNotNull;
import static org.apache.fluss.utils.Preconditions.checkState;
/** Column projection util on Arrow format {@link FileLogRecords}. */
@@ -85,8 +83,8 @@ public class FileLogProjection {
private static final int ARROW_HEADER_SIZE =
ARROW_IPC_CONTINUATION_LENGTH + ARROW_IPC_METADATA_SIZE_LENGTH;
- final Map<ProjectionKey, ProjectionInfo> projectionsCache = new HashMap<>();
- ProjectionInfo currentProjection;
+ // the projection cache shared in the TabletServer
+ private final ProjectionPushdownCache projectionsCache;
// shared resources for multiple projections
private final ByteArrayOutputStream outputStream;
@@ -103,9 +101,10 @@ public class FileLogProjection {
private SchemaGetter schemaGetter;
private long tableId;
private ArrowCompressionInfo compressionInfo;
- private int[] selectedFieldPositions;
+ private int[] selectedFieldIds;
- public FileLogProjection() {
+ public FileLogProjection(ProjectionPushdownCache projectionsCache) {
+ this.projectionsCache = projectionsCache;
this.outputStream = new ByteArrayOutputStream();
this.writeChannel = new WriteChannel(Channels.newChannel(outputStream));
// fluss use little endian for encoding log records batch
@@ -122,14 +121,7 @@ public class FileLogProjection {
this.tableId = tableId;
this.schemaGetter = schemaGetter;
this.compressionInfo = compressionInfo;
-
- // Currently, only add last column is supported.Thus selectedFieldPositions is always same
- // from same selectedFieldIds.
- // TODO: if support drop column or add column in middle, this selectedFieldPositions should
- // be re-calculated for each schema.
- this.selectedFieldPositions =
- selectedFieldPositions(
- schemaGetter.getLatestSchemaInfo().getSchema(), selectedFieldIds);
+ this.selectedFieldIds = selectedFieldIds;
}
/**
@@ -144,6 +136,8 @@ public class FileLogProjection {
MultiBytesView.Builder builder = MultiBytesView.builder();
int position = start;
+ ProjectionInfo currentProjection = null;
+ short prevSchemaId = -1;
// The condition is an optimization to avoid read log header when there is no enough bytes,
// So we use V0 header size here for a conservative judgment. In the end, the condition
// of (position >= end - recordBatchHeaderSize) will ensure the final correctness.
@@ -162,8 +156,12 @@ public class FileLogProjection {
int recordBatchHeaderSize = recordBatchHeaderSize(magic);
int batchSizeInBytes = LOG_OVERHEAD + logHeaderBuffer.getInt(LENGTH_OFFSET);
short schemaId = logHeaderBuffer.getShort(schemaIdOffset(magic));
- setCurrentSchema(schemaId);
- checkNotNull(currentProjection, "There is no projection registered yet.");
+
+ // reuse projection in the current log file
+ if (currentProjection == null || prevSchemaId != schemaId) {
+ prevSchemaId = schemaId;
+ currentProjection = getOrCreateProjectionInfo(schemaId);
+ }
if (position > end - batchSizeInBytes) {
// the remaining bytes in the file are not enough to read a full batch
@@ -405,20 +403,22 @@ public class FileLogProjection {
return logHeaderBuffer;
}
- private void setCurrentSchema(short schemaId) {
+ private ProjectionInfo getOrCreateProjectionInfo(short schemaId) {
+ ProjectionInfo cachedProjection =
+ projectionsCache.getProjectionInfo(tableId, schemaId, selectedFieldIds);
+ if (cachedProjection == null) {
+ cachedProjection = createProjectionInfo(schemaId, selectedFieldIds);
+ projectionsCache.setProjectionInfo(
+ tableId, schemaId, selectedFieldIds, cachedProjection);
+ }
+ return cachedProjection;
+ }
+
+ private ProjectionInfo createProjectionInfo(short schemaId, int[] selectedFieldIds) {
org.apache.fluss.metadata.Schema schema = schemaGetter.getSchema(schemaId);
+ int[] selectedFieldPositions =
+ selectedFieldPositions(schemaGetter.getSchema(schemaId), selectedFieldIds);
RowType rowType = schema.getRowType();
- ProjectionKey projectionKey = new ProjectionKey(tableId, schemaId);
- if (projectionsCache.containsKey(projectionKey)) {
- // the schema and projection should identical for the same table id.
- currentProjection = projectionsCache.get(projectionKey);
- if (!Arrays.equals(currentProjection.selectedFieldPositions, selectedFieldPositions)
- || !currentProjection.schema.equals(rowType)) {
- throw new InvalidColumnProjectionException(
- "The schema and projection should be identical for the same table id.");
- }
- return;
- }
// initialize the projection util information
Schema arrowSchema = ArrowUtils.toArrowSchema(rowType);
@@ -451,16 +451,13 @@ public class FileLogProjection {
CompressionUtil.createBodyCompression(compressionInfo.createCompressionCodec());
int metadataLength =
ArrowUtils.estimateArrowMetadataLength(projectedArrowSchema, bodyCompression);
- currentProjection =
- new ProjectionInfo(
- nodesProjection,
- buffersProjection,
- bufferIndex,
- rowType,
- metadataLength,
- bodyCompression,
- selectedFieldPositions);
- projectionsCache.put(projectionKey, currentProjection);
+ return new ProjectionInfo(
+ nodesProjection,
+ buffersProjection,
+ bufferIndex,
+ metadataLength,
+ bodyCompression,
+ selectedFieldPositions);
}
int[] selectedFieldPositions(org.apache.fluss.metadata.Schema schema, int[] projectedFields) {
@@ -478,7 +475,8 @@ public class FileLogProjection {
if (position == null) {
throw new InvalidColumnProjectionException(
String.format(
- "Projected field id %s is not contains in %s", fieldId, columnIds));
+ "Projected field id %s is not contained in %s",
+ fieldId, columnIds));
}
selectedFieldPositions[i] = position;
@@ -493,35 +491,11 @@ public class FileLogProjection {
return selectedFieldPositions;
}
- static final class ProjectionKey {
- private final long tableId;
- private final short schemaId;
-
- ProjectionKey(long tableId, short schemaId) {
- this.tableId = tableId;
- this.schemaId = schemaId;
- }
-
- @Override
- public boolean equals(Object o) {
- if (!(o instanceof ProjectionKey)) {
- return false;
- }
- ProjectionKey that = (ProjectionKey) o;
- return tableId == that.tableId && schemaId == that.schemaId;
- }
-
- @Override
- public int hashCode() {
- return Objects.hash(tableId, schemaId);
- }
- }
-
- static final class ProjectionInfo {
+ /** Projection pushdown information for a specific schema and selected fields. */
+ public static final class ProjectionInfo {
final BitSet nodesProjection;
final BitSet buffersProjection;
final int bufferCount;
- final RowType schema;
final int arrowMetadataLength;
final ArrowBodyCompression bodyCompression;
final int[] selectedFieldPositions;
@@ -530,14 +504,12 @@ public class FileLogProjection {
BitSet nodesProjection,
BitSet buffersProjection,
int bufferCount,
- RowType schema,
int arrowMetadataLength,
ArrowBodyCompression bodyCompression,
int[] selectedFieldPositions) {
this.nodesProjection = nodesProjection;
this.buffersProjection = buffersProjection;
this.bufferCount = bufferCount;
- this.schema = schema;
this.arrowMetadataLength = arrowMetadataLength;
this.bodyCompression = bodyCompression;
this.selectedFieldPositions = selectedFieldPositions;
@@ -545,7 +517,7 @@ public class FileLogProjection {
}
/** Metadata of a projected arrow record batch. */
- public static final class ProjectedArrowBatch {
+ static final class ProjectedArrowBatch {
/** Number of records. */
final long numRecords;
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
index 50ec7ee66..5ec5c7c77 100644
--- a/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
+++ b/fluss-common/src/main/java/org/apache/fluss/record/LogRecordReadContext.java
@@ -31,18 +31,20 @@ import org.apache.fluss.shaded.arrow.org.apache.arrow.vector.VectorSchemaRoot;
import org.apache.fluss.types.DataType;
import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.ArrowUtils;
+import org.apache.fluss.utils.MapUtils;
import org.apache.fluss.utils.Projection;
import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
-import java.util.HashMap;
import java.util.List;
-import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.stream.IntStream;
import static org.apache.fluss.utils.Preconditions.checkNotNull;
/** A simple implementation for {@link LogRecordBatch.ReadContext}. */
+@ThreadSafe
public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoCloseable {
// the log format of the table
@@ -50,9 +52,7 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
// the schema of the date read form server or remote. (which is projected in the server side)
private final RowType dataRowType;
// the static schemaId of the table, should support dynamic schema evolution in the future
- private final int expectedSchemaId;
- // the Arrow vector schema root of the table, should be null if not ARROW log format
- @Nullable private final VectorSchemaRoot vectorSchemaRoot;
+ private final int targetSchemaId;
// the Arrow memory buffer allocator for the table, should be null if not ARROW log format
@Nullable private final BufferAllocator bufferAllocator;
// the final selected fields of the read data
@@ -60,7 +60,8 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
// whether the projection is push downed to the server side and the returned data is pruned.
private final boolean projectionPushDowned;
private final SchemaGetter schemaGetter;
- private final Map<Integer, VectorSchemaRoot> vectorSchemaRootMap;
+ private final ConcurrentHashMap<Integer, VectorSchemaRoot> vectorSchemaRootMap =
+ MapUtils.newConcurrentHashMap();
public static LogRecordReadContext createReadContext(
TableInfo tableInfo,
@@ -115,14 +116,11 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
SchemaGetter schemaGetter) {
// TODO: use a more reasonable memory limit
BufferAllocator allocator = new RootAllocator(Long.MAX_VALUE);
- VectorSchemaRoot vectorRoot =
- VectorSchemaRoot.create(ArrowUtils.toArrowSchema(dataRowType), allocator);
FieldGetter[] fieldGetters = buildProjectedFieldGetters(dataRowType, selectedFields);
return new LogRecordReadContext(
LogFormat.ARROW,
dataRowType,
schemaId,
- vectorRoot,
allocator,
fieldGetters,
projectionPushDowned,
@@ -130,10 +128,12 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
}
/**
- * Creates a LogRecordReadContext for INDEXED log format.
+ * Creates a LogRecordReadContext for ARROW log format, that underlying Arrow resources are not
+ * reused.
*
* @param rowType the schema of the table
* @param schemaId the schemaId of the table
+ * @param schemaGetter the schema getter of to get schema by schemaId
*/
@VisibleForTesting
public static LogRecordReadContext createArrowReadContext(
@@ -142,6 +142,13 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
return createArrowReadContext(rowType, schemaId, selectedFields, false, schemaGetter);
}
+ /**
+ * Creates a LogRecordReadContext for INDEXED log format.
+ *
+ * @param rowType the schema of the table
+ * @param schemaId the schemaId of the table
+ * @param schemaGetter the schema getter of to get schema by schemaId
+ */
@VisibleForTesting
public static LogRecordReadContext createIndexedReadContext(
RowType rowType, int schemaId, SchemaGetter schemaGetter) {
@@ -155,40 +162,31 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
* @param rowType the schema of the read data
* @param schemaId the schemaId of the table
* @param selectedFields the final selected fields of the read data
+ * @param schemaGetter the schema getter of to get schema by schemaId
*/
public static LogRecordReadContext createIndexedReadContext(
RowType rowType, int schemaId, int[] selectedFields, SchemaGetter schemaGetter) {
FieldGetter[] fieldGetters = buildProjectedFieldGetters(rowType, selectedFields);
// for INDEXED log format, the projection is NEVER push downed to the server side
return new LogRecordReadContext(
- LogFormat.INDEXED,
- rowType,
- schemaId,
- null,
- null,
- fieldGetters,
- false,
- schemaGetter);
+ LogFormat.INDEXED, rowType, schemaId, null, fieldGetters, false, schemaGetter);
}
private LogRecordReadContext(
LogFormat logFormat,
RowType targetDataRowType,
- int expectedSchemaId,
- VectorSchemaRoot vectorSchemaRoot,
+ int targetSchemaId,
BufferAllocator bufferAllocator,
FieldGetter[] selectedFieldGetters,
boolean projectionPushDowned,
SchemaGetter schemaGetter) {
this.logFormat = logFormat;
this.dataRowType = targetDataRowType;
- this.expectedSchemaId = expectedSchemaId;
- this.vectorSchemaRoot = vectorSchemaRoot;
+ this.targetSchemaId = targetSchemaId;
this.bufferAllocator = bufferAllocator;
this.selectedFieldGetters = selectedFieldGetters;
this.projectionPushDowned = projectionPushDowned;
this.schemaGetter = schemaGetter;
- this.vectorSchemaRootMap = new HashMap<>();
}
@Override
@@ -223,11 +221,6 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
"Only Arrow log format provides vector schema root.");
}
- if (schemaId == this.expectedSchemaId) {
- checkNotNull(vectorSchemaRoot, "The vector schema root is not available.");
- return vectorSchemaRoot;
- }
-
RowType rowType = getRowType(schemaId);
return vectorSchemaRootMap.computeIfAbsent(
schemaId,
@@ -251,15 +244,13 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
if (isSameRowType(schemaId)) {
return null;
}
+ // TODO: should we cache the projection?
Schema originSchema = schemaGetter.getSchema(schemaId);
- Schema expectedSchema = schemaGetter.getSchema(expectedSchemaId);
+ Schema expectedSchema = schemaGetter.getSchema(targetSchemaId);
return ProjectedRow.from(originSchema, expectedSchema);
}
public void close() {
- if (vectorSchemaRoot != null) {
- vectorSchemaRoot.close();
- }
vectorSchemaRootMap.values().forEach(VectorSchemaRoot::close);
if (bufferAllocator != null) {
bufferAllocator.close();
@@ -267,7 +258,7 @@ public class LogRecordReadContext implements LogRecordBatch.ReadContext, AutoClo
}
private boolean isSameRowType(int schemaId) {
- return expectedSchemaId == schemaId || isProjectionPushDowned();
+ return targetSchemaId == schemaId || isProjectionPushDowned();
}
private static FieldGetter[] buildProjectedFieldGetters(RowType rowType, int[] selectedFields) {
diff --git a/fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java b/fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java
new file mode 100644
index 000000000..a2d3da1ae
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/record/ProjectionPushdownCache.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.record;
+
+import org.apache.fluss.record.FileLogProjection.ProjectionInfo;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.Cache;
+import org.apache.fluss.shaded.guava32.com.google.common.cache.CacheBuilder;
+
+import javax.annotation.Nullable;
+import javax.annotation.concurrent.ThreadSafe;
+
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Objects;
+
+/**
+ * A cache for projection pushdown information. The cache key is composed of table id, schema id,
+ * and selected column ids. The cache is designed to be shared across different tables and schemas.
+ */
+@ThreadSafe
+public class ProjectionPushdownCache {
+
+ final Cache<ProjectionKey, ProjectionInfo> projectionCache;
+
+ public ProjectionPushdownCache() {
+ // currently, the cache is shared at TabletServer level, so we give a large max size, but
+ // give a short expiration time.
+ // TODO: make the cache parameter configurable
+ this.projectionCache =
+ CacheBuilder.newBuilder()
+ .maximumSize(1000)
+ .expireAfterAccess(Duration.ofMinutes(3))
+ .build();
+ }
+
+ @Nullable
+ public ProjectionInfo getProjectionInfo(long tableId, short schemaId, int[] selectedColumnIds) {
+ ProjectionKey key = new ProjectionKey(tableId, schemaId, selectedColumnIds);
+ return projectionCache.getIfPresent(key);
+ }
+
+ public void setProjectionInfo(
+ long tableId, short schemaId, int[] selectedColumnIds, ProjectionInfo projectionInfo) {
+ ProjectionKey key = new ProjectionKey(tableId, schemaId, selectedColumnIds);
+ projectionCache.put(key, projectionInfo);
+ }
+
+ static final class ProjectionKey {
+ private final long tableId;
+ private final short schemaId;
+ private final int[] selectedColumnIds;
+
+ ProjectionKey(long tableId, short schemaId, int[] selectedColumnIds) {
+ this.tableId = tableId;
+ this.schemaId = schemaId;
+ this.selectedColumnIds = selectedColumnIds;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (!(o instanceof ProjectionKey)) {
+ return false;
+ }
+ ProjectionKey that = (ProjectionKey) o;
+ return tableId == that.tableId
+ && schemaId == that.schemaId
+ && Arrays.equals(selectedColumnIds, that.selectedColumnIds);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(tableId, schemaId, Arrays.hashCode(selectedColumnIds));
+ }
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java b/fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java
new file mode 100644
index 000000000..db431f2ac
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/row/PaddingRow.java
@@ -0,0 +1,133 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.row;
+
+/**
+ * An {@link InternalRow} that pads another {@link InternalRow} with nulls up to a target field
+ * count.
+ */
+public class PaddingRow implements InternalRow {
+
+ private final int targetFieldCount;
+ private InternalRow row;
+
+ public PaddingRow(int targetFieldCount) {
+ this.targetFieldCount = targetFieldCount;
+ }
+
+ /**
+ * Replaces the underlying {@link InternalRow} backing this {@link PaddingRow}.
+ *
+ * <p>This method replaces the row data in place and does not return a new object. This is done
+ * for performance reasons.
+ */
+ public PaddingRow replaceRow(InternalRow row) {
+ this.row = row;
+ return this;
+ }
+
+ @Override
+ public int getFieldCount() {
+ return targetFieldCount;
+ }
+
+ @Override
+ public boolean isNullAt(int pos) {
+ if (row.getFieldCount() > pos) {
+ return row.isNullAt(pos);
+ } else {
+ // padding null if pos exceeds the underlying row's field count
+ return true;
+ }
+ }
+
+ @Override
+ public boolean getBoolean(int pos) {
+ return row.getBoolean(pos);
+ }
+
+ @Override
+ public byte getByte(int pos) {
+ return row.getByte(pos);
+ }
+
+ @Override
+ public short getShort(int pos) {
+ return row.getShort(pos);
+ }
+
+ @Override
+ public int getInt(int pos) {
+ return row.getInt(pos);
+ }
+
+ @Override
+ public long getLong(int pos) {
+ return row.getLong(pos);
+ }
+
+ @Override
+ public float getFloat(int pos) {
+ return row.getFloat(pos);
+ }
+
+ @Override
+ public double getDouble(int pos) {
+ return row.getDouble(pos);
+ }
+
+ @Override
+ public BinaryString getChar(int pos, int length) {
+ return row.getChar(pos, length);
+ }
+
+ @Override
+ public BinaryString getString(int pos) {
+ return row.getString(pos);
+ }
+
+ @Override
+ public Decimal getDecimal(int pos, int precision, int scale) {
+ return row.getDecimal(pos, precision, scale);
+ }
+
+ @Override
+ public TimestampNtz getTimestampNtz(int pos, int precision) {
+ return row.getTimestampNtz(pos, precision);
+ }
+
+ @Override
+ public TimestampLtz getTimestampLtz(int pos, int precision) {
+ return row.getTimestampLtz(pos, precision);
+ }
+
+ @Override
+ public byte[] getBinary(int pos, int length) {
+ return row.getBinary(pos, length);
+ }
+
+ @Override
+ public byte[] getBytes(int pos) {
+ return row.getBytes(pos);
+ }
+
+ @Override
+ public InternalArray getArray(int pos) {
+ return row.getArray(pos);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
index 6b4c516d1..d793f342a 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/ProjectedRow.java
@@ -36,11 +36,11 @@ import static org.apache.fluss.utils.SchemaUtil.getIndexMapping;
public class ProjectedRow implements InternalRow {
public static final int UNEXIST_MAPPING = -1;
- protected final int[] indexMapping;
+ private final int[] indexMapping;
- protected InternalRow row;
+ private InternalRow row;
- protected ProjectedRow(int[] indexMapping) {
+ private ProjectedRow(int[] indexMapping) {
this.indexMapping = indexMapping;
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java
new file mode 100644
index 000000000..d809a095f
--- /dev/null
+++ b/fluss-common/src/main/java/org/apache/fluss/row/decode/FixedSchemaDecoder.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.row.decode;
+
+import org.apache.fluss.memory.MemorySegment;
+import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.record.ValueRecord;
+import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.types.DataType;
+import org.apache.fluss.utils.SchemaUtil;
+
+import static org.apache.fluss.row.encode.ValueEncoder.SCHEMA_ID_LENGTH;
+
+/**
+ * A decoder that deserializes raw byte arrays of {@link ValueRecord} with dynamic or
+ * self-describing schemas into structured {@link InternalRow} objects conforming to a predefined,
+ * fixed output schema.
+ */
+public class FixedSchemaDecoder {
+
+ /** The underlying row decoder used to decode the source schema rows. */
+ private final RowDecoder rowDecoder;
+
+ /**
+ * Schema projection mapping from source schema id to target schema field index mapping. The
+ * projection mapping is an array where the index is the target schema field index and the value
+ * is the source schema field index.
+ */
+ private final int[] fieldIdMapping;
+
+ /** Indicates whether there is no projection between source schema and target schema. */
+ private final boolean noProjection;
+
+ public FixedSchemaDecoder(KvFormat kvFormat, Schema sourceSchema, Schema targetSchema) {
+ this.rowDecoder =
+ RowDecoder.create(
+ kvFormat, sourceSchema.getRowType().getChildren().toArray(new DataType[0]));
+ this.fieldIdMapping = SchemaUtil.getIndexMapping(sourceSchema, targetSchema);
+ this.noProjection = false;
+ }
+
+ /**
+ * Creates a FixedSchemaDecoder without projection, i.e., the source schema is the same as the
+ * target schema.
+ */
+ public FixedSchemaDecoder(KvFormat kvFormat, Schema schema) {
+ this.rowDecoder =
+ RowDecoder.create(
+ kvFormat, schema.getRowType().getChildren().toArray(new DataType[0]));
+ this.fieldIdMapping = null;
+ this.noProjection = true;
+ }
+
+ /**
+ * Decode the bytes in the memory segment to {@link InternalRow} that adheres to the fixed
+ * {@code targetSchema}.
+ *
+ * @param segment the memory segment to read.
+ * @param offset the offset in the memory segment to read from.
+ * @param sizeInBytes the total size in bytes to read.
+ */
+ public InternalRow decode(MemorySegment segment, int offset, int sizeInBytes) {
+ if (noProjection) {
+ return rowDecoder.decode(segment, offset, sizeInBytes);
+ } else {
+ InternalRow sourceRow = rowDecoder.decode(segment, offset, sizeInBytes);
+ return ProjectedRow.from(fieldIdMapping).replaceRow(sourceRow);
+ }
+ }
+
+ /**
+ * Decode the value memory segment (in {@link ValueRecord} format) to {@link InternalRow} that
+ * adheres to the fixed {@code targetSchema}.
+ */
+ public InternalRow decode(MemorySegment valueSegment) {
+ return decode(valueSegment, SCHEMA_ID_LENGTH, valueSegment.size() - SCHEMA_ID_LENGTH);
+ }
+}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java
index cb3fafa6a..e257d1998 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/decode/RowDecoder.java
@@ -20,13 +20,14 @@ package org.apache.fluss.row.decode;
import org.apache.fluss.annotation.PublicEvolving;
import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.KvFormat;
+import org.apache.fluss.record.ValueRecord;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.DataType;
/**
- * A decoder to read {@link BinaryRow binary format InternalRow} from a byte array or memory
- * segment.
+ * A decoder to read {@link BinaryRow binary format InternalRow} from a byte array or memory segment
+ * of a value record {@link ValueRecord}.
*
* @since 0.2
*/
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
index e0252ae11..794e6ad6b 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueDecoder.java
@@ -21,6 +21,7 @@ import org.apache.fluss.memory.MemorySegment;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
import org.apache.fluss.metadata.SchemaGetter;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.BinaryRow;
import org.apache.fluss.row.decode.RowDecoder;
import org.apache.fluss.types.DataType;
@@ -47,7 +48,7 @@ public class ValueDecoder {
}
/** Decode the value bytes and return the schema id and the row encoded in the value bytes. */
- public Value decodeValue(byte[] valueBytes) {
+ public BinaryValue decodeValue(byte[] valueBytes) {
MemorySegment memorySegment = MemorySegment.wrap(valueBytes);
short schemaId = memorySegment.getShort(0);
@@ -64,17 +65,6 @@ public class ValueDecoder {
BinaryRow row =
rowDecoder.decode(
memorySegment, SCHEMA_ID_LENGTH, valueBytes.length - SCHEMA_ID_LENGTH);
- return new Value(schemaId, row);
- }
-
- /** The schema id and {@link BinaryRow} stored as the value of kv store. */
- public static class Value {
- public final short schemaId;
- public final BinaryRow row;
-
- private Value(short schemaId, BinaryRow row) {
- this.schemaId = schemaId;
- this.row = row;
- }
+ return new BinaryValue(schemaId, row);
}
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
index bbfc6a7f5..2d842ab17 100644
--- a/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
+++ b/fluss-common/src/main/java/org/apache/fluss/row/encode/ValueEncoder.java
@@ -17,16 +17,13 @@
package org.apache.fluss.row.encode;
-import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.row.BinaryRow;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.types.RowType;
import org.apache.fluss.utils.UnsafeUtils;
/** An encoder to encode {@link BinaryRow} with a schema id as value to be stored in kv store. */
public class ValueEncoder {
- static final int SCHEMA_ID_LENGTH = 2;
+ public static final int SCHEMA_ID_LENGTH = 2;
/**
* Encode the {@code row} with a {@code schemaId} to a byte array value to be expected persisted
@@ -41,24 +38,4 @@ public class ValueEncoder {
row.copyTo(values, SCHEMA_ID_LENGTH);
return values;
}
-
- public static byte[] encodeRow(
- short schemaId, KvFormat kvFormat, RowType currentRowType, InternalRow row)
- throws Exception {
- if (row instanceof BinaryRow) {
- return encodeValue(schemaId, (BinaryRow) row);
- }
- // todo: reuse the encoder here
- try (RowEncoder rowEncoder = RowEncoder.create(kvFormat, currentRowType)) {
- rowEncoder.startNewRow();
- for (int i = 0; i < currentRowType.getFieldCount(); i++) {
- rowEncoder.encodeField(
- i,
- InternalRow.createFieldGetter(currentRowType.getTypeAt(i), i)
- .getFieldOrNull(row));
- }
- BinaryRow binaryRow = rowEncoder.finishRow();
- return encodeValue(schemaId, binaryRow);
- }
- }
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
index c597caf81..7c392b055 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/Projection.java
@@ -112,11 +112,7 @@ public class Projection {
return projectionPositions;
}
- /**
- * The id of the projection, which is used for fluss server scan.
- *
- * @return
- */
+ /** The id of the projection, which is used for fluss server scan. */
public int[] getProjectionIdInOrder() {
return projectionIdsInOrder;
}
diff --git a/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
index 6b66c0cd6..f10782ea1 100644
--- a/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
+++ b/fluss-common/src/main/java/org/apache/fluss/utils/SchemaUtil.java
@@ -20,7 +20,6 @@ package org.apache.fluss.utils;
import org.apache.fluss.exception.SchemaChangeException;
import org.apache.fluss.metadata.Schema;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -32,37 +31,17 @@ import static org.apache.fluss.row.ProjectedRow.UNEXIST_MAPPING;
public class SchemaUtil {
/**
- * get the target columns in expected schema.
+ * Get the index mapping from origin schema to expected schema. This mapping can be used to
+ * convert the row with origin schema to the row with expected schema via {@code
+ * ProjectedRow.from(getIndexMapping(inputSchema, targetSchema)).replace(inputRow)}.
*
- * @param targetColumns
- * @param originSchema
- * @param expectedSchema
- * @return
- */
- public static int[] getTargetColumns(
- int[] targetColumns, Schema originSchema, Schema expectedSchema) {
- if (targetColumns == null) {
- return null;
- }
-
- // get the mapping from expected schema to origin schema
- int[] indexMapping = getIndexMapping(expectedSchema, originSchema);
- ArrayList<Integer> targetColumnsInExpectedSchema = new ArrayList<>();
- for (int i = 0; i < targetColumns.length; i++) {
- if (indexMapping[targetColumns[i]] != UNEXIST_MAPPING) {
- targetColumnsInExpectedSchema.add(indexMapping[targetColumns[i]]);
- }
- }
- return targetColumnsInExpectedSchema.stream().mapToInt(value -> value).toArray();
- }
-
- /**
- * Get the index of each value of expectedSchema from originSchema.
- *
- * @param originSchema
- * @param expectedSchema
- * @return indexMapping the value of i-th means the expectedSchema's i-th column is the
- * originSchema's indexMapping[i]-th column.
+ * @param originSchema The origin schema.
+ * @param expectedSchema The expected schema.
+ * @return The index mapping array. The length of the array is the number of columns in the
+ * expected schema. Each element in the array is the index of the corresponding column in
+ * the origin schema. If a column in the expected schema does not exist in the origin
+ * schema, the corresponding element is UNEXIST_MAPPING (-1).
+ * @throws SchemaChangeException if there is a datatype mismatch between the two schemas.
*/
public static int[] getIndexMapping(Schema originSchema, Schema expectedSchema) {
List<Schema.Column> originColumns = originSchema.getColumns();
@@ -84,7 +63,7 @@ public class SchemaUtil {
if (originColumn != null
&& !Objects.equals(
expectedColumn.getDataType().copy(true),
- originColumns.get(indexMapping[i]).getDataType().copy(true))) {
+ originColumn.getDataType().copy(true))) {
throw new SchemaChangeException(
String.format(
@@ -92,7 +71,8 @@ public class SchemaUtil {
expectedColumn.getColumnId(),
expectedColumn.getName(),
expectedColumn.getDataType(),
- originColumn.getDataType()));
+ originColumn.getDataType()),
+ true);
}
}
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
index 2c3ff36ef..9a27089ac 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/FileLogProjectionTest.java
@@ -78,7 +78,8 @@ class FileLogProjectionTest {
TestData.DATA2,
TestData.DATA2);
- FileLogProjection projection = new FileLogProjection();
+ ProjectionPushdownCache cache = new ProjectionPushdownCache();
+ FileLogProjection projection = new FileLogProjection(cache);
// get schema during running.
doProjection(
1L,
@@ -87,16 +88,13 @@ class FileLogProjectionTest {
recordsOfData2RowType,
new int[] {0, 2},
recordsOfData2RowType.sizeInBytes());
- FileLogProjection.ProjectionInfo info1 = projection.currentProjection;
+ assertThat(cache.projectionCache.size()).isEqualTo(1);
+ FileLogProjection.ProjectionInfo info1 =
+ cache.getProjectionInfo(1L, schemaId, new int[] {0, 2});
assertThat(info1).isNotNull();
assertThat(info1.nodesProjection.stream().toArray()).isEqualTo(new int[] {0, 2});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info1.buffersProjection.stream().toArray()).isEqualTo(new int[] {0, 1, 5, 6, 7});
- assertThat(projection.projectionsCache).hasSize(1);
- assertThat(
- projection.projectionsCache.get(
- new FileLogProjection.ProjectionKey(1L, schemaId)))
- .isSameAs(info1);
doProjection(
2L,
@@ -105,37 +103,13 @@ class FileLogProjectionTest {
recordsOfData2RowType,
new int[] {1},
recordsOfData2RowType.sizeInBytes());
- FileLogProjection.ProjectionInfo info2 = projection.currentProjection;
+ assertThat(cache.projectionCache.size()).isEqualTo(2);
+ FileLogProjection.ProjectionInfo info2 =
+ cache.getProjectionInfo(2L, schemaId, new int[] {1});
assertThat(info2).isNotNull();
assertThat(info2.nodesProjection.stream().toArray()).isEqualTo(new int[] {1});
// a int: [0,1] ; b string: [2,3,4] ; c string: [5,6,7]
assertThat(info2.buffersProjection.stream().toArray()).isEqualTo(new int[] {2, 3, 4});
- assertThat(projection.projectionsCache).hasSize(2);
- assertThat(
- projection.projectionsCache.get(
- new FileLogProjection.ProjectionKey(2L, schemaId)))
- .isSameAs(info2);
-
- doProjection(
- 1L,
- schemaId,
- projection,
- recordsOfData2RowType,
- new int[] {0, 2},
- recordsOfData2RowType.sizeInBytes());
- assertThat(projection.currentProjection).isNotNull().isSameAs(info1);
-
- assertThatThrownBy(
- () ->
- doProjection(
- 1L,
- schemaId,
- projection,
- recordsOfData2RowType,
- new int[] {1},
- recordsOfData2RowType.sizeInBytes()))
- .isInstanceOf(InvalidColumnProjectionException.class)
- .hasMessage("The schema and projection should be identical for the same table id.");
}
@Test
@@ -149,7 +123,7 @@ class FileLogProjectionTest {
TestData.DATA2_ROW_TYPE,
TestData.DATA2,
TestData.DATA2);
- FileLogProjection projection = new FileLogProjection();
+ FileLogProjection projection = new FileLogProjection(new ProjectionPushdownCache());
assertThatThrownBy(
() ->
doProjection(
@@ -160,7 +134,7 @@ class FileLogProjectionTest {
new int[] {3},
recordsOfData2RowType.sizeInBytes()))
.isInstanceOf(InvalidColumnProjectionException.class)
- .hasMessage("Projected field id 3 is not contains in [0, 1, 2]");
+ .hasMessage("Projected field id 3 is not contained in [0, 1, 2]");
assertThatThrownBy(
() ->
@@ -220,7 +194,7 @@ class FileLogProjectionTest {
doProjection(
1L,
schemaId,
- new FileLogProjection(),
+ new FileLogProjection(new ProjectionPushdownCache()),
fileLogRecords,
projectedFields,
Integer.MAX_VALUE);
@@ -248,7 +222,7 @@ class FileLogProjectionTest {
TestData.DATA1_ROW_TYPE,
TestData.DATA1,
TestData.ANOTHER_DATA1);
- FileLogProjection projection = new FileLogProjection();
+ FileLogProjection projection = new FileLogProjection(new ProjectionPushdownCache());
// overwrite the wrong decoding byte order endian
projection.getLogHeaderBuffer().order(ByteOrder.BIG_ENDIAN);
assertThatThrownBy(
@@ -305,7 +279,10 @@ class FileLogProjectionTest {
int maxBytes = totalSize / i;
List<Object[]> results =
doProjection(
- new FileLogProjection(), fileLogRecords, new int[] {0, 1}, maxBytes);
+ new FileLogProjection(new ProjectionPushdownCache()),
+ fileLogRecords,
+ new int[] {0, 1},
+ maxBytes);
if (results.isEmpty()) {
hasEmpty = true;
} else if (results.size() == TestData.DATA1.size()) {
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
index 108ab2acf..36aef5b9d 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/LogTestBase.java
@@ -72,7 +72,6 @@ public abstract class LogTestBase {
assertLogRecordsListEquals(expected, actual, baseRowType);
}
- // todo: 在这个基础上可以加更多最简单的schema evolution测试,非常好
protected void assertIndexedLogRecordBatchAndRowEquals(
LogRecordBatch actual,
LogRecordBatch expected,
diff --git a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
index f74f379eb..9a9581cab 100644
--- a/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
+++ b/fluss-common/src/test/java/org/apache/fluss/record/TestingSchemaGetter.java
@@ -24,11 +24,12 @@ import org.apache.fluss.metadata.SchemaInfo;
import java.util.HashMap;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
/** A testing schema metadata subscriber. */
public class TestingSchemaGetter implements SchemaGetter {
private SchemaInfo schemaInfo;
- private Map<Integer, Schema> schemaCaches;
+ private final Map<Integer, Schema> schemaCaches;
public TestingSchemaGetter() {
this.schemaInfo = null;
@@ -56,6 +57,15 @@ public class TestingSchemaGetter implements SchemaGetter {
return schemaCaches.get(schemaId);
}
+ @Override
+ public CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId) {
+ if (!schemaCaches.containsKey(schemaId)) {
+ throw new SchemaNotExistException("Schema not exist");
+ }
+ return CompletableFuture.completedFuture(
+ new SchemaInfo(schemaCaches.get(schemaId), schemaId));
+ }
+
@Override
public SchemaInfo getLatestSchemaInfo() {
return schemaInfo;
diff --git a/fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java b/fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java
new file mode 100644
index 000000000..453f02be3
--- /dev/null
+++ b/fluss-common/src/test/java/org/apache/fluss/row/PaddingRowTest.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.fluss.row;
+
+import org.junit.jupiter.api.Test;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for {@link PaddingRow}. */
+class PaddingRowTest {
+
+ @Test
+ void testPaddingRowIsNullAt() {
+ // mock an InternalRow with only 2 columns
+ InternalRow row = GenericRow.of(1, null);
+ PaddingRow paddingRow = new PaddingRow(4).replaceRow(row);
+
+ assertThat(paddingRow.getFieldCount()).isEqualTo(4);
+ assertThat(paddingRow.isNullAt(0)).isFalse();
+ assertThat(paddingRow.isNullAt(1)).isTrue();
+ assertThat(paddingRow.isNullAt(2)).isTrue(); // exceed original row, should be null
+ assertThat(paddingRow.isNullAt(3)).isTrue();
+ }
+
+ @Test
+ void testPaddingRowDelegatesGetters() {
+ InternalRow row =
+ GenericRow.of(
+ true,
+ (byte) 2,
+ (short) 3,
+ 4,
+ 5L,
+ 6.5f,
+ 7.5,
+ new byte[] {1, 2, 3},
+ new byte[] {4, 5},
+ BinaryString.fromString("abc"),
+ TimestampLtz.fromEpochMillis(1000));
+ PaddingRow paddingRow = new PaddingRow(9).replaceRow(row);
+
+ assertThat(paddingRow.getBoolean(0)).isTrue();
+ assertThat(paddingRow.getByte(1)).isEqualTo((byte) 2);
+ assertThat(paddingRow.getShort(2)).isEqualTo((short) 3);
+ assertThat(paddingRow.getInt(3)).isEqualTo(4);
+ assertThat(paddingRow.getLong(4)).isEqualTo(5L);
+ assertThat(paddingRow.getFloat(5)).isEqualTo(6.5f);
+ assertThat(paddingRow.getDouble(6)).isEqualTo(7.5);
+ assertThat(paddingRow.getBinary(7, 3)).isEqualTo(new byte[] {1, 2, 3});
+ assertThat(paddingRow.getBytes(8)).isEqualTo(new byte[] {4, 5});
+ assertThat(paddingRow.getString(9)).isEqualTo(BinaryString.fromString("abc"));
+ assertThat(paddingRow.getTimestampLtz(10, 3)).isEqualTo(TimestampLtz.fromEpochMillis(1000));
+ }
+
+ @Test
+ void testReplaceRowInPlace() {
+ InternalRow row1 = GenericRow.of(1);
+ InternalRow row2 = GenericRow.of(2);
+ PaddingRow paddingRow = new PaddingRow(1).replaceRow(row1);
+ assertThat(paddingRow.getInt(0)).isEqualTo(1);
+ paddingRow.replaceRow(row2);
+ assertThat(paddingRow.getInt(0)).isEqualTo(2);
+ }
+}
diff --git a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
index 09467d31a..80b21a719 100644
--- a/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
+++ b/fluss-common/src/test/java/org/apache/fluss/row/ProjectedRowTest.java
@@ -183,6 +183,7 @@ class ProjectedRowTest {
.isExactlyInstanceOf(SchemaChangeException.class)
.hasMessage(
"Expected datatype of column(id=0,name=a) is [INT], while the actual datatype is [BIGINT]");
+
assertThatThrownBy(
() ->
ProjectedRow.from(
diff --git a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
index bd1b6af81..12b05f186 100644
--- a/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
+++ b/fluss-flink/fluss-flink-1.20/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level = INFO
+rootLogger.level = OFF
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
index 314e4b40f..003e910ea 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/row/FlinkAsFlussRow.java
@@ -23,34 +23,17 @@ import org.apache.fluss.row.InternalArray;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
-import org.apache.fluss.types.RowType;
import org.apache.flink.table.data.DecimalData;
import org.apache.flink.table.data.RowData;
import org.apache.flink.table.data.TimestampData;
-import javax.annotation.Nullable;
-
-import java.util.List;
-
/** Wraps a Flink {@link RowData} as a Fluss {@link InternalRow}. */
public class FlinkAsFlussRow implements InternalRow {
private RowData flinkRow;
- /**
- * Creates a FlinkAsFlussRow with a mapping from Fluss field index to Flink field index. If the
- * indexMapping is null, flink and flink share same field name.
- */
- @Nullable private final int[] indexMapping;
-
- public FlinkAsFlussRow() {
- this(null);
- }
-
- public FlinkAsFlussRow(int[] indexMapping) {
- this.indexMapping = indexMapping;
- }
+ public FlinkAsFlussRow() {}
public FlinkAsFlussRow replace(RowData flinkRow) {
this.flinkRow = flinkRow;
@@ -59,94 +42,62 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public int getFieldCount() {
- if (indexMapping == null) {
- return flinkRow.getArity();
- } else {
- return indexMapping.length;
- }
+ return flinkRow.getArity();
}
@Override
public boolean isNullAt(int pos) {
- // if no mapping, means that the field is not found in Flink, just ignore it.
- if ((indexMapping != null && indexMapping[pos] == -1)) {
- return true;
- }
-
- // If pos is larger than flinkRow.getArity(), it indicates that the Fluss table's data type
- // is wider than the data provided by Flink.
- // This often occurs when a schema change happens to the catalog table before job restart.
- // Only appending columns at the end is compatible, just need to ignore the trailing
- // columns.
- // Other types of schema changes (e.g., dropping columns) would lead to incompatibility.
- // Therefore, Fluss currently only supports appending columns, and here we simply ignore any
- // extra columns.
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- if (flussPos >= flinkRow.getArity()) {
- return true;
- }
-
- return flinkRow.isNullAt(flussPos);
+ return flinkRow.isNullAt(pos);
}
@Override
public boolean getBoolean(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getBoolean(flussPos);
+ return flinkRow.getBoolean(pos);
}
@Override
public byte getByte(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getByte(flussPos);
+ return flinkRow.getByte(pos);
}
@Override
public short getShort(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getShort(flussPos);
+ return flinkRow.getShort(pos);
}
@Override
public int getInt(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getInt(flussPos);
+ return flinkRow.getInt(pos);
}
@Override
public long getLong(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getLong(flussPos);
+ return flinkRow.getLong(pos);
}
@Override
public float getFloat(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getFloat(flussPos);
+ return flinkRow.getFloat(pos);
}
@Override
public double getDouble(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getDouble(flussPos);
+ return flinkRow.getDouble(pos);
}
@Override
public BinaryString getChar(int pos, int length) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
+ return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
}
@Override
public BinaryString getString(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return BinaryString.fromBytes(flinkRow.getString(flussPos).toBytes());
+ return BinaryString.fromBytes(flinkRow.getString(pos).toBytes());
}
@Override
public Decimal getDecimal(int pos, int precision, int scale) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return fromFlinkDecimal(flinkRow.getDecimal(flussPos, precision, scale));
+ return fromFlinkDecimal(flinkRow.getDecimal(pos, precision, scale));
}
public static Decimal fromFlinkDecimal(DecimalData decimal) {
@@ -161,43 +112,26 @@ public class FlinkAsFlussRow implements InternalRow {
@Override
public TimestampNtz getTimestampNtz(int pos, int precision) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
+ TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
return TimestampNtz.fromMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public TimestampLtz getTimestampLtz(int pos, int precision) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- TimestampData timestamp = flinkRow.getTimestamp(flussPos, precision);
+ TimestampData timestamp = flinkRow.getTimestamp(pos, precision);
return TimestampLtz.fromEpochMillis(
timestamp.getMillisecond(), timestamp.getNanoOfMillisecond());
}
@Override
public byte[] getBinary(int pos, int length) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getBinary(flussPos);
+ return flinkRow.getBinary(pos);
}
@Override
public byte[] getBytes(int pos) {
- int flussPos = indexMapping == null ? pos : indexMapping[pos];
- return flinkRow.getBinary(flussPos);
- }
-
- /** Converts a Flink {@link RowType} to a Fluss {@link RowType}, based on the field name. */
- public static FlinkAsFlussRow from(
- org.apache.flink.table.types.logical.RowType flinkRowType, RowType flussRowType) {
- int[] indexMapping = new int[flussRowType.getFieldCount()];
- List<String> fieldNames = flussRowType.getFieldNames();
- for (int i = 0; i < flussRowType.getFieldCount(); i++) {
- String fieldName = fieldNames.get(i);
- int fieldIndex = flinkRowType.getFieldIndex(fieldName);
- indexMapping[i] = fieldIndex;
- }
- return new FlinkAsFlussRow(indexMapping);
+ return flinkRow.getBinary(pos);
}
@Override
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
index 0f3fff210..2096b1095 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/FlussSerializationSchema.java
@@ -69,10 +69,11 @@ public interface FlussSerializationSchema<T> extends Serializable {
RowType getRowSchema();
/**
- * Returns the Flink table row schema.
+ * Returns the input row schema which is the Flink {@link
+ * org.apache.flink.table.data.RowData}.
*
- * @return The schema of the Flink table row.
+ * @return The schema of the input Flink row.
*/
- org.apache.flink.table.types.logical.RowType getTableRowType();
+ org.apache.flink.table.types.logical.RowType getInputRowSchema();
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
index 05e2921fc..1a46ec0bc 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/RowDataSerializationSchema.java
@@ -21,11 +21,49 @@ import org.apache.fluss.flink.row.FlinkAsFlussRow;
import org.apache.fluss.flink.row.OperationType;
import org.apache.fluss.flink.row.RowWithOp;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.PaddingRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.flink.table.data.RowData;
import org.apache.flink.types.RowKind;
-/** Default implementation of RowDataConverter for RowData. */
+import javax.annotation.Nullable;
+
+import java.util.List;
+
+/**
+ * Default implementation of RowDataConverter for RowData.
+ *
+ * <p>Schema Evolution Handling:
+ *
+ * <p>There are three schema in sink case.
+ *
+ * <ul>
+ * <li>(1) Consumed Row Schema (RowData): the schema of RowData consumed, currently, Flink doesn't
+ * provide API to fetch it, but we can get the row arity from RowData.getArity().
+ * <li>(2) Plan Row Schema ({@link InitializationContext#getInputRowSchema()}): the compiled input
+ * row schema of the sink, but the schema is fetched from catalog, so it is not the consumed
+ * row schema. This is updated when job is re-compiled.
+ * <li>(3) Table Latest Row Schema ({@link InitializationContext#getRowSchema()}): the latest
+ * schema of the sink table, which is fetched at open() during runtime, so it will be updated
+ * when Flink job restarts.
+ * </ul>
+ *
+ * <p>We always want to use latest schema to write data, so we need to add conversions from consumed
+ * Flink RowData.
+ *
+ * <ul>
+ * <li>The {@link #converter} is used to wrap Flink {@link RowData} into Fluss {@link InternalRow}
+ * without any schema transformation.
+ * <li>The {@link #outputPadding} is used to pad nulls for new columns when new columns are added.
+ * This may happen when table is added new columns after Flink SQL job restored from
+ * CompiledPlan that the Consumed Row Schema is old but the Plan Row Schema is new, so the
+ * Consumed Row Schema has less fields than Plan Row Schema.
+ * <li>The {@link #outputProjection} is used to re-arrange the fields according to latest schema
+ * if Plan Row Schema is not match Table Latest Row Schema. This may happen when table is
+ * added new columns after the Flink job compiled and before job restarts.
+ * </ul>
+ */
public class RowDataSerializationSchema implements FlussSerializationSchema<RowData> {
private static final long serialVersionUID = 1L;
@@ -44,6 +82,19 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
*/
private transient FlinkAsFlussRow converter;
+ /**
+ * The padding row for output, used when the input row has fewer fields than the target schema.
+ * This may happen when table is added new columns between Flink job restarts.
+ */
+ private transient PaddingRow outputPadding;
+
+ /**
+ * The projected row for output, used when schema evolution occurs, because we want to write
+ * rows using new schema (e.g., fill null for new columns). This may happen when table is *
+ * added new columns after the Flink job compiled and before job restarts.
+ */
+ @Nullable private transient ProjectedRow outputProjection;
+
/**
* Constructs a new {@code RowSerializationSchema}.
*
@@ -63,7 +114,20 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
*/
@Override
public void open(InitializationContext context) throws Exception {
- this.converter = FlinkAsFlussRow.from(context.getTableRowType(), context.getRowSchema());
+ this.converter = new FlinkAsFlussRow();
+ List<String> targetFieldNames = context.getRowSchema().getFieldNames();
+ List<String> inputFieldNames = context.getInputRowSchema().getFieldNames();
+ this.outputPadding = new PaddingRow(inputFieldNames.size());
+ if (targetFieldNames.size() != inputFieldNames.size()) {
+ // there is a schema evolution happens (e.g., ADD COLUMN), need to build index mapping
+ int[] indexMapping = new int[targetFieldNames.size()];
+ for (int i = 0; i < targetFieldNames.size(); i++) {
+ String fieldName = targetFieldNames.get(i);
+ int fieldIndex = inputFieldNames.indexOf(fieldName);
+ indexMapping[i] = fieldIndex;
+ }
+ outputProjection = ProjectedRow.from(indexMapping);
+ }
}
/**
@@ -82,6 +146,14 @@ public class RowDataSerializationSchema implements FlussSerializationSchema<RowD
"Converter not initialized. The open() method must be called before serializing records.");
}
InternalRow row = converter.replace(value);
+ // handling schema evolution for changes before job compilation
+ if (row.getFieldCount() < outputPadding.getFieldCount()) {
+ row = outputPadding.replaceRow(row);
+ }
+ // handling schema evolution for changes after job compilation
+ if (outputProjection != null) {
+ row = outputProjection.replaceRow(row);
+ }
OperationType opType = toOperationType(value.getRowKind());
return new RowWithOp(row, opType);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
index 93b849e58..cc1d168cd 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/serializer/SerializerInitContextImpl.java
@@ -47,7 +47,7 @@ public class SerializerInitContextImpl implements FlussSerializationSchema.Initi
}
@Override
- public org.apache.flink.table.types.logical.RowType getTableRowType() {
+ public org.apache.flink.table.types.logical.RowType getInputRowSchema() {
return flinkRowType;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
index a4469fd32..fe0eb28ea 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/sink/writer/FlinkSinkWriter.java
@@ -29,7 +29,6 @@ import org.apache.fluss.flink.row.OperationType;
import org.apache.fluss.flink.row.RowWithOp;
import org.apache.fluss.flink.sink.serializer.FlussSerializationSchema;
import org.apache.fluss.flink.sink.serializer.SerializerInitContextImpl;
-import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.metadata.TableInfo;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.metrics.Gauge;
@@ -111,10 +110,10 @@ public abstract class FlinkSinkWriter<InputT> implements SinkWriter<InputT> {
connection = ConnectionFactory.createConnection(flussConfig, flinkMetricRegistry);
table = connection.getTable(tablePath);
LOG.info(
- "Current Fluss Schema is {}, Table RowType is {}",
+ "Current Fluss Schema is {}, input Flink RowType is {}",
table.getTableInfo().getSchema(),
tableRowType);
- // sanityCheck(table.getTableInfo());
+ sanityCheck(table.getTableInfo());
try {
this.serializationSchema.open(
@@ -209,22 +208,6 @@ public abstract class FlinkSinkWriter<InputT> implements SinkWriter<InputT> {
"Primary key constraint is not matched between metadata in Fluss (%s) and Flink (%s).",
flussTableInfo.hasPrimaryKey(), hasPrimaryKey));
}
- RowType currentTableRowType = FlinkConversions.toFlinkRowType(flussTableInfo.getRowType());
- if (!this.tableRowType.copy(false).equals(currentTableRowType.copy(false))) {
- // The default nullability of Flink row type and Fluss row type might be not the same,
- // thus we need to compare the row type without nullability here.
-
- // Throw exception if the schema is the not same, this should rarely happen because we
- // only allow fluss tables derived from fluss catalog. But this can happen if an ALTER
- // TABLE command executed on the fluss table, after the job is submitted but before the
- // SinkFunction is opened.
- throw new ValidationException(
- "The Flink query schema is not matched to current Fluss table schema. "
- + "\nFlink query schema: "
- + this.tableRowType
- + "\nFluss table schema: "
- + currentTableRowType);
- }
}
private long computeSendTime() {
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
index 0bf9f187c..bc15390fe 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkSource.java
@@ -60,6 +60,7 @@ public class FlinkSource<OUT>
private final boolean hasPrimaryKey;
private final boolean isPartitioned;
private final RowType sourceOutputType;
+ @Nullable private final int[] projectedFields;
protected final OffsetsInitializer offsetsInitializer;
protected final long scanPartitionDiscoveryIntervalMs;
private final boolean streaming;
@@ -73,6 +74,7 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -84,6 +86,7 @@ public class FlinkSource<OUT>
hasPrimaryKey,
isPartitioned,
sourceOutputType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
@@ -98,6 +101,7 @@ public class FlinkSource<OUT>
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -109,6 +113,7 @@ public class FlinkSource<OUT>
this.hasPrimaryKey = hasPrimaryKey;
this.isPartitioned = isPartitioned;
this.sourceOutputType = sourceOutputType;
+ this.projectedFields = projectedFields;
this.offsetsInitializer = offsetsInitializer;
this.scanPartitionDiscoveryIntervalMs = scanPartitionDiscoveryIntervalMs;
this.deserializationSchema = deserializationSchema;
@@ -182,7 +187,6 @@ public class FlinkSource<OUT>
context.getUserCodeClassLoader(),
sourceOutputType));
FlinkRecordEmitter<OUT> recordEmitter = new FlinkRecordEmitter<>(deserializationSchema);
- // recall to projectedFields
return new FlinkSourceReader<>(
elementsQueue,
@@ -190,6 +194,7 @@ public class FlinkSource<OUT>
tablePath,
sourceOutputType,
context,
+ projectedFields,
flinkSourceReaderMetrics,
recordEmitter,
lakeSource);
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
index b110e9f88..ef4b63812 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlinkTableSource.java
@@ -325,6 +325,7 @@ public class FlinkTableSource
hasPrimaryKey(),
isPartitioned(),
flussRowType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
new RowDataDeserializationSchema(),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
index 5f0b2ff8c..427741834 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSource.java
@@ -24,6 +24,8 @@ import org.apache.fluss.flink.source.enumerator.initializer.OffsetsInitializer;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.types.RowType;
+import javax.annotation.Nullable;
+
/**
* A Flink DataStream source implementation for reading data from Fluss tables.
*
@@ -61,6 +63,7 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
boolean hasPrimaryKey,
boolean isPartitioned,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
OffsetsInitializer offsetsInitializer,
long scanPartitionDiscoveryIntervalMs,
FlussDeserializationSchema<OUT> deserializationSchema,
@@ -72,6 +75,7 @@ public class FlussSource<OUT> extends FlinkSource<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
index 4891ab256..f16639753 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/FlussSourceBuilder.java
@@ -296,6 +296,7 @@ public class FlussSourceBuilder<OUT> {
hasPrimaryKey,
isPartitioned,
sourceOutputType,
+ projectedFields,
offsetsInitializer,
scanPartitionDiscoveryIntervalMs,
deserializationSchema,
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
index 6e4647f80..82fd1bbc3 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkAsyncLookupFunction.java
@@ -27,10 +27,12 @@ import org.apache.fluss.config.Configuration;
import org.apache.fluss.exception.TableNotExistException;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
import org.apache.fluss.flink.source.lookup.LookupNormalizer.RemainingFilter;
+import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.row.ProjectedRow;
import org.apache.fluss.utils.ExceptionUtils;
import org.apache.flink.table.data.RowData;
@@ -47,6 +49,7 @@ import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.concurrent.CompletableFuture;
+import java.util.stream.IntStream;
/** A flink async lookup function for fluss. */
public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
@@ -59,7 +62,7 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
private final TablePath tablePath;
private final RowType flinkRowType;
private final LookupNormalizer lookupNormalizer;
- @Nullable private final int[] projection;
+ @Nullable private int[] projection;
private transient FlussRowToFlinkRowConverter flussRowToFlinkRowConverter;
private transient Connection connection;
@@ -85,20 +88,22 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
- org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
- LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
- // Currently, only primary key and prefix lookup are supported. These keys must be primary
- // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
if (projection == null) {
outputRowType = flinkRowType;
+ // we force to do projection if no projection pushdown, in order to handle schema
+ // changes (ADD COLUMN LAST), this guarantees the input row of
+ // flussRowToFlinkRowConverter is in expected schema even new columns are added.
+ projection = IntStream.range(0, flinkRowType.getFieldCount()).toArray();
} else {
outputRowType = FlinkUtils.projectRowType(flinkRowType, projection);
}
-
- flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
+ // TODO: currently, we assume only ADD COLUMN LAST schema changes, so the projection
+ // positions can still work even after such changes.
+ flussRowToFlinkRowConverter =
+ new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -164,7 +169,7 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
List<RowData> projectedRow = new ArrayList<>();
for (InternalRow row : lookupResult) {
if (row != null) {
- RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
+ RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRow.add(flinkRow);
}
@@ -173,6 +178,14 @@ public class FlinkAsyncLookupFunction extends AsyncLookupFunction {
resultFuture.complete(projectedRow);
}
+ private InternalRow maybeProject(InternalRow row) {
+ if (projection == null) {
+ return row;
+ }
+ // should not reuse objects for async operations
+ return ProjectedRow.from(projection).replaceRow(row);
+ }
+
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
index e9a5ad915..477666737 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunction.java
@@ -25,6 +25,7 @@ import org.apache.fluss.client.lookup.Lookuper;
import org.apache.fluss.client.table.Table;
import org.apache.fluss.config.Configuration;
import org.apache.fluss.flink.row.FlinkAsFlussRow;
+import org.apache.fluss.flink.utils.FlinkConversions;
import org.apache.fluss.flink.utils.FlinkUtils;
import org.apache.fluss.flink.utils.FlussRowToFlinkRowConverter;
import org.apache.fluss.metadata.TablePath;
@@ -44,6 +45,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
+import java.util.stream.IntStream;
/** A flink lookup function for fluss. */
public class FlinkLookupFunction extends LookupFunction {
@@ -82,22 +84,25 @@ public class FlinkLookupFunction extends LookupFunction {
LOG.info("start open ...");
connection = ConnectionFactory.createConnection(flussConfig);
table = connection.getTable(tablePath);
- org.apache.fluss.types.RowType flussRowType = table.getTableInfo().getRowType();
- LOG.info("Current Fluss Schema is {}, Flink RowType is {}", flussRowType, flinkRowType);
- // Currently, only primary key and prefix lookup are supported. These keys must be primary
- // key which cannot modified.
lookupRow = new FlinkAsFlussRow();
final RowType outputRowType;
if (projection == null) {
outputRowType = flinkRowType;
- projectedRow = null;
+ // we force to do projection if no projection pushdown, in order to handle schema
+ // changes (ADD COLUMN LAST), this guarantees the input row of
+ // flussRowToFlinkRowConverter is in expected schema even new columns are added.
+ projectedRow =
+ ProjectedRow.from(IntStream.range(0, flinkRowType.getFieldCount()).toArray());
} else {
outputRowType = FlinkUtils.projectRowType(flinkRowType, projection);
// reuse the projected row
projectedRow = ProjectedRow.from(projection);
}
- flussRowToFlinkRowConverter = new FlussRowToFlinkRowConverter(flussRowType, outputRowType);
+ // TODO: currently, we assume only ADD COLUMN LAST schema changes, so the projection
+ // positions can still work even after such changes.
+ flussRowToFlinkRowConverter =
+ new FlussRowToFlinkRowConverter(FlinkConversions.toFlussRowType(outputRowType));
Lookup lookup = table.newLookup();
if (lookupNormalizer.getLookupType() == LookupType.PREFIX_LOOKUP) {
@@ -133,7 +138,8 @@ public class FlinkLookupFunction extends LookupFunction {
List<RowData> projectedRows = new ArrayList<>();
for (InternalRow row : lookupRows) {
if (row != null) {
- RowData flinkRow = flussRowToFlinkRowConverter.toFlinkRowData(row);
+ RowData flinkRow =
+ flussRowToFlinkRowConverter.toFlinkRowData(maybeProject(row));
if (remainingFilter == null || remainingFilter.isMatch(flinkRow)) {
projectedRows.add(flinkRow);
}
@@ -146,6 +152,13 @@ public class FlinkLookupFunction extends LookupFunction {
}
}
+ private InternalRow maybeProject(InternalRow row) {
+ if (projectedRow == null) {
+ return row;
+ }
+ return projectedRow.replaceRow(row);
+ }
+
@Override
public void close() throws Exception {
LOG.info("start close ...");
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
index 639d52513..6364532bb 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceReader.java
@@ -40,6 +40,8 @@ import org.apache.flink.api.connector.source.SourceReaderContext;
import org.apache.flink.connector.base.source.reader.RecordsWithSplitIds;
import org.apache.flink.connector.base.source.reader.synchronization.FutureCompletingBlockingQueue;
+import javax.annotation.Nullable;
+
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
@@ -55,6 +57,7 @@ public class FlinkSourceReader<OUT>
TablePath tablePath,
RowType sourceOutputType,
SourceReaderContext context,
+ @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
FlinkRecordEmitter<OUT> recordEmitter,
LakeSource<LakeSplit> lakeSource) {
@@ -67,6 +70,7 @@ public class FlinkSourceReader<OUT>
flussConfig,
tablePath,
sourceOutputType,
+ projectedFields,
flinkSourceReaderMetrics,
lakeSource),
(ignore) -> {}),
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
index 10997e91e..ce50fe406 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReader.java
@@ -58,6 +58,7 @@ import java.io.IOException;
import java.time.Duration;
import java.util.ArrayDeque;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
@@ -119,6 +120,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
Configuration flussConf,
TablePath tablePath,
RowType sourceOutputType,
+ @Nullable int[] projectedFields,
FlinkSourceReaderMetrics flinkSourceReaderMetrics,
@Nullable LakeSource<LakeSplit> lakeSource) {
this.flinkMetricRegistry =
@@ -128,17 +130,15 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
this.sourceOutputType = sourceOutputType;
this.boundedSplits = new ArrayDeque<>();
this.subscribedBuckets = new HashMap<>();
+ this.projectedFields = projectedFields;
+ if (projectedFields == null) {}
+
this.flinkSourceReaderMetrics = flinkSourceReaderMetrics;
- this.projectedFields =
- reCalculateProjectedFields(sourceOutputType, table.getTableInfo().getRowType());
+ sanityCheck(table.getTableInfo().getRowType(), projectedFields);
this.logScanner = table.newScan().project(projectedFields).createLogScanner();
this.stoppingOffsets = new HashMap<>();
this.emptyLogSplits = new HashSet<>();
this.lakeSource = lakeSource;
- LOG.info(
- "fluss table schema: {}, flink table output type:{}",
- table.getTableInfo().getSchema(),
- sourceOutputType);
}
@Override
@@ -216,8 +216,7 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
LakeSnapshotAndFlussLogSplit lakeSnapshotAndFlussLogSplit =
(LakeSnapshotAndFlussLogSplit) sourceSplitBase;
if (lakeSnapshotAndFlussLogSplit.isStreaming()) {
- // is streaming split which has no stopping offset, we need also
- // subscribe
+ // is streaming split which has no stopping offset, we need also subscribe
// change log
subscribeLog(
lakeSnapshotAndFlussLogSplit,
@@ -564,33 +563,41 @@ public class FlinkSourceSplitReader implements SplitReader<RecordAndPos, SourceS
flinkMetricRegistry.close();
}
- /**
- * The projected fields for the fluss table from the source output types. Mapping based on
- * column name rather thn column id.
- */
- private static int[] reCalculateProjectedFields(
- RowType sourceOutputType, RowType flussRowType) {
- if (sourceOutputType.copy(false).equals(flussRowType.copy(false))) {
- return null;
- }
-
- List<String> fieldNames = sourceOutputType.getFieldNames();
- int[] projectedFlussFields = new int[fieldNames.size()];
- for (int i = 0; i < fieldNames.size(); i++) {
- int fieldIndex = flussRowType.getFieldIndex(fieldNames.get(i));
- if (fieldIndex == -1) {
- throw new ValidationException(
- String.format(
- "The field %s is not found in the fluss table.",
- fieldNames.get(i)));
+ private void sanityCheck(RowType flussTableRowType, @Nullable int[] projectedFields) {
+ RowType tableRowType =
+ projectedFields != null
+ ? flussTableRowType.project(projectedFields)
+ // only read the output fields from source
+ : flussTableRowType.project(sourceOutputType.getFieldNames());
+ if (!sourceOutputType.copy(false).equals(tableRowType.copy(false))) {
+ // The default nullability of Flink row type and Fluss row type might be not the same,
+ // thus we need to compare the row type without nullability here.
+
+ final String flussSchemaMsg;
+ if (projectedFields == null) {
+ flussSchemaMsg = "\nFluss table schema: " + tableRowType;
+ } else {
+ flussSchemaMsg =
+ "\nFluss table schema: "
+ + tableRowType
+ + " (projection "
+ + Arrays.toString(projectedFields)
+ + ")";
}
- projectedFlussFields[i] = fieldIndex;
+ // Throw exception if the schema is the not same, this should rarely happen because we
+ // only allow fluss tables derived from fluss catalog. But this can happen if an ALTER
+ // TABLE command executed on the fluss table, after the job is submitted but before the
+ // SinkFunction is opened.
+ throw new ValidationException(
+ "The Flink query schema is not matched to Fluss table schema. "
+ + "\nFlink query schema: "
+ + sourceOutputType
+ + flussSchemaMsg);
}
- return projectedFlussFields;
}
@VisibleForTesting
- public int[] getProjectedFields() {
+ int[] getProjectedFields() {
return projectedFields;
}
}
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
index 74375e24c..2553b0c18 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlinkConversions.java
@@ -86,18 +86,6 @@ public class FlinkConversions {
private FlinkConversions() {}
- public static int[] toFlinkRowTypeIndexMapping(
- RowType sourceFlussRowType,
- org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
- int[] indexMapping = new int[targetFlinkRowType.getFieldCount()];
- for (int i = 0; i < targetFlinkRowType.getFieldCount(); i++) {
- String fieldName = targetFlinkRowType.getFieldNames().get(i);
- int index = sourceFlussRowType.getFieldIndex(fieldName);
- indexMapping[i] = index;
- }
- return indexMapping;
- }
-
/** Convert Fluss's type to Flink's type. */
@VisibleForTesting
public static org.apache.flink.table.types.DataType toFlinkType(DataType flussDataType) {
diff --git a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
index 4dc5287b0..e52a83a82 100644
--- a/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
+++ b/fluss-flink/fluss-flink-common/src/main/java/org/apache/fluss/flink/utils/FlussRowToFlinkRowConverter.java
@@ -39,32 +39,18 @@ import org.apache.flink.types.RowKind;
import java.io.Serializable;
import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowKind;
-import static org.apache.fluss.flink.utils.FlinkConversions.toFlinkRowType;
/** A converter to convert Fluss's {@link InternalRow} to Flink's {@link RowData}. */
public class FlussRowToFlinkRowConverter {
private final FlussDeserializationConverter[] toFlinkFieldConverters;
private final InternalRow.FieldGetter[] flussFieldGetters;
- private final int[] indexMapping;
- // todo: remove
public FlussRowToFlinkRowConverter(RowType rowType) {
- this(rowType, toFlinkRowType(rowType));
- }
-
- public FlussRowToFlinkRowConverter(
- RowType sourceFlussRowType,
- org.apache.flink.table.types.logical.RowType targetFlinkRowType) {
- this.indexMapping =
- FlinkConversions.toFlinkRowTypeIndexMapping(sourceFlussRowType, targetFlinkRowType);
- this.toFlinkFieldConverters =
- new FlussDeserializationConverter[sourceFlussRowType.getFieldCount()];
- this.flussFieldGetters = new InternalRow.FieldGetter[sourceFlussRowType.getFieldCount()];
- for (int i = 0; i < sourceFlussRowType.getFieldCount(); i++) {
- toFlinkFieldConverters[i] =
- createNullableInternalConverter(sourceFlussRowType.getTypeAt(i));
- flussFieldGetters[i] =
- InternalRow.createFieldGetter(sourceFlussRowType.getTypeAt(i), i);
+ this.toFlinkFieldConverters = new FlussDeserializationConverter[rowType.getFieldCount()];
+ this.flussFieldGetters = new InternalRow.FieldGetter[rowType.getFieldCount()];
+ for (int i = 0; i < rowType.getFieldCount(); i++) {
+ toFlinkFieldConverters[i] = createNullableInternalConverter(rowType.getTypeAt(i));
+ flussFieldGetters[i] = InternalRow.createFieldGetter(rowType.getTypeAt(i), i);
}
}
@@ -77,18 +63,11 @@ public class FlussRowToFlinkRowConverter {
}
private RowData toFlinkRowData(InternalRow flussRow, RowKind rowKind) {
- int targetLength = indexMapping.length;
- GenericRowData genericRowData = new GenericRowData(targetLength);
+ GenericRowData genericRowData = new GenericRowData(toFlinkFieldConverters.length);
genericRowData.setRowKind(rowKind);
- for (int i = 0; i < targetLength; i++) {
- int flussIndex = indexMapping[i];
- if (flussIndex == -1) {
- genericRowData.setField(i, null);
- } else {
- Object flussField = flussFieldGetters[flussIndex].getFieldOrNull(flussRow);
- genericRowData.setField(
- i, toFlinkFieldConverters[flussIndex].deserialize(flussField));
- }
+ for (int i = 0; i < toFlinkFieldConverters.length; i++) {
+ Object flussField = flussFieldGetters[i].getFieldOrNull(flussRow);
+ genericRowData.setField(i, toFlinkFieldConverters[i].deserialize(flussField));
}
return genericRowData;
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
index 1f147751a..73ba044d8 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/row/FlinkAsFlussRowTest.java
@@ -31,7 +31,6 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.time.Instant;
-import java.util.stream.IntStream;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.AssertionsForClassTypes.offset;
@@ -60,7 +59,7 @@ class FlinkAsFlussRowTest {
TimestampData.fromEpochMillis(1672531200000L, 3),
new byte[] {1, 2, 3},
null);
- row = new FlinkAsFlussRow(IntStream.range(0, 14).toArray()).replace(flinkRow);
+ row = new FlinkAsFlussRow().replace(flinkRow);
}
@Test
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
index 596780302..9903e7c45 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/lookup/FlinkLookupFunctionTest.java
@@ -64,6 +64,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
tablePath,
flinkRowType,
createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ // no projection when job compiling, new column added after that.
null);
ListOutputCollector collector = new ListOutputCollector();
@@ -102,6 +103,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
tablePath,
flinkRowType,
createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
+ // no projection when job compiling, new column added after that.
null);
asyncLookupFunction.open(null);
@@ -154,7 +156,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
tablePath,
flinkRowType,
createPrimaryKeyLookupNormalizer(new int[] {0}, flinkRowType),
- null);
+ new int[] {1, 0});
ListOutputCollector collector = new ListOutputCollector();
lookupFunction.setCollector(collector);
@@ -191,7 +193,7 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
.sorted()
.collect(Collectors.toList());
assertThat(result)
- .containsExactly("+I(0,name0)", "+I(1,name1)", "+I(2,name2)", "+I(3,name3)");
+ .containsExactly("+I(name0,0)", "+I(name1,1)", "+I(name2,2)", "+I(name3,3)");
lookupFunction.close();
// start lookup job after schema change.
@@ -227,8 +229,9 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
.schema(
Schema.newBuilder()
.primaryKey("id")
- .column("name", DataTypes.STRING())
.column("id", DataTypes.INT())
+ .column("name", DataTypes.STRING())
+ // added an extra column 'age'
.column("age", DataTypes.INT())
.build())
.distributedBy(DEFAULT_BUCKET_NUM, "id")
@@ -241,7 +244,8 @@ class FlinkLookupFunctionTest extends FlinkTestBase {
try (Table table = conn.getTable(tablePath)) {
UpsertWriter upsertWriter = table.newUpsert().createWriter();
for (int i = 0; i < rows; i++) {
- upsertWriter.upsert(schemaNotMatch ? row("name" + i, i, i) : row(i, "name" + i));
+ upsertWriter.upsert(
+ schemaNotMatch ? row(i, "name" + i, i * 2) : row(i, "name" + i));
}
upsertWriter.flush();
}
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
index 88e2cb539..6634236aa 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceReaderTest.java
@@ -178,6 +178,7 @@ class FlinkSourceReaderTest extends FlinkTestBase {
tablePath,
sourceOutputType,
context,
+ null,
new FlinkSourceReaderMetrics(context.metricGroup()),
recordEmitter,
null);
diff --git a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
index c30c9cf72..44bd1e98d 100644
--- a/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
+++ b/fluss-flink/fluss-flink-common/src/test/java/org/apache/fluss/flink/source/reader/FlinkSourceSplitReaderTest.java
@@ -79,6 +79,24 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
TableDescriptor descriptor1 = TableDescriptor.builder().schema(schema1).build();
createTable(tablePath1, descriptor1);
+ assertThatThrownBy(
+ () ->
+ new FlinkSourceSplitReader(
+ clientConf,
+ tablePath1,
+ DataTypes.ROW(
+ DataTypes.FIELD(
+ "id", DataTypes.BIGINT().copy(false)),
+ DataTypes.FIELD("name", DataTypes.STRING())),
+ new int[] {1, 0},
+ createMockSourceReaderMetrics(),
+ null))
+ .isInstanceOf(ValidationException.class)
+ .hasMessage(
+ "The Flink query schema is not matched to Fluss table schema. \n"
+ + "Flink query schema: ROW<`id` BIGINT NOT NULL, `name` STRING>\n"
+ + "Fluss table schema: ROW<`name` STRING, `id` BIGINT NOT NULL> (projection [1, 0])");
+
assertThatThrownBy(
() ->
new FlinkSourceSplitReader(
@@ -87,10 +105,11 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
DataTypes.ROW(
DataTypes.FIELD("name2", DataTypes.STRING()),
DataTypes.FIELD("id", DataTypes.BIGINT())),
+ null,
createMockSourceReaderMetrics(),
null))
- .isInstanceOf(ValidationException.class)
- .hasMessage("The field name2 is not found in the fluss table.");
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessage("Field name2 does not exist in the row type.");
FlinkSourceSplitReader flinkSourceSplitReader =
new FlinkSourceSplitReader(
@@ -99,9 +118,10 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
DataTypes.ROW(
DataTypes.FIELD("name", DataTypes.STRING()),
DataTypes.FIELD("id", DataTypes.BIGINT().copy(false))),
+ null,
createMockSourceReaderMetrics(),
null);
- assertThat(flinkSourceSplitReader.getProjectedFields()).isEqualTo(new int[] {1, 0});
+ assertThat(flinkSourceSplitReader.getProjectedFields()).isNull();
}
@Test
@@ -384,7 +404,7 @@ class FlinkSourceSplitReaderTest extends FlinkTestBase {
private FlinkSourceSplitReader createSplitReader(TablePath tablePath, RowType rowType) {
return new FlinkSourceSplitReader(
- clientConf, tablePath, rowType, createMockSourceReaderMetrics(), null);
+ clientConf, tablePath, rowType, null, createMockSourceReaderMetrics(), null);
}
private FlinkSourceReaderMetrics createMockSourceReaderMetrics() {
diff --git a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
index 9c86a92c1..5aad1d5bf 100644
--- a/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
+++ b/fluss-lake/fluss-lake-iceberg/src/main/java/org/apache/fluss/lake/iceberg/conf/HadoopUtils.java
@@ -79,8 +79,7 @@ public class HadoopUtils {
}
// Approach 3: Fluss configuration
- // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop
- // conf
+ // add all configuration key with prefix 'iceberg.hadoop.' in fluss conf to hadoop conf
for (String key : flussConfiguration.keySet()) {
for (String prefix : FLUSS_CONFIG_PREFIXES) {
if (key.startsWith(prefix)) {
@@ -109,7 +108,7 @@ public class HadoopUtils {
}
/**
- * Search Hadoop configuration files in the given path, andadd them to the configuration if
+ * Search Hadoop configuration files in the given path, and add them to the configuration if
* found.
*/
private static boolean addHadoopConfIfFound(
diff --git a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
index 133c23322..4b6d9de71 100644
--- a/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
+++ b/fluss-lake/fluss-lake-iceberg/src/test/java/org/apache/fluss/lake/iceberg/maintenance/IcebergRewriteITCase.java
@@ -147,7 +147,7 @@ class IcebergRewriteITCase extends FlinkIcebergTieringTestBase {
// add pos-delete and trigger compaction
rows = Arrays.asList(row(4, "v1"), row(4, "v2"));
flussRows.add(writeIcebergTableRecords(t1, t1Bucket, 6, false, rows).get(1));
- // rewritten files should fail to commit due to conflict,add check here
+ // rewritten files should fail to commit due to conflict, add check here
checkRecords(getIcebergRecords(t1), flussRows);
// 4 data file and 1 delete file
checkFileStatusInIcebergTable(t1, 4, true);
diff --git a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
index 7f4d6e84e..a09a50c09 100644
--- a/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
+++ b/fluss-rpc/src/main/java/org/apache/fluss/rpc/netty/client/ServerConnection.java
@@ -69,8 +69,7 @@ final class ServerConnection {
private final ServerNode node;
- // TODO: add max inflight requests limit like Kafka's
- // "max.in.flight.requests.per.connection"
+ // TODO: add max inflight requests limit like Kafka's "max.in.flight.requests.per.connection"
private final Map<Integer, InflightRequest> inflightRequests = MapUtils.newConcurrentHashMap();
private final CompletableFuture<Void> closeFuture = new CompletableFuture<>();
private final ConnectionMetrics connectionMetrics;
diff --git a/fluss-rpc/src/main/proto/FlussApi.proto b/fluss-rpc/src/main/proto/FlussApi.proto
index 9feb38dc1..a08eeaba4 100644
--- a/fluss-rpc/src/main/proto/FlussApi.proto
+++ b/fluss-rpc/src/main/proto/FlussApi.proto
@@ -918,33 +918,33 @@ message PbAlterConfig {
required int32 op_type = 3; // SET=0, DELETE=1, APPEND=2, SUBTRACT=3
}
-message PbAddColumn{
+message PbAddColumn {
required string column_name = 1;
required bytes data_type_json = 2;
optional string comment = 3;
required int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
}
-message PbDropColumn{
+message PbDropColumn {
required string column_name = 1;
}
-message PbRenameColumn{
+message PbRenameColumn {
required string old_column_name = 1;
required string new_column_name = 2;
}
-message PbModifyColumn{
+message PbModifyColumn {
required string column_name = 1;
- required bytes data_type_json = 2;
+ optional bytes data_type_json = 2;
optional string comment = 3;
optional int32 column_position_type = 4; // LAST=0,FIRST=1,AFTER=3
}
-message PbDescribeConfig{
+message PbDescribeConfig {
required string config_key = 1;
optional string config_value = 2;
required string config_source = 3;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
index ededf510f..82f3d5ec2 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/RpcServiceBase.java
@@ -384,7 +384,7 @@ public abstract class RpcServiceBase extends RpcGatewayService implements AdminR
@Override
public CompletableFuture<GetFileSystemSecurityTokenResponse> getFileSystemSecurityToken(
GetFileSystemSecurityTokenRequest request) {
- // TODO:add ACL for per-table in https://github.com/apache/fluss/issues/752
+ // TODO: add ACL for per-table in https://github.com/apache/fluss/issues/752
try {
// In order to avoid repeatedly obtaining security token, cache it for a while.
long currentTimeMs = System.currentTimeMillis();
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
index def57231d..b7c92289b 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManager.java
@@ -148,7 +148,7 @@ public class CompletedSnapshotStoreManager {
() -> getAllSnapshotSize(bucket));
} else {
LOG.warn(
- "Failed toadd bucketMetricGroup for tableBucket {} when creating completed snapshot.",
+ "Failed to add bucketMetricGroup for tableBucket {} when creating completed snapshot.",
bucket);
}
return snapshotStore;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
index fbe0986b1..d1d551121 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/CoordinatorEventProcessor.java
@@ -556,7 +556,6 @@ public class CoordinatorEventProcessor implements EventProcessor {
private void processCreateTable(CreateTableEvent createTableEvent) {
long tableId = createTableEvent.getTableInfo().getTableId();
- int schemaId = createTableEvent.getTableInfo().getSchemaId();
// skip the table if it already exists
if (coordinatorContext.containsTableId(tableId)) {
return;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
index 19b13aaa9..a074827d8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/MetadataManager.java
@@ -297,7 +297,7 @@ public class MetadataManager {
// first register a schema to the zk, if then register the table
// to zk fails, there's no harm to register a new schema to zk again
try {
- zookeeperClient.registerSchema(tablePath, tableToCreate.getSchema());
+ zookeeperClient.registerFirstSchema(tablePath, tableToCreate.getSchema());
} catch (Exception e) {
throw new FlussRuntimeException(
"Fail to register schema when creating table " + tablePath, e);
@@ -330,13 +330,9 @@ public class MetadataManager {
// validate the table column changes
if (!schemaChanges.isEmpty()) {
- UpdateSchema schemaUpdate = new SchemaUpdate(table);
- for (TableChange schemaChange : schemaChanges) {
- schemaUpdate = schemaUpdate.applySchemaChange(schemaChange);
- }
- Schema newSchema = schemaUpdate.getSchema();
+ Schema newSchema = SchemaUpdate.applySchemaChanges(table, schemaChanges);
// update the schema
- zookeeperClient.registerSchema(tablePath, newSchema);
+ zookeeperClient.registerSchema(tablePath, newSchema, table.getSchemaId() + 1);
}
} catch (Exception e) {
if (e instanceof TableNotExistException) {
@@ -485,7 +481,7 @@ public class MetadataManager {
if (toEnableDataLake) {
TableInfo newTableInfo = newTableRegistration.toTableInfo(tablePath, schemaInfo);
- // if the table is lake table, we need toadd it to lake table tiering manager
+ // if the table is lake table, we need to add it to lake table tiering manager
lakeTableTieringManager.addNewLakeTable(newTableInfo);
} else if (toDisableDataLake) {
lakeTableTieringManager.removeLakeTable(newTableRegistration.tableId);
@@ -580,14 +576,14 @@ public class MetadataManager {
zookeeperClient.getTables(tablePaths);
// currently, we don't support schema evolution, so all schemas are version 1
Map<TablePath, SchemaInfo> tablePath2SchemaInfos =
- zookeeperClient.getV1Schemas(tablePaths);
+ zookeeperClient.getLatestSchemas(tablePaths);
for (TablePath tablePath : tablePaths) {
if (!tablePath2TableRegistrations.containsKey(tablePath)) {
throw new TableNotExistException("Table '" + tablePath + "' does not exist.");
}
if (!tablePath2SchemaInfos.containsKey(tablePath)) {
throw new SchemaNotExistException(
- "Schema for '" + tablePath + "' with schema_id=1 does not exist.");
+ "Schema for '" + tablePath + "' does not exist.");
}
TableRegistration tableReg = tablePath2TableRegistrations.get(tablePath);
SchemaInfo schemaInfo = tablePath2SchemaInfos.get(tablePath);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
index 1cdc702d7..9efab2a51 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/SchemaUpdate.java
@@ -29,12 +29,20 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
/** Schema update. */
-public class SchemaUpdate implements UpdateSchema {
+public class SchemaUpdate {
+
+ /** Apply schema changes to the given table info and return the updated schema. */
+ public static Schema applySchemaChanges(TableInfo tableInfo, List<TableChange> changes) {
+ SchemaUpdate schemaUpdate = new SchemaUpdate(tableInfo);
+ for (TableChange change : changes) {
+ schemaUpdate = schemaUpdate.applySchemaChange(change);
+ }
+ return schemaUpdate.getSchema();
+ }
+
private final List<Schema.Column> columns;
private final AtomicInteger highestFieldId;
private final List<String> primaryKeys;
- private final List<String> bucketKeys;
- private final List<String> partitionKeys;
private final Map<String, Schema.Column> existedColumns;
public SchemaUpdate(TableInfo tableInfo) {
@@ -42,15 +50,12 @@ public class SchemaUpdate implements UpdateSchema {
this.existedColumns = new HashMap<>();
this.highestFieldId = new AtomicInteger(tableInfo.getSchema().getHighestFieldId());
this.primaryKeys = tableInfo.getPrimaryKeys();
- this.bucketKeys = tableInfo.getBucketKeys();
- this.partitionKeys = tableInfo.getPartitionKeys();
this.columns.addAll(tableInfo.getSchema().getColumns());
for (Schema.Column column : columns) {
existedColumns.put(column.getName(), column);
}
}
- @Override
public Schema getSchema() {
Schema.Builder builder =
Schema.newBuilder()
@@ -63,8 +68,7 @@ public class SchemaUpdate implements UpdateSchema {
return builder.build();
}
- @Override
- public UpdateSchema applySchemaChange(TableChange columnChange) {
+ public SchemaUpdate applySchemaChange(TableChange columnChange) {
if (columnChange instanceof TableChange.AddColumn) {
return addColumn((TableChange.AddColumn) columnChange);
} else if (columnChange instanceof TableChange.ModifyColumn) {
@@ -78,7 +82,7 @@ public class SchemaUpdate implements UpdateSchema {
"Unknown column change type " + columnChange.getClass().getName());
}
- private UpdateSchema addColumn(TableChange.AddColumn addColumn) {
+ private SchemaUpdate addColumn(TableChange.AddColumn addColumn) {
if (existedColumns.containsKey(addColumn.getName())) {
throw new IllegalArgumentException(
"Column " + addColumn.getName() + " already exists.");
@@ -105,15 +109,15 @@ public class SchemaUpdate implements UpdateSchema {
return this;
}
- private UpdateSchema dropColumn(TableChange.DropColumn dropColumn) {
+ private SchemaUpdate dropColumn(TableChange.DropColumn dropColumn) {
throw new SchemaChangeException("Not support drop column now.");
}
- private UpdateSchema modifiedColumn(TableChange.ModifyColumn modifyColumn) {
+ private SchemaUpdate modifiedColumn(TableChange.ModifyColumn modifyColumn) {
throw new SchemaChangeException("Not support modify column now.");
}
- private UpdateSchema renameColumn(TableChange.RenameColumn renameColumn) {
+ private SchemaUpdate renameColumn(TableChange.RenameColumn renameColumn) {
throw new SchemaChangeException("Not support rename column now.");
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
deleted file mode 100644
index 1d51704ac..000000000
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/UpdateSchema.java
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.fluss.server.coordinator;
-
-import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.metadata.TableChange;
-
-/** Schema update. */
-public interface UpdateSchema {
- Schema getSchema();
-
- UpdateSchema applySchemaChange(TableChange columnChange);
-}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
index acfb7e689..09c273e7d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/coordinator/event/watcher/TableChangeWatcher.java
@@ -246,16 +246,7 @@ public class TableChangeWatcher {
private void processSchemaChange(TablePath tablePath, int schemaId) {
try {
- int currentSchemaId = zooKeeperClient.getCurrentSchemaId(tablePath);
SchemaInfo schemaInfo;
- if (schemaId != currentSchemaId) {
- LOG.warn(
- "Schema id {} is not equal to current schema id {}. Skipping schema change processing.",
- schemaId,
- currentSchemaId);
- return;
- }
-
Optional<SchemaInfo> optSchema = zooKeeperClient.getSchemaById(tablePath, schemaId);
if (!optSchema.isPresent()) {
LOG.error("No schema for table {} in zookeeper.", tablePath);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
index fad96a596..460562bc1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/entity/BatchRegisterLeadAndIsr.java
@@ -55,7 +55,7 @@ public class BatchRegisterLeadAndIsr {
tableBucket, leaderAndIsr, partitionName, liveReplicas));
} else {
throw new IllegalArgumentException(
- "Try toadd a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "Try to add a bucket with different tableId or partitionId in collection when try to batch register to Zookeeper."
+ "batch tableId="
+ tableId
+ " batch partitionId:"
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
index 1f1c4192d..96bb0eba8 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvRecoverHelper.java
@@ -66,15 +66,6 @@ public class KvRecoverHelper {
private InternalRow.FieldGetter[] currentFieldGetters;
- public KvRecoverHelper(
- KvTablet kvTablet,
- LogTablet logTablet,
- long recoverPointOffset,
- KvRecoverContext recoverContext,
- KvFormat kvFormat) {
- throw new UnsupportedOperationException();
- }
-
public KvRecoverHelper(
KvTablet kvTablet,
LogTablet logTablet,
@@ -136,64 +127,52 @@ public class KvRecoverHelper {
FetchIsolation fetchIsolation,
ThrowingConsumer<KeyValueAndLogOffset, Exception> resumeRecordConsumer)
throws Exception {
- long nextFetchOffset = startFetchOffset;
- while (true) {
- LogRecords logRecords =
- logTablet
- .read(
- nextFetchOffset,
- recoverContext.maxFetchLogSizeInRecoverKv,
- fetchIsolation,
- true,
- null)
- .getRecords();
- if (logRecords == MemoryLogRecords.EMPTY) {
- break;
- }
-
- for (LogRecordBatch logRecordBatch : logRecords.batches()) {
- // short schemaId = logRecordBatch.schemaId();
- // if (currentSchemaId == null) {
- // initSchema(schemaId);
- // } else if (currentSchemaId != schemaId) {
- // throw new KvStorageException(
- // String.format(
- // "Can't recover kv tablet for table bucket from
- // log %s since the schema changes from schema id %d to schema id %d. "
- // + "Currently, schema change is not
- // supported.",
- // recoverContext.tableBucket, currentSchemaId,
- // schemaId));
- // }
+ try (LogRecordReadContext readContext =
+ LogRecordReadContext.createArrowReadContext(
+ currentRowType, currentSchemaId, schemaGetter)) {
+ long nextFetchOffset = startFetchOffset;
+ while (true) {
+ LogRecords logRecords =
+ logTablet
+ .read(
+ nextFetchOffset,
+ recoverContext.maxFetchLogSizeInRecoverKv,
+ fetchIsolation,
+ true,
+ null)
+ .getRecords();
+ if (logRecords == MemoryLogRecords.EMPTY) {
+ break;
+ }
- // todo: currentRowType和currentSchemaId无法对齐
- try (LogRecordReadContext readContext =
- LogRecordReadContext.createArrowReadContext(
- currentRowType, currentSchemaId, schemaGetter);
- CloseableIterator<LogRecord> logRecordIter =
- logRecordBatch.records(readContext)) {
- while (logRecordIter.hasNext()) {
- LogRecord logRecord = logRecordIter.next();
- if (logRecord.getChangeType() != ChangeType.UPDATE_BEFORE) {
- InternalRow logRow = logRecord.getRow();
- byte[] key = keyEncoder.encodeKey(logRow);
- byte[] value = null;
- if (logRecord.getChangeType() != ChangeType.DELETE) {
- // the log row format may not compatible with kv row format,
- // e.g, arrow vs. compacted, thus needs a conversion here.
- BinaryRow row = toKvRow(logRecord.getRow());
- // todo: short value是否会有问题,感觉可以先check一下
- value = ValueEncoder.encodeValue(currentSchemaId.shortValue(), row);
+ for (LogRecordBatch logRecordBatch : logRecords.batches()) {
+ try (CloseableIterator<LogRecord> logRecordIter =
+ logRecordBatch.records(readContext)) {
+ while (logRecordIter.hasNext()) {
+ LogRecord logRecord = logRecordIter.next();
+ if (logRecord.getChangeType() != ChangeType.UPDATE_BEFORE) {
+ InternalRow logRow = logRecord.getRow();
+ byte[] key = keyEncoder.encodeKey(logRow);
+ byte[] value = null;
+ if (logRecord.getChangeType() != ChangeType.DELETE) {
+ // the log row format may not compatible with kv row format,
+ // e.g, arrow vs. compacted, thus needs a conversion here.
+ BinaryRow row = toKvRow(logRecord.getRow());
+ value =
+ ValueEncoder.encodeValue(
+ currentSchemaId.shortValue(), row);
+ }
+ resumeRecordConsumer.accept(
+ new KeyValueAndLogOffset(
+ key, value, logRecord.logOffset()));
}
- resumeRecordConsumer.accept(
- new KeyValueAndLogOffset(key, value, logRecord.logOffset()));
}
}
+ nextFetchOffset = logRecordBatch.nextLogOffset();
}
- nextFetchOffset = logRecordBatch.nextLogOffset();
}
+ return nextFetchOffset;
}
- return nextFetchOffset;
}
// TODO: this is very in-efficient, because the conversion is CPU heavy. Should be optimized in
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
index 176e749f5..b6957c5dc 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/KvTablet.java
@@ -34,16 +34,16 @@ import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.metadata.TableBucket;
import org.apache.fluss.metadata.TablePath;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.record.ChangeType;
import org.apache.fluss.record.KvRecord;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.record.KvRecordReadContext;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.row.PaddingRow;
import org.apache.fluss.row.arrow.ArrowWriterPool;
import org.apache.fluss.row.arrow.ArrowWriterProvider;
import org.apache.fluss.row.encode.ValueDecoder;
-import org.apache.fluss.row.encode.ValueEncoder;
import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer;
import org.apache.fluss.server.kv.prewrite.KvPreWriteBuffer.TruncateReason;
import org.apache.fluss.server.kv.rocksdb.RocksDBKv;
@@ -83,7 +83,6 @@ import java.util.concurrent.Executor;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import static org.apache.fluss.utils.SchemaUtil.getTargetColumns;
import static org.apache.fluss.utils.concurrent.LockUtils.inReadLock;
import static org.apache.fluss.utils.concurrent.LockUtils.inWriteLock;
@@ -255,6 +254,20 @@ public final class KvTablet {
/**
* Put the KvRecordBatch into the kv storage, and return the appended wal log info.
*
+ * <p>Schema Evolution Handling:
+ *
+ * <p>We don't allow shema of input kv records to be larger than the latest schema id known by
+ * the tablet. Besides, we currently only support ADD COLUMN LAST operation, so the input row or
+ * old row must have same or fewer columns than latest schema. This helps to simplify the schema
+ * change handling.
+ *
+ * <p>1. We write the kv records into KvStore without converting it into latest schema for
+ * performance consideration. We have mechanisms that writer client dynamically use latest
+ * schema for writing records.
+ *
+ * <p>2. We always use the latest schema for writing WAL logs, because it anyway happens
+ * deserialization&serialization to convert the compacted format into Arrow format.
+ *
* @param kvRecords the kv records to put into
* @param targetColumns the target columns to put, null if put all columns
*/
@@ -267,32 +280,28 @@ public final class KvTablet {
SchemaInfo schemaInfo = schemaGetter.getLatestSchemaInfo();
Schema latestSchema = schemaInfo.getSchema();
- // In the following scenario, we need to support schema migration:
- // schema1: old data's schema -> valueDecoder
- // schema2: new data's schema
- // schema3: current schema (also the latest schema)
- // schema1 <= schema2 <= current schema id
short latestSchemaId = (short) schemaInfo.getSchemaId();
short schemaIdOfNewData = kvRecords.schemaId();
if (schemaIdOfNewData > latestSchemaId || schemaIdOfNewData < 0) {
+ // TODO: we may need to support retriable exception here
throw new SchemaNotExistException(
"Invalid schema id: "
+ schemaIdOfNewData
+ ", latest schema id: "
+ latestSchemaId);
}
- Schema schemaOfNewData = schemaGetter.getSchema(schemaIdOfNewData);
+
+ // we only support ADD COLUMN, so targetColumns is fine to be used directly
RowMerger currentMerger =
rowMerger.configureTargetColumns(
- schemaIdOfNewData == latestSchemaId
- ? targetColumns
- : getTargetColumns(
- targetColumns, schemaOfNewData, latestSchema),
- latestSchemaId,
- latestSchema);
- RowType currentRowType = latestSchema.getRowType();
- WalBuilder walBuilder = createWalBuilder(latestSchemaId, currentRowType);
+ targetColumns, latestSchemaId, latestSchema);
+ RowType latestRowType = latestSchema.getRowType();
+ WalBuilder walBuilder = createWalBuilder(latestSchemaId, latestRowType);
walBuilder.setWriterState(kvRecords.writerId(), kvRecords.batchSequence());
+ // we only support ADD COLUMN LAST, so the BinaryRow after RowMerger is
+ // only has fewer ending columns than latest schema, so we pad nulls to
+ // the end of the BinaryRow to get the latest schema row.
+ PaddingRow latestSchemaRow = new PaddingRow(latestRowType.getFieldCount());
// get offset to track the offset corresponded to the kv record
long logEndOffsetOfPrevBatch = logTablet.localLogEndOffset();
try {
@@ -301,22 +310,15 @@ public final class KvTablet {
// TODO: reuse the read context and decoder
KvRecordBatch.ReadContext readContext =
KvRecordReadContext.createReadContext(kvFormat, schemaGetter);
-
- // replace new row with current schema if the schema id is older than now
- ProjectedRow projectedRow =
- ProjectedRow.from(schemaOfNewData, latestSchema);
ValueDecoder valueDecoder = new ValueDecoder(schemaGetter, kvFormat);
for (KvRecord kvRecord : kvRecords.records(readContext)) {
-
byte[] keyBytes = BytesUtils.toArray(kvRecord.getKey());
KvPreWriteBuffer.Key key = KvPreWriteBuffer.Key.of(keyBytes);
- InternalRow row = kvRecord.getRow();
- if (row != null && schemaIdOfNewData < latestSchemaId) {
- row = projectedRow.replaceRow(row);
- }
-
- if (row == null) {
- DeleteBehavior deleteBehavior = rowMerger.deleteBehavior();
+ BinaryRow row = kvRecord.getRow();
+ BinaryValue currentValue =
+ row == null ? null : new BinaryValue(schemaIdOfNewData, row);
+ if (currentValue == null) {
+ DeleteBehavior deleteBehavior = currentMerger.deleteBehavior();
if (deleteBehavior == DeleteBehavior.IGNORE) {
// skip delete rows if the merger doesn't support yet
continue;
@@ -326,99 +328,69 @@ public final class KvTablet {
+ "The table.delete.behavior is set to 'disable'.");
}
// it's for deletion
- byte[] oldValue = getFromBufferOrKv(key);
+ byte[] oldValueBytes = getFromBufferOrKv(key);
- if (oldValue == null) {
+ if (oldValueBytes == null) {
// there might be large amount of such deletion, so we don't log
LOG.debug(
"The specific key can't be found in kv tablet although the kv record is for deletion, "
+ "ignore it directly as it doesn't exist in the kv tablet yet.");
} else {
- ValueDecoder.Value oldRowAndSchemaId =
- valueDecoder.decodeValue(oldValue);
- InternalRow oldRow = oldRowAndSchemaId.row;
- if (oldRow != null
- && oldRowAndSchemaId.schemaId != latestSchemaId) {
- // todo: 后续想办法复用对应的projected row.
- Schema schemaOfOldData =
- schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
- oldRow =
- ProjectedRow.from(schemaOfOldData, latestSchema)
- .replaceRow(oldRow);
- }
- InternalRow newRow = currentMerger.delete(oldRow);
+ BinaryValue oldValue = valueDecoder.decodeValue(oldValueBytes);
+ BinaryValue newValue = currentMerger.delete(oldValue);
// if newRow is null, it means the row should be deleted
- if (newRow == null) {
- walBuilder.append(ChangeType.DELETE, oldRow);
+ if (newValue == null) {
+ walBuilder.append(
+ ChangeType.DELETE,
+ latestSchemaRow.replaceRow(oldValue.row));
kvPreWriteBuffer.delete(key, logOffset++);
} else {
// otherwise, it's a partial update, should produce -U,+U
- walBuilder.append(ChangeType.UPDATE_BEFORE, oldRow);
- walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
+ walBuilder.append(
+ ChangeType.UPDATE_BEFORE,
+ latestSchemaRow.replaceRow(oldValue.row));
+ walBuilder.append(
+ ChangeType.UPDATE_AFTER,
+ latestSchemaRow.replaceRow(newValue.row));
kvPreWriteBuffer.put(
- key,
- ValueEncoder.encodeRow(
- latestSchemaId,
- kvFormat,
- currentRowType,
- newRow),
- logOffset + 1);
+ key, newValue.encodeValue(), logOffset + 1);
logOffset += 2;
}
}
} else {
// upsert operation
- byte[] oldValue = getFromBufferOrKv(key);
+ byte[] oldValueBytes = getFromBufferOrKv(key);
// it's update
- if (oldValue != null) {
- ValueDecoder.Value oldRowAndSchemaId =
- valueDecoder.decodeValue(oldValue);
- InternalRow oldRow = oldRowAndSchemaId.row;
- if (oldRow != null
- && oldRowAndSchemaId.schemaId != latestSchemaId) {
- // todo: 后续想办法复用对应的projected row.
- Schema schemaOfOldData =
- schemaGetter.getSchema(oldRowAndSchemaId.schemaId);
- oldRow =
- ProjectedRow.from(schemaOfOldData, latestSchema)
- .replaceRow(oldRow);
- }
-
- InternalRow newRow = currentMerger.merge(oldRow, row);
- if (newRow == oldRow) {
- // newRow is the same to oldRow, means nothing
+ if (oldValueBytes != null) {
+ BinaryValue oldValue = valueDecoder.decodeValue(oldValueBytes);
+ BinaryValue newValue =
+ currentMerger.merge(oldValue, currentValue);
+ if (newValue == oldValue) {
+ // newValue is the same to oldValue, means nothing
// happens (no update/delete), and input should be ignored
continue;
}
- walBuilder.append(ChangeType.UPDATE_BEFORE, oldRow);
- walBuilder.append(ChangeType.UPDATE_AFTER, newRow);
+ walBuilder.append(
+ ChangeType.UPDATE_BEFORE,
+ latestSchemaRow.replaceRow(oldValue.row));
+ walBuilder.append(
+ ChangeType.UPDATE_AFTER,
+ latestSchemaRow.replaceRow(newValue.row));
// logOffset is for -U, logOffset + 1 is for +U, we need to use
// the log offset for +U
kvPreWriteBuffer.put(
- key,
- ValueEncoder.encodeRow(
- latestSchemaId,
- kvFormat,
- currentRowType,
- newRow),
- logOffset + 1);
+ key, newValue.encodeValue(), logOffset + 1);
logOffset += 2;
} else {
// it's insert
- // TODO: we shouldadd guarantees that all non-specified
- // columns
+ // TODO: we should add guarantees that all non-specified columns
// of the input row are set to null.
- InternalRow newRow = row;
- walBuilder.append(ChangeType.INSERT, newRow);
+ walBuilder.append(
+ ChangeType.INSERT,
+ latestSchemaRow.replaceRow(currentValue.row));
kvPreWriteBuffer.put(
- key,
- ValueEncoder.encodeRow(
- latestSchemaId,
- kvFormat,
- currentRowType,
- newRow),
- logOffset++);
+ key, currentValue.encodeValue(), logOffset++);
}
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java
index 688ac8dab..d8741e010 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdater.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.partialupdate;
import org.apache.fluss.exception.InvalidTargetColumnException;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.encode.RowEncoder;
import org.apache.fluss.types.DataType;
@@ -34,6 +34,7 @@ import java.util.BitSet;
@NotThreadSafe
public class PartialUpdater {
+ private final short targetSchemaId;
private final InternalRow.FieldGetter[] flussFieldGetters;
private final RowEncoder rowEncoder;
@@ -42,7 +43,8 @@ public class PartialUpdater {
private final BitSet primaryKeyCols = new BitSet();
private final DataType[] fieldDataTypes;
- public PartialUpdater(KvFormat kvFormat, Schema schema, int[] targetColumns) {
+ public PartialUpdater(KvFormat kvFormat, short schemaId, Schema schema, int[] targetColumns) {
+ this.targetSchemaId = schemaId;
for (int targetColumn : targetColumns) {
partialUpdateCols.set(targetColumn);
}
@@ -89,43 +91,44 @@ public class PartialUpdater {
}
/**
- * Partial update the {@code oldRow} with the given new row {@code partialRow}. The {@code
- * oldRow} may be null, in this case, the field don't exist in the {@code partialRow} will be
+ * Partial update the {@code oldValue} with the given new row {@code partialValue}. The {@code
+ * oldValue} may be null, in this case, the field don't exist in the {@code partialRow} will be
* set to null.
*
- * @param oldRow the old row to be updated
- * @param partialRow the new row to be updated.
- * @return the updated row
+ * @param oldValue the old value to be updated
+ * @param partialValue the new value to be updated.
+ * @return the updated value (schema id + row bytes)
*/
- public BinaryRow updateRow(@Nullable InternalRow oldRow, InternalRow partialRow) {
+ public BinaryValue updateRow(@Nullable BinaryValue oldValue, BinaryValue partialValue) {
rowEncoder.startNewRow();
// write each field
for (int i = 0; i < fieldDataTypes.length; i++) {
// use the partial row value
if (partialUpdateCols.get(i)) {
- rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(partialRow));
+ rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(partialValue.row));
} else {
// use the old row value
- if (oldRow == null) {
+ if (oldValue == null) {
rowEncoder.encodeField(i, null);
} else {
- rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(oldRow));
+ rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(oldValue.row));
}
}
}
- return rowEncoder.finishRow();
+ return new BinaryValue(targetSchemaId, rowEncoder.finishRow());
}
/**
- * Partial delete the given {@code row}. If all the fields except for {@link #partialUpdateCols}
- * in {@code row} are null, return null. Otherwise, update all the {@link #partialUpdateCols} in
- * the {@code row} except for the primary key columns to null values, return the updated row.
+ * Partial delete the given {@code value}. If all the fields except for {@link
+ * #partialUpdateCols} in {@code value.row} are null, return null. Otherwise, update all the
+ * {@link #partialUpdateCols} in the {@code value.row} except for the primary key columns to
+ * null values, return the updated value.
*
- * @param row the row to be deleted
- * @return the row after partial deleted
+ * @param value the value to be deleted
+ * @return the value after partial deleted
*/
- public @Nullable BinaryRow deleteRow(InternalRow row) {
- if (isFieldsNull(row, partialUpdateCols)) {
+ public @Nullable BinaryValue deleteRow(BinaryValue value) {
+ if (isFieldsNull(value.row, partialUpdateCols)) {
return null;
} else {
rowEncoder.startNewRow();
@@ -137,10 +140,10 @@ public class PartialUpdater {
rowEncoder.encodeField(i, null);
} else {
// use the old row value
- rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(row));
+ rowEncoder.encodeField(i, flussFieldGetters[i].getFieldOrNull(value.row));
}
}
- return rowEncoder.finishRow();
+ return new BinaryValue(targetSchemaId, rowEncoder.finishRow());
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
index 3ce22854c..0c227aaf2 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/partialupdate/PartialUpdaterCache.java
@@ -25,6 +25,7 @@ import com.github.benmanes.caffeine.cache.Caffeine;
import javax.annotation.concurrent.ThreadSafe;
+import java.time.Duration;
import java.util.Arrays;
/** The cache for {@link PartialUpdater}. */
@@ -37,7 +38,11 @@ public class PartialUpdaterCache {
// currently, the cache is used per-bucket, so we limit the cache size to 5 to have a
// maximal 5 parallel partial updaters. This is a temporary solution and should be
// shared across all buckets in the future.
- this.rowPartialUpdaters = Caffeine.newBuilder().maximumSize(5).build();
+ this.rowPartialUpdaters =
+ Caffeine.newBuilder()
+ .maximumSize(5)
+ .expireAfterAccess(Duration.ofMinutes(5))
+ .build();
}
// TODO: extend to tableId and schemaId when the cache is shared across all tables
@@ -45,7 +50,7 @@ public class PartialUpdaterCache {
KvFormat kvFormat, short schemaId, Schema schema, int[] targetColumns) {
return rowPartialUpdaters.get(
getPartialUpdaterKey(targetColumns, schemaId),
- k -> new PartialUpdater(kvFormat, schema, targetColumns));
+ k -> new PartialUpdater(kvFormat, schemaId, schema, targetColumns));
}
private String getPartialUpdaterKey(int[] targetColumns, int schemaId) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
index af3500c59..2db73c819 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMerger.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.server.kv.partialupdate.PartialUpdater;
import org.apache.fluss.server.kv.partialupdate.PartialUpdaterCache;
@@ -46,14 +46,14 @@ public class DefaultRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
// always retain the new row (latest row)
- return newRow;
+ return newValue;
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
// returns null to indicate the row is deleted
return null;
}
@@ -98,13 +98,13 @@ public class DefaultRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
- return partialUpdater.updateRow(oldRow, newRow);
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
+ return partialUpdater.updateRow(oldValue, newValue);
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
return partialUpdater.deleteRow(oldRow);
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
index 79592e177..33bd6ee1f 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/FirstRowRowMerger.java
@@ -20,7 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.record.BinaryValue;
import javax.annotation.Nullable;
@@ -44,14 +44,14 @@ public class FirstRowRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
// always retain the old row (first row)
- return oldRow;
+ return oldValue;
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the first_row merge engine.");
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
index 5f3414574..89bff8c0a 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/RowMerger.java
@@ -23,7 +23,7 @@ import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.InternalRow;
+import org.apache.fluss.record.BinaryValue;
import javax.annotation.Nullable;
@@ -33,14 +33,14 @@ import java.util.Optional;
public interface RowMerger {
/**
- * Merge the old row with the new row.
+ * Merge the old value with the new value.
*
- * @param oldRow the old row
- * @param newRow the new row
- * @return the merged row, if the returned row is the same to the old row, then nothing happens
- * to the row (no update, no delete).
+ * @param oldValue the old value
+ * @param newValue the new row
+ * @return the merged value, if the returned row is the same to the old row, then nothing
+ * happens to the row (no update, no delete).
*/
- InternalRow merge(InternalRow oldRow, InternalRow newRow);
+ BinaryValue merge(BinaryValue oldValue, BinaryValue newValue);
/**
* Merge the old row with a delete row.
@@ -52,7 +52,7 @@ public interface RowMerger {
* @return the merged row, or null if the row is deleted.
*/
@Nullable
- InternalRow delete(InternalRow oldRow);
+ BinaryValue delete(BinaryValue oldRow);
/**
* The behavior of delete operations on primary key tables.
@@ -61,7 +61,13 @@ public interface RowMerger {
*/
DeleteBehavior deleteBehavior();
- /** Dynamically configure the target columns to merge and return the effective merger. */
+ /**
+ * Dynamically configure the target columns to merge and return the effective merger.
+ *
+ * @param targetColumns the partial update target column positions, null means full update
+ * @param schemaId the schema id used to generate new rows
+ * @param schema the schema used to generate new rows
+ */
RowMerger configureTargetColumns(@Nullable int[] targetColumns, short schemaId, Schema schema);
/** Create a row merger based on the given configuration. */
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
index 9706230b5..4b555602e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMerger.java
@@ -20,6 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.MergeEngineType;
import org.apache.fluss.metadata.Schema;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
@@ -64,14 +65,14 @@ public class VersionedRowMerger implements RowMerger {
@Nullable
@Override
- public InternalRow merge(InternalRow oldRow, InternalRow newRow) {
+ public BinaryValue merge(BinaryValue oldValue, BinaryValue newValue) {
// return newRow if newRow's version is larger or equal than oldRow's version
- return versionComparator.compare(oldRow, newRow) <= 0 ? newRow : oldRow;
+ return versionComparator.compare(oldValue.row, newValue.row) <= 0 ? newValue : oldValue;
}
@Nullable
@Override
- public InternalRow delete(InternalRow oldRow) {
+ public BinaryValue delete(BinaryValue oldRow) {
throw new UnsupportedOperationException(
"DELETE is not supported for the versioned merge engine.");
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
index b8d941221..3e93cc46a 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStore.java
@@ -99,7 +99,7 @@ public class CompletedSnapshotStore {
* Synchronously writes the new snapshots to snapshot handle store and asynchronously removes
* older ones.
*
- * @param snapshot Completed snapshot toadd.
+ * @param snapshot Completed snapshot to add.
*/
@VisibleForTesting
CompletedSnapshot addSnapshotAndSubsumeOldestOne(
@@ -116,7 +116,7 @@ public class CompletedSnapshotStore {
completedSnapshotHandleStore.add(
snapshot.getTableBucket(), snapshot.getSnapshotID(), completedSnapshotHandle);
- // Nowadd the new one. If it fails, we don't want to lose existing data.
+ // Now add the new one. If it fails, we don't want to lose existing data.
completedSnapshots.addLast(snapshot);
// Remove completed snapshot from queue and snapshotStateHandleStore, not discard.
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
index e09c93b22..fe517f268 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/kv/snapshot/SnapshotLocation.java
@@ -328,7 +328,7 @@ public class SnapshotLocation {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: mayadd entropy injection?
+ // todo: may add entropy injection?
this.kvFilePath = createFilePath();
this.outStream = fs.create(kvFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
index 847bfa883..0ec42a090 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/FetchParams.java
@@ -21,6 +21,7 @@ import org.apache.fluss.annotation.VisibleForTesting;
import org.apache.fluss.compression.ArrowCompressionInfo;
import org.apache.fluss.metadata.SchemaGetter;
import org.apache.fluss.record.FileLogProjection;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.rpc.messages.FetchLogRequest;
import javax.annotation.Nullable;
@@ -66,7 +67,7 @@ public final class FetchParams {
private final int minFetchBytes;
private final long maxWaitMs;
- // TODO:add more params like epoch etc.
+ // TODO: add more params like epoch etc.
public FetchParams(int replicaId, int maxFetchBytes) {
this(replicaId, true, maxFetchBytes, DEFAULT_MIN_FETCH_BYTES, DEFAULT_MAX_WAIT_MS);
@@ -99,13 +100,14 @@ public final class FetchParams {
int maxFetchBytes,
SchemaGetter schemaGetter,
ArrowCompressionInfo compressionInfo,
- @Nullable int[] projectedFields) {
+ @Nullable int[] projectedFields,
+ ProjectionPushdownCache projectionCache) {
this.fetchOffset = fetchOffset;
this.maxFetchBytes = maxFetchBytes;
if (projectedFields != null) {
projectionEnabled = true;
if (fileLogProjection == null) {
- fileLogProjection = new FileLogProjection();
+ fileLogProjection = new FileLogProjection(projectionCache);
}
fileLogProjection.setCurrentProjection(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
index 086e23b0e..130b3855d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/LocalLog.java
@@ -405,8 +405,6 @@ public final class LocalLog {
(maxOffsetMetadata.getSegmentBaseOffset() == segment.getBaseOffset())
? maxOffsetMetadata.getRelativePositionInSegment()
: segment.getSizeInBytes();
-
- // segment需要查询
fetchDataInfo =
segment.read(readOffset, maxLength, maxPosition, minOneMessage, projection);
if (fetchDataInfo == null) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
index 05233e0ff..835a94ff6 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/FsRemoteLogOutputStream.java
@@ -203,7 +203,7 @@ public class FsRemoteLogOutputStream extends FSDataOutputStream {
Exception latestException = null;
for (int attempt = 0; attempt < 10; attempt++) {
try {
- // todo: mayadd entropy injection?
+ // todo: may add entropy injection?
this.outStream = fs.create(remoteLogFilePath, FileSystem.WriteMode.NO_OVERWRITE);
return;
} catch (Exception e) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
index 86a6bbf9e..73aee521e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogSegmentFiles.java
@@ -40,7 +40,7 @@ public class LogSegmentFiles {
private final Path offsetIndex;
private final Path timeIndex;
private final @Nullable Path writerIdIndex;
- // TODOadd leader epoch index after introduce leader epoch.
+ // TODO add leader epoch index after introduce leader epoch.
public LogSegmentFiles(
Path logSegment, Path offsetIndex, Path timeIndex, @Nullable Path writerIdIndex) {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
index 0b425b8fc..cdde3842d 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/log/remote/LogTieringTask.java
@@ -227,7 +227,7 @@ public class LogTieringTask implements Runnable {
}
/**
- * Copy the given log segments to remote andadd the successfully copied segment to the {@code
+ * Copy the given log segments to remote and add the successfully copied segment to the {@code
* copiedSegments} parameter.
*
* @return the end offset of the last segment copied to remote.
@@ -347,7 +347,7 @@ public class LogTieringTask implements Runnable {
// the commit failed, it means the commit snapshot is invalid or register zk
// failed, we will revert this commit and delete the remote log manifest
// file.
- // TODO:add the fail reason in the future.
+ // TODO: add the fail reason in the future.
LOG.error(
"Commit remote log manifest failed for table bucket {}. We will delete the"
+ " written remote log manifest file",
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
similarity index 68%
rename from fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
rename to fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
index f56429bf1..374e20501 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/SchemaMetadataManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/ServerSchemaCache.java
@@ -29,8 +29,11 @@ import org.apache.fluss.shaded.guava32.com.google.common.cache.CacheBuilder;
import org.apache.fluss.shaded.guava32.com.google.common.util.concurrent.UncheckedExecutionException;
import org.apache.fluss.utils.MapUtils;
+import javax.annotation.concurrent.ThreadSafe;
+
import java.util.Map;
import java.util.Objects;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutionException;
/**
@@ -40,74 +43,77 @@ import java.util.concurrent.ExecutionException;
* 1. latest schema of each subscribed table, updated by UpdateMetadata request. 2. history schemas
* of each table, updated by lookup from tablet server.
*/
-public class SchemaMetadataManager {
+@ThreadSafe
+public class ServerSchemaCache {
- private MetadataManager metadataManager;
- private final Map<TablePath, SchemaInfo> latestSchemaByTablePath;
- private final Map<TablePath, Integer> subscriberCounters;
+ private final MetadataManager metadataManager;
+ private final Map<Long, SchemaInfo> latestSchemaByTableId;
+ // table_id -> subscriber count
+ private final Map<Long, Integer> subscriberCounters;
private final Cache<TableSchemaKey, Schema> schemaCache;
- public SchemaMetadataManager(MetadataManager metadataManager) {
+ public ServerSchemaCache(MetadataManager metadataManager) {
this.metadataManager = metadataManager;
// thread safe is guaranteed by subscriberCounters.
this.subscriberCounters = MapUtils.newConcurrentHashMap();
- this.latestSchemaByTablePath = MapUtils.newConcurrentHashMap();
+ this.latestSchemaByTableId = MapUtils.newConcurrentHashMap();
this.schemaCache = CacheBuilder.newBuilder().maximumSize(100).build();
}
public SchemaGetter subscribeWithInitialSchema(
- TablePath tablePath, int initialSchemaId, Schema initialSchema) {
+ long tableId, TablePath tablePath, int initialSchemaId, Schema initialSchema) {
subscriberCounters.compute(
- tablePath,
+ tableId,
(key, oldValue) -> {
- updateSchema(tablePath, initialSchemaId, initialSchema);
+ updateSchema(tableId, initialSchemaId, initialSchema);
if (oldValue == null) {
return 1;
} else {
return oldValue + 1;
}
});
- return new SchemaMetadataSubscriberImpl(tablePath);
+ return new SchemaMetadataSubscriberImpl(tableId, tablePath);
}
- public void updateLatestSchema(TablePath tablePath, short schemaId, Schema schema) {
+ public void updateLatestSchema(long tableId, short schemaId, Schema schema) {
// only update if tablePath is subscribed.
subscriberCounters.computeIfPresent(
- tablePath,
+ tableId,
(key, oldValue) -> {
- updateSchema(tablePath, schemaId, schema);
+ updateSchema(tableId, schemaId, schema);
return oldValue;
});
}
- private void unsubscribe(TablePath tablePath) {
+ private void unsubscribe(long tableId) {
subscriberCounters.compute(
- tablePath,
+ tableId,
(key, oldValue) -> {
if (oldValue != null && oldValue > 1) {
return oldValue - 1;
}
- latestSchemaByTablePath.remove(tablePath);
+ latestSchemaByTableId.remove(tableId);
return null;
});
}
- private Schema getFlussSchema(TablePath tablePath, int schemaId) throws ExecutionException {
+ private Schema getFlussSchema(long tableId, TablePath tablePath, int schemaId)
+ throws ExecutionException {
return schemaCache.get(
- new TableSchemaKey(tablePath, schemaId),
+ new TableSchemaKey(tableId, schemaId),
() -> {
- SchemaInfo schemaInfo = latestSchemaByTablePath.get(tablePath);
+ SchemaInfo schemaInfo = latestSchemaByTableId.get(tableId);
if (schemaInfo != null && schemaInfo.getSchemaId() == schemaId) {
- return latestSchemaByTablePath.get(tablePath).getSchema();
+ return latestSchemaByTableId.get(tableId).getSchema();
} else {
return metadataManager.getSchemaById(tablePath, schemaId).getSchema();
}
});
}
- private void updateSchema(TablePath tablePath, int schemaId, Schema schema) {
- latestSchemaByTablePath.compute(
- tablePath,
+ private void updateSchema(long tableId, int schemaId, Schema schema) {
+ latestSchemaByTableId.compute(
+ tableId,
(key, oldValue) -> {
if (oldValue == null || oldValue.getSchemaId() < schemaId) {
return new SchemaInfo(schema, schemaId);
@@ -116,25 +122,25 @@ public class SchemaMetadataManager {
}
});
- schemaCache.put(new TableSchemaKey(tablePath, schemaId), schema);
+ schemaCache.put(new TableSchemaKey(tableId, schemaId), schema);
}
@VisibleForTesting
- public Map<TablePath, SchemaInfo> getLatestSchemaByTablePath() {
- return latestSchemaByTablePath;
+ public Map<Long, SchemaInfo> getLatestSchemaByTableId() {
+ return latestSchemaByTableId;
}
@VisibleForTesting
- Map<TablePath, Integer> getSubscriberCounters() {
+ Map<Long, Integer> getSubscriberCounters() {
return subscriberCounters;
}
private static class TableSchemaKey {
- private final TablePath tablePath;
+ private final long tableId;
private final int schemaId;
- TableSchemaKey(TablePath tablePath, int schemaId) {
- this.tablePath = tablePath;
+ TableSchemaKey(long tableId, int schemaId) {
+ this.tableId = tableId;
this.schemaId = schemaId;
}
@@ -144,34 +150,35 @@ public class SchemaMetadataManager {
return false;
}
TableSchemaKey that = (TableSchemaKey) o;
- return schemaId == that.schemaId && Objects.equals(tablePath, that.tablePath);
+ return schemaId == that.schemaId && Objects.equals(tableId, that.tableId);
}
@Override
public int hashCode() {
- return Objects.hash(tablePath, schemaId);
+ return Objects.hash(tableId, schemaId);
}
}
private class SchemaMetadataSubscriberImpl implements SchemaGetter {
+ private final long tableId;
private final TablePath tablePath;
private volatile boolean released;
- public SchemaMetadataSubscriberImpl(TablePath tablePath) {
+ public SchemaMetadataSubscriberImpl(long tableId, TablePath tablePath) {
+ this.tableId = tableId;
this.tablePath = tablePath;
this.released = false;
}
@Override
public SchemaInfo getLatestSchemaInfo() {
- return latestSchemaByTablePath.get(tablePath);
+ return latestSchemaByTableId.get(tableId);
}
@Override
public Schema getSchema(int schemaId) {
- // todo: support with get table.
try {
- return getFlussSchema(tablePath, schemaId);
+ return getFlussSchema(tableId, tablePath, schemaId);
} catch (ExecutionException | UncheckedExecutionException executionException) {
Throwable cause = executionException.getCause();
if (cause instanceof SchemaNotExistException) {
@@ -182,10 +189,16 @@ public class SchemaMetadataManager {
}
}
+ @Override
+ public CompletableFuture<SchemaInfo> getSchemaInfoAsync(int schemaId) {
+ // TODO: make it async using async zookeeper API.
+ return CompletableFuture.completedFuture(new SchemaInfo(getSchema(schemaId), schemaId));
+ }
+
@Override
public void release() {
if (!released) {
- unsubscribe(tablePath);
+ unsubscribe(tableId);
released = true;
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
index 6a11a53ae..f4ecff6f0 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/metadata/TabletServerMetadataCache.java
@@ -68,13 +68,12 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
private final MetadataManager metadataManager;
- private final SchemaMetadataManager schemaMetadataManager;
+ private final ServerSchemaCache serverSchemaCache;
- // todo: replace this in test with schemaMetadataManager.
public TabletServerMetadataCache(MetadataManager metadataManager) {
this.serverMetadataSnapshot = ServerMetadataSnapshot.empty();
this.metadataManager = metadataManager;
- this.schemaMetadataManager = new SchemaMetadataManager(metadataManager);
+ this.serverSchemaCache = new ServerSchemaCache(metadataManager);
}
@Override
@@ -141,14 +140,14 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
}
public SchemaGetter subscribeWithInitialSchema(
- TablePath tablePath, int initialSchemaId, Schema initialSchema) {
- return schemaMetadataManager.subscribeWithInitialSchema(
- tablePath, initialSchemaId, initialSchema);
+ TablePath tablePath, long tableId, int initialSchemaId, Schema initialSchema) {
+ return serverSchemaCache.subscribeWithInitialSchema(
+ tableId, tablePath, initialSchemaId, initialSchema);
}
- public void updateLatestSchema(TablePath tablePath, SchemaInfo schemaInfo) {
- schemaMetadataManager.updateLatestSchema(
- tablePath, (short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
+ public void updateLatestSchema(long tableId, SchemaInfo schemaInfo) {
+ serverSchemaCache.updateLatestSchema(
+ tableId, (short) schemaInfo.getSchemaId(), schemaInfo.getSchema());
}
public Optional<PartitionMetadata> getPartitionMetadata(PhysicalTablePath partitionPath) {
@@ -207,8 +206,7 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
// todo: apply schema id and schema info if needs
int schemaId = tableInfo.getSchemaId();
Schema schema = tableInfo.getSchema();
- schemaMetadataManager.updateLatestSchema(
- tablePath, (short) schemaId, schema);
+ serverSchemaCache.updateLatestSchema(tableId, (short) schemaId, schema);
if (tableId == DELETED_TABLE_ID) {
Long removedTableId = tableIdByPath.remove(tablePath);
@@ -437,7 +435,7 @@ public class TabletServerMetadataCache implements ServerMetadataCache {
}
@VisibleForTesting
- public SchemaMetadataManager getSchemaMetadataManager() {
- return schemaMetadataManager;
+ public ServerSchemaCache getServerSchemaCache() {
+ return serverSchemaCache;
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
index 251d670b6..d3b5d63c9 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/AdjustIsrManager.java
@@ -77,7 +77,7 @@ public class AdjustIsrManager {
public CompletableFuture<LeaderAndIsr> submit(
TableBucket tableBucket, LeaderAndIsr leaderAndIsr) {
- // TODOadd coordinatorEpoch.
+ // TODO add coordinatorEpoch.
CompletableFuture<LeaderAndIsr> future = new CompletableFuture<>();
AdjustIsrItem adjustIsrItem = new AdjustIsrItem(tableBucket, leaderAndIsr, future);
boolean enqueued = unsentAdjustIsrMap.putIfAbsent(tableBucket, adjustIsrItem) == null;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
index ad2dcb24d..d8c88cbd7 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/Replica.java
@@ -237,6 +237,7 @@ public final class Replica {
this.schemaGetter =
metadataCache.subscribeWithInitialSchema(
physicalPath.getTablePath(),
+ tableInfo.getTableId(),
tableInfo.getSchemaId(),
tableInfo.getSchema());
this.tableConfig = tableInfo.getTableConfig();
@@ -1517,7 +1518,7 @@ public final class Replica {
List<Integer> isrToSend = new ArrayList<>(isrState.isr());
isrToSend.add(newInSyncReplicaId);
- // TODOadd server epoch to isr.
+ // TODO add server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
@@ -1539,7 +1540,7 @@ public final class Replica {
// erroneously advance the HW if the `AdjustIsr` were to fail. Hence, the "maximal ISR"
// for `PendingShrinkIsr` is the current ISR.
- // TODOadd server epoch to isr.
+ // TODO add server epoch to isr.
LeaderAndIsr newLeaderAndIsr =
new LeaderAndIsr(
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
index 9f6e4d8ad..206586bf1 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/ReplicaManager.java
@@ -39,6 +39,7 @@ import org.apache.fluss.metrics.MetricNames;
import org.apache.fluss.metrics.groups.MetricGroup;
import org.apache.fluss.record.KvRecordBatch;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.remote.RemoteLogFetchInfo;
import org.apache.fluss.remote.RemoteLogSegment;
import org.apache.fluss.rpc.RpcClient;
@@ -149,6 +150,7 @@ public class ReplicaManager {
private final Map<TableBucket, HostedReplica> allReplicas = MapUtils.newConcurrentHashMap();
private final TabletServerMetadataCache metadataCache;
+ private final ProjectionPushdownCache projectionsCache = new ProjectionPushdownCache();
private final Lock replicaStateChangeLock = new ReentrantLock();
/**
@@ -1083,7 +1085,8 @@ public class ReplicaManager {
adjustedMaxBytes,
replica.getSchemaGetter(),
replica.getArrowCompressionInfo(),
- fetchReqInfo.getProjectFields());
+ fetchReqInfo.getProjectFields(),
+ projectionsCache);
LogReadInfo readInfo = replica.fetchRecords(fetchParams);
// Once we read from a non-empty bucket, we stop ignoring request and bucket
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
index 06e94e9ae..9e4479b3e 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherManager.java
@@ -225,7 +225,7 @@ public class ReplicaFetcherManager {
fetcherThread.getLeader().leaderServerId(),
initialFetchStatusMap);
} catch (InterruptedException e) {
- LOG.error("Interrupted whileadd buckets to fetcher threads.", e);
+ LOG.error("Interrupted while add buckets to fetcher threads.", e);
}
}
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
index 76195a0df..398bfb6ff 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThread.java
@@ -80,7 +80,7 @@ final class ReplicaFetcherThread extends ShutdownableThread {
private final LeaderEndpoint leader;
private final int fetchBackOffMs;
- // manuallyadd timout logic in here, todo remove this timeout logic if
+ // manually add timout logic in here, todo remove this timeout logic if
// we support global request timeout in #279
private final int timeoutSeconds = 30;
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
index 062eec73d..c89fca6f7 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/RpcGatewayManager.java
@@ -65,7 +65,7 @@ public class RpcGatewayManager<T extends RpcGateway> implements AutoCloseable {
}
/**
- * Add a server to the manager. It'll create a new gateway for the server andadd it to the
+ * Add a server to the manager. It'll create a new gateway for the server and add it to the
* manager. If the server has already existed, it'll remove the already existing server before
* adding the new one.
*/
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
index 165fd23e9..815771166 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/ServerRpcMessageUtils.java
@@ -339,7 +339,7 @@ public class ServerRpcMessageUtils {
.collect(Collectors.toList());
}
- private static @Nullable TableChange.ColumnPosition toColumnPosition(int columnPositionType) {
+ private static TableChange.ColumnPosition toColumnPosition(int columnPositionType) {
ColumnPositionType opType = ColumnPositionType.from(columnPositionType);
switch (opType) {
case LAST:
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
index 270d334ee..ea6aead7b 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/Timer.java
@@ -35,7 +35,7 @@ public interface Timer {
* Add a new task to this executor. It will be executed after the task's delay (beginning from
* the time of submission).
*
- * @param timerTask the task toadd
+ * @param timerTask the task to add
*/
void add(TimerTask timerTask);
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
index 934777c93..1dab9f229 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/utils/timer/TimingWheel.java
@@ -108,8 +108,8 @@ import java.util.concurrent.atomic.AtomicInteger;
* while priority queue based timers takes O(log N) for both insert and delete where N is the number
* of items in the queue.
*
- * <p>This class is not thread-safe. There should not be anyadd calls while advanceClock is
- * executing. It is caller's responsibility to enforce it. Simultaneousadd calls are thread-safe.
+ * <p>This class is not thread-safe. There should not be any add calls while advanceClock is
+ * executing. It is caller's responsibility to enforce it. Simultaneous add calls are thread-safe.
*/
@NotThreadSafe
final class TimingWheel {
diff --git a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
index e27ffd9c3..75ca2b7b4 100644
--- a/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
+++ b/fluss-server/src/main/java/org/apache/fluss/server/zk/ZooKeeperClient.java
@@ -536,18 +536,46 @@ public class ZooKeeperClient implements AutoCloseable {
"tables registration");
}
- /** Get the v1 schema for given tables in ZK. */
- public Map<TablePath, SchemaInfo> getV1Schemas(Collection<TablePath> tablePaths)
+ /** Get the latest schema for given tables in ZK. */
+ public Map<TablePath, SchemaInfo> getLatestSchemas(Collection<TablePath> tablePaths)
throws Exception {
- Map<String, TablePath> path2TablePathMap =
- tablePaths.stream()
- .collect(toMap(p -> SchemaZNode.path(p, DEFAULT_SCHEMA_ID), path -> path));
+ Map<String, TablePath> schemaChildren2TablePathMap =
+ tablePaths.stream().collect(toMap(SchemasZNode::path, path -> path));
+ List<ZkGetChildrenResponse> childrenResponses =
+ getChildrenInBackground(schemaChildren2TablePathMap.keySet());
+ // get the schema ids for each table
+ Map<TablePath, List<String>> schemaIdsForTables =
+ processGetChildrenResponses(
+ childrenResponses,
+ response -> schemaChildren2TablePathMap.get(response.getPath()),
+ "schema children for tables");
+
+ // get the schema info for each latest schema id
+ Map<TablePath, Integer> latestSchemaIdMap = new HashMap<>();
+ Map<String, TablePath> path2TablePathMap = new HashMap<>();
+ schemaIdsForTables.forEach(
+ (tp, schemaIds) -> {
+ int latestSchemaId =
+ schemaIds.stream().map(Integer::parseInt).reduce(Math::max).orElse(0);
+ latestSchemaIdMap.put(tp, latestSchemaId);
+ path2TablePathMap.put(SchemaZNode.path(tp, latestSchemaId), tp);
+ });
+
List<ZkGetDataResponse> responses = getDataInBackground(path2TablePathMap.keySet());
- return processGetDataResponses(
- responses,
- resp -> path2TablePathMap.get(resp.getPath()),
- b -> new SchemaInfo(SchemaZNode.decode(b), DEFAULT_SCHEMA_ID),
- "schema");
+ Map<TablePath, Schema> schemasForTables =
+ processGetDataResponses(
+ responses,
+ resp -> path2TablePathMap.get(resp.getPath()),
+ SchemaZNode::decode,
+ "schema");
+
+ Map<TablePath, SchemaInfo> result = new HashMap<>();
+ schemasForTables.forEach(
+ (tp, schema) -> {
+ int schemaId = latestSchemaIdMap.get(tp);
+ result.put(tp, new SchemaInfo(schema, schemaId));
+ });
+ return result;
}
/** Update the table in ZK. */
@@ -809,8 +837,8 @@ public class ZooKeeperClient implements AutoCloseable {
// --------------------------------------------------------------------------------------------
/** Register schema to ZK metadata and return the schema id. */
- public int registerSchema(TablePath tablePath, Schema schema) throws Exception {
- return registerSchema(tablePath, schema, getCurrentSchemaId(tablePath) + 1);
+ public int registerFirstSchema(TablePath tablePath, Schema schema) throws Exception {
+ return registerSchema(tablePath, schema, DEFAULT_SCHEMA_ID);
}
public int registerSchema(TablePath tablePath, Schema schema, int schemaId) throws Exception {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
index abc575fd5..af69caffe 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/authorizer/DefaultAuthorizerTest.java
@@ -487,7 +487,7 @@ public class DefaultAuthorizerTest {
Set<AccessControlEntry> expectedAcls = new HashSet<>();
for (int i = 0; i < 50; i++) {
- // each task represent that addColumn acl to different user on same resource.
+ // each task represent that add acl to different user on same resource.
final AccessControlEntry accessControlEntry =
createAclEntry(String.valueOf(i), "host-1", READ);
int finalI = i;
@@ -550,7 +550,7 @@ public class DefaultAuthorizerTest {
// generate 50 concurrent acl operation tasks.
List<Runnable> concurrentTasks = new ArrayList<>();
for (int i = 0; i < 50; i++) {
- // each task represent that addColumn acl to same user on same resource.
+ // each task represent that add acl to same user on same resource.
Runnable runnable =
() -> {
addAcls(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
index 4460cfc26..b8fec59ca 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CompletedSnapshotStoreManagerTest.java
@@ -99,13 +99,13 @@ class CompletedSnapshotStoreManagerTest {
CompletedSnapshotStoreManager completedSnapshotStoreManager =
createCompletedSnapshotStoreManager(maxNumberOfSnapshotsToRetain);
- // addColumn snapshots for a series of buckets
+ // add snapshots for a series of buckets
Set<TableBucket> tableBuckets = createTableBuckets(2, 3);
int snapshotNum = 3;
Map<TableBucket, CompletedSnapshot> tableBucketLatestCompletedSnapshots = new HashMap<>();
for (TableBucket tableBucket : tableBuckets) {
- // addColumn some snapshots
+ // add some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
@@ -137,7 +137,7 @@ class CompletedSnapshotStoreManagerTest {
assertThat(completedSnapshot)
.isEqualTo(tableBucketLatestCompletedSnapshots.get(tableBucket));
- // addColumn a new snapshot
+ // add a new snapshot
long snapshotId = completedSnapshot.getSnapshotID() + 1;
completedSnapshot = KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshotId);
addCompletedSnapshot(completedSnapshotStoreManager, completedSnapshot);
@@ -169,7 +169,7 @@ class CompletedSnapshotStoreManagerTest {
Set<TableBucket> tableBuckets = createTableBuckets(1, 2);
int snapshotNum = 3;
for (TableBucket tableBucket : tableBuckets) {
- // addColumn some snapshots
+ // add some snapshots
for (int snapshot = 0; snapshot < snapshotNum; snapshot++) {
CompletedSnapshot completedSnapshot =
KvTestUtils.mockCompletedSnapshot(tempDir, tableBucket, snapshot);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
index 581edafcc..bd4467c0a 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/CoordinatorChannelManagerTest.java
@@ -69,14 +69,14 @@ class CoordinatorChannelManagerTest {
// now, shouldn't send as we already remove the tablet server
checkSendRequest(coordinatorChannelManager, server0.id(), false);
- // test addColumn tablet server
- // before addColumn, shouldn't send
+ // test add tablet server
+ // before add, shouldn't send
ServerNode server1 = tabletServersNode.get(1);
checkSendRequest(coordinatorChannelManager, server1.id(), false);
coordinatorChannelManager.addTabletServer(server1);
- // after addColumn the tablet server, should send
+ // after add the tablet server, should send
// try to send message
checkSendRequest(coordinatorChannelManager, server1.id(), true);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
index b8b348bdb..3c417efe8 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/coordinator/TableManagerTest.java
@@ -211,8 +211,6 @@ class TableManagerTest {
assertThat(coordinatorContext.getAllReplicasForTable(tableId)).isEmpty();
}
- void testSchemaChange() throws Exception {}
-
@Test
void testResumeDeletionAfterRestart() throws Exception {
// first, create a table
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
index 87b768608..f2013a91a 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/KvTabletTest.java
@@ -38,6 +38,7 @@ import org.apache.fluss.record.KvRecordTestUtils;
import org.apache.fluss.record.LogRecords;
import org.apache.fluss.record.LogTestBase;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.record.TestData;
import org.apache.fluss.record.TestingSchemaGetter;
import org.apache.fluss.record.bytesview.MultiBytesView;
@@ -651,9 +652,10 @@ class KvTabletTest {
TablePath tablePath = TablePath.of("testDb", tableName);
initLogTabletAndKvTablet(tablePath, DATA1_SCHEMA_PK, config);
RowType rowType = DATA1_SCHEMA_PK.getRowType();
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
FileLogProjection logProjection = null;
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
@@ -749,7 +751,7 @@ class KvTabletTest {
schemaGetter.updateLatestSchemaInfo(new SchemaInfo(DATA2_SCHEMA, 2));
readLogRowType = doProjection ? DATA2_ROW_TYPE.project(new int[] {0}) : DATA2_ROW_TYPE;
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
@@ -834,9 +836,10 @@ class KvTabletTest {
KvRecordTestUtils.KvRecordFactory kvRecordFactory =
KvRecordTestUtils.KvRecordFactory.of(rowType);
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
FileLogProjection logProjection = null;
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
@@ -956,7 +959,7 @@ class KvTabletTest {
? newSchema.getRowType().project(new int[] {0})
: newSchema.getRowType();
if (doProjection) {
- logProjection = new FileLogProjection();
+ logProjection = new FileLogProjection(projectionCache);
logProjection.setCurrentProjection(
0L, schemaGetter, DEFAULT_COMPRESSION, new int[] {0});
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java
index 74d145bda..b39ec39ac 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/prewrite/KvPreWriteBufferTest.java
@@ -163,13 +163,13 @@ class KvPreWriteBufferTest {
}
assertThat(getValue(buffer, "key6")).isNull();
- // addColumn delete records.
+ // add delete records.
elementCount = 5;
bufferDelete(buffer, "key4", elementCount++);
bufferDelete(buffer, "key3", elementCount++);
assertThat(getValue(buffer, "key3")).isNull();
- // addColumn update records
+ // add update records
bufferPut(buffer, "key2", "value2-1", elementCount++);
bufferPut(buffer, "key1", "value1-1", elementCount++);
assertThat(getValue(buffer, "key1")).isEqualTo("value1-1");
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java
index 95afb907d..3b5fc88c4 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/DefaultRowMergerTest.java
@@ -20,9 +20,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.KvFormat;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.types.DataTypes;
import org.junit.jupiter.params.ParameterizedTest;
@@ -47,18 +45,21 @@ class DefaultRowMergerTest {
Schema.newBuilder()
.fromColumns(
Arrays.asList(
- new Schema.Column("age", DataTypes.STRING(), null, (short) 2),
new Schema.Column("id", DataTypes.INT(), null, (short) 0),
- new Schema.Column("name", DataTypes.STRING(), null, (short) 1)))
+ new Schema.Column("name", DataTypes.STRING(), null, (short) 1),
+ // add new column at end
+ new Schema.Column("age", DataTypes.STRING(), null, (short) 2)))
.primaryKey("id")
.build();
- private BinaryRow createBinaryRow(int id, String name) {
- return compactedRow(SCHEMA.getRowType(), new Object[] {id, name});
+ private BinaryValue createBinaryValue(int id, String name) {
+ return new BinaryValue(
+ (short) 1, compactedRow(SCHEMA.getRowType(), new Object[] {id, name}));
}
- private BinaryRow createBinaryRow(String age, int id, String name) {
- return compactedRow(SCHEMA_2.getRowType(), new Object[] {age, id, name});
+ private BinaryValue createBinaryValue(int id, String name, String age) {
+ return new BinaryValue(
+ (short) 2, compactedRow(SCHEMA_2.getRowType(), new Object[] {id, name, age}));
}
@ParameterizedTest
@@ -67,26 +68,25 @@ class DefaultRowMergerTest {
DefaultRowMerger merger = new DefaultRowMerger(KvFormat.COMPACTED, deleteBehavior);
merger.configureTargetColumns(null, (byte) 1, SCHEMA);
- InternalRow oldRow = createBinaryRow(1, "old");
- BinaryRow newRow = createBinaryRow(1, "new");
+ BinaryValue oldValue = createBinaryValue(1, "old");
+ BinaryValue newValue = createBinaryValue(1, "new");
// Test merge operation - should return new row
- InternalRow mergedRow = merger.merge(oldRow, newRow);
- assertThat(mergedRow).isSameAs(newRow);
+ BinaryValue mergedValue = merger.merge(oldValue, newValue);
+ assertThat(mergedValue).isSameAs(newValue);
// Test delete operation - should return null (deleted)
- InternalRow deletedRow = merger.delete(oldRow);
- assertThat(deletedRow).isNull();
+ BinaryValue deletedValue = merger.delete(oldValue);
+ assertThat(deletedValue).isNull();
// Test supportsDelete - should return true
assertThat(merger.deleteBehavior()).isEqualTo(deleteBehavior);
// Test schema change.
merger.configureTargetColumns(null, (byte) 2, SCHEMA_2);
- oldRow = ProjectedRow.from(SCHEMA, SCHEMA_2).replaceRow(oldRow);
- newRow = createBinaryRow("20", 1, "new2");
- assertThat(merger.merge(oldRow, newRow)).isSameAs(newRow);
- assertThat(merger.delete(newRow)).isNull();
+ newValue = createBinaryValue(1, "new2", "20");
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(newValue);
+ assertThat(merger.delete(newValue)).isNull();
}
@ParameterizedTest
@@ -98,20 +98,19 @@ class DefaultRowMergerTest {
RowMerger partialMerger =
merger.configureTargetColumns(new int[] {0, 1}, (byte) 1, SCHEMA); // id + name
- InternalRow oldRow = createBinaryRow(1, "old");
+ BinaryValue oldValue = createBinaryValue(1, "old");
- InternalRow ignoredRow = partialMerger.delete(oldRow);
- assertThat(ignoredRow).isNull();
+ BinaryValue ignoredValue = partialMerger.delete(oldValue);
+ assertThat(ignoredValue).isNull();
assertThat(partialMerger.deleteBehavior()).isEqualTo(deleteBehavior);
- assertThat(partialMerger.merge(null, oldRow)).isEqualTo(oldRow);
+ assertThat(partialMerger.merge(null, oldValue)).isEqualTo(oldValue);
// schema change then partial update (except name column).
- partialMerger = merger.configureTargetColumns(new int[] {0, 1}, (byte) 2, SCHEMA_2);
- oldRow = ProjectedRow.from(SCHEMA, SCHEMA_2).replaceRow(oldRow);
- BinaryRow newRow = createBinaryRow("20", 1, null);
- BinaryRow mergeRow = createBinaryRow("20", 1, "old");
- assertThat(partialMerger.merge(oldRow, newRow)).isEqualTo(mergeRow);
- assertThat(partialMerger.delete(mergeRow)).isEqualTo(createBinaryRow(null, 1, "old"));
+ partialMerger = merger.configureTargetColumns(new int[] {0, 2}, (byte) 2, SCHEMA_2);
+ BinaryValue newValue = createBinaryValue(1, null, "20");
+ BinaryValue mergeValue = createBinaryValue(1, "old", "20");
+ assertThat(partialMerger.merge(oldValue, newValue)).isEqualTo(mergeValue);
+ assertThat(partialMerger.delete(mergeValue)).isEqualTo(createBinaryValue(1, "old", null));
}
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java
index 008c2fafa..9163b23b0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/rowmerger/VersionedRowMergerTest.java
@@ -19,9 +19,7 @@ package org.apache.fluss.server.kv.rowmerger;
import org.apache.fluss.metadata.DeleteBehavior;
import org.apache.fluss.metadata.Schema;
-import org.apache.fluss.row.BinaryRow;
-import org.apache.fluss.row.InternalRow;
-import org.apache.fluss.row.ProjectedRow;
+import org.apache.fluss.record.BinaryValue;
import org.apache.fluss.row.TimestampLtz;
import org.apache.fluss.row.TimestampNtz;
import org.apache.fluss.types.DataType;
@@ -132,13 +130,13 @@ class VersionedRowMergerTest {
merger.configureTargetColumns(null, (short) 1, schema);
for (TestSpec testSpec : testSpecs) {
- BinaryRow oldRow = compactedRow(rowType, new Object[] {testSpec.oldValue, "dummy"});
- BinaryRow newRow = compactedRow(rowType, new Object[] {testSpec.newValue, "dummy"});
- InternalRow mergedRow = merger.merge(oldRow, newRow);
+ BinaryValue oldValue = binaryValue(rowType, new Object[] {testSpec.oldValue, "dummy"});
+ BinaryValue newValue = binaryValue(rowType, new Object[] {testSpec.newValue, "dummy"});
+ BinaryValue mergedValue = merger.merge(oldValue, newValue);
if (testSpec.expected.equals("old")) {
- assertThat(mergedRow).isSameAs(oldRow);
+ assertThat(mergedValue).isSameAs(oldValue);
} else if (testSpec.expected.equals("new")) {
- assertThat(mergedRow).isSameAs(newRow);
+ assertThat(mergedValue).isSameAs(newValue);
} else {
throw new IllegalArgumentException("Unknown expected value: " + testSpec.expected);
}
@@ -169,27 +167,29 @@ class VersionedRowMergerTest {
RowType rowType = schema.getRowType();
VersionedRowMerger merger = new VersionedRowMerger("a", DeleteBehavior.DISABLE);
merger.configureTargetColumns(null, (short) 1, schema);
- InternalRow oldRow = compactedRow(rowType, new Object[] {11, "dummy"});
- BinaryRow newRow = compactedRow(rowType, new Object[] {2, "dummy"});
- assertThat(merger.merge(oldRow, newRow)).isSameAs(oldRow);
+ BinaryValue oldValue = binaryValue(rowType, new Object[] {11, "dummy"});
+ BinaryValue newValue = binaryValue(rowType, new Object[] {2, "dummy"});
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(oldValue);
Schema schema2 =
Schema.newBuilder()
.fromColumns(
Arrays.asList(
- new Schema.Column("c", DataTypes.STRING(), null, (short) 2),
new Schema.Column("a", DataTypes.INT(), null, (short) 0),
+ new Schema.Column("b", DataTypes.STRING(), null, (short) 1),
+ // add new column at end
new Schema.Column(
- "b", DataTypes.STRING(), null, (short) 1)))
+ "c", DataTypes.STRING(), null, (short) 2)))
.build();
rowType = schema2.getRowType();
merger.configureTargetColumns(null, (short) 2, schema2);
- oldRow = ProjectedRow.from(schema, schema2).replaceRow(oldRow);
- newRow = compactedRow(rowType, new Object[] {"a", 2, "dummy"});
- assertThat(merger.merge(oldRow, newRow)).isSameAs(oldRow);
- newRow = compactedRow(rowType, new Object[] {"b", 20, "dummy"});
- assertThat(merger.merge(oldRow, newRow)).isSameAs(newRow);
+ newValue = binaryValue(rowType, new Object[] {2, "dummy", "a"});
+ // version is smaller than old value
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(oldValue);
+ newValue = binaryValue(rowType, new Object[] {20, "dummy", "b"});
+ // version is greater than old value
+ assertThat(merger.merge(oldValue, newValue)).isSameAs(newValue);
}
private static TimestampNtz timestampNtz(String timestamp) {
@@ -201,6 +201,11 @@ class VersionedRowMergerTest {
return TimestampLtz.fromInstant(instant);
}
+ private static BinaryValue binaryValue(RowType rowType, Object[] objects) {
+ int schemaId = rowType.getFieldCount() == 2 ? 1 : 2;
+ return new BinaryValue((short) schemaId, compactedRow(rowType, objects));
+ }
+
/** Test specification for {@link VersionedRowMerger}. */
private static class TestSpec {
private final Object oldValue;
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java
index 87fb23259..8d45628d8 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/CompletedSnapshotStoreTest.java
@@ -161,7 +161,7 @@ class CompletedSnapshotStoreTest {
final CompletedSnapshot ckp = getSnapshot(ckpId);
assertThatThrownBy(() -> completedSnapshotStore.add(ckp))
- .as("We should get an exception when addColumn snapshot to failed..")
+ .as("We should get an exception when add snapshot to failed..")
.hasMessageContaining(errMsg)
.isInstanceOf(FlussException.class);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
index 2451e7ecc..9f9ea078d 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/KvTabletSnapshotTargetTest.java
@@ -247,7 +247,7 @@ class KvTabletSnapshotTargetTest {
final String errMsg = "Add to snapshot handle failed.";
AtomicBoolean shouldFail = new AtomicBoolean(true);
// we use a store will fail when the variable shouldFail is true
- // addColumn the snapshot to store will fail
+ // add the snapshot to store will fail
CompletedSnapshotHandleStore completedSnapshotHandleStore =
TestCompletedSnapshotHandleStore.newBuilder()
.setAddFunction(
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java
index 5b890acd6..c6e90ff63 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/kv/snapshot/ZooKeeperCompletedSnapshotStoreTest.java
@@ -94,8 +94,8 @@ class ZooKeeperCompletedSnapshotStoreTest {
}
/**
- * Tests that the snapshot does not exist in the store when we fail to addColumn it into the
- * store (i.e., there exists an exception thrown by the method).
+ * Tests that the snapshot does not exist in the store when we fail to add it into the store
+ * (i.e., there exists an exception thrown by the method).
*/
@Test
void testAddSnapshotWithFailedRemove(@TempDir Path tmpDir) throws Exception {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java
index b217d8234..cea1513c5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/DroppedTableRecoveryTest.java
@@ -109,7 +109,7 @@ final class DroppedTableRecoveryTest extends LogTestBase {
ZOO_KEEPER_EXTENSION_WRAPPER.getCustomExtension().cleanupRoot();
zkClient.registerTable(
tablePath, TableRegistration.newTable(DATA1_TABLE_ID, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(tablePath, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(tablePath, DATA1_SCHEMA);
}
@AfterEach
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java
index 132de4482..d4c2be57b 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/FetchParamsTest.java
@@ -19,6 +19,7 @@ package org.apache.fluss.server.log;
import org.apache.fluss.metadata.SchemaInfo;
import org.apache.fluss.record.FileLogProjection;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.record.TestData;
import org.apache.fluss.record.TestingSchemaGetter;
@@ -33,13 +34,15 @@ class FetchParamsTest {
@Test
void testSetCurrentFetch() {
FetchParams fetchParams = new FetchParams(1, 100);
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
fetchParams.setCurrentFetch(
1L,
20L,
1024,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA1_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- null);
+ null,
+ projectionCache);
assertThat(fetchParams.fetchOffset()).isEqualTo(20L);
assertThat(fetchParams.maxFetchBytes()).isEqualTo(1024);
assertThat(fetchParams.projection()).isNull();
@@ -50,7 +53,8 @@ class FetchParamsTest {
512,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA2_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- new int[] {0, 2});
+ new int[] {0, 2},
+ projectionCache);
assertThat(fetchParams.fetchOffset()).isEqualTo(30L);
assertThat(fetchParams.maxFetchBytes()).isEqualTo(512);
assertThat(fetchParams.projection()).isNotNull();
@@ -63,7 +67,8 @@ class FetchParamsTest {
256,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA1_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- null);
+ null,
+ projectionCache);
assertThat(fetchParams.projection()).isNull();
fetchParams.setCurrentFetch(
@@ -72,7 +77,8 @@ class FetchParamsTest {
512,
new TestingSchemaGetter(new SchemaInfo(TestData.DATA2_SCHEMA, (short) 1)),
DEFAULT_COMPRESSION,
- new int[] {0, 2});
+ new int[] {0, 2},
+ projectionCache);
// the FileLogProjection should be cached
assertThat(fetchParams.projection()).isNotNull().isSameAs(prevProjection);
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java
index f4e536b86..80e99c2f5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogManagerTest.java
@@ -79,7 +79,7 @@ final class LogManagerTest extends LogTestBase {
private TableBucket tableBucket2;
private LogManager logManager;
- // TODOadd more tests refer to kafka's LogManagerTest.
+ // TODO add more tests refer to kafka's LogManagerTest.
@BeforeAll
static void baseBeforeAll() {
@@ -114,10 +114,10 @@ final class LogManagerTest extends LogTestBase {
ZOO_KEEPER_EXTENSION_WRAPPER.getCustomExtension().cleanupRoot();
zkClient.registerTable(
tablePath1, TableRegistration.newTable(DATA1_TABLE_ID, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(tablePath1, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(tablePath1, DATA1_SCHEMA);
zkClient.registerTable(
tablePath2, TableRegistration.newTable(DATA2_TABLE_ID, DATA2_TABLE_DESCRIPTOR));
- zkClient.registerSchema(tablePath2, DATA2_SCHEMA);
+ zkClient.registerFirstSchema(tablePath2, DATA2_SCHEMA);
}
static List<String> partitionProvider() {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java
index 7af086823..7b88a53a5 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/LogTabletTest.java
@@ -73,7 +73,7 @@ final class LogTabletTest extends LogTestBase {
private FlussScheduler scheduler;
private File logDir;
- // TODOadd more tests refer to kafka's UnifiedLogTest.
+ // TODO add more tests refer to kafka's UnifiedLogTest.
@BeforeEach
public void setup() throws Exception {
@@ -374,7 +374,7 @@ final class LogTabletTest extends LogTestBase {
log.updateHighWatermark(log.localLogEndOffset());
- // TODOadd delete log segment logic.
+ // TODO add delete log segment logic.
}
@Test
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java
index 5bedd921e..0d571e1ea 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/log/remote/RemoteLogTabletTest.java
@@ -94,7 +94,7 @@ class RemoteLogTabletTest extends RemoteLogTestBase {
assertThat(remoteLogTablet.getRemoteLogStartOffset()).isEqualTo(0);
assertThat(remoteLogTablet.getRemoteLogEndOffset()).isEqualTo(OptionalLong.of(30));
- // delete first remote log segment and addColumn another one remote log segments.
+ // delete first remote log segment and add another one remote log segments.
remoteLogTablet.addAndDeleteLogSegments(
Collections.singletonList(remoteLogSegmentList.get(3)),
Collections.singletonList(remoteLogSegmentList.get(0)));
@@ -102,7 +102,7 @@ class RemoteLogTabletTest extends RemoteLogTestBase {
assertThat(remoteLogTablet.getRemoteLogStartOffset()).isEqualTo(10);
assertThat(remoteLogTablet.getRemoteLogEndOffset()).isEqualTo(OptionalLong.of(40));
- // delete all exist and append one. we will first addColumn then delete.
+ // delete all exist and append one. we will first add then delete.
remoteLogTablet.addAndDeleteLogSegments(
Collections.singletonList(remoteLogSegmentList.get(4)), remoteLogSegmentList);
assertThat(remoteLogTablet.getIdToRemoteLogSegmentMap()).hasSize(0);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java
index 35c36d30c..b6fdd9e93 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/metadata/MetadataUpdateITCase.java
@@ -157,7 +157,7 @@ class MetadataUpdateITCase {
expectedTablePathById,
Collections.emptyMap()));
- // addColumn back tablet server2
+ // add back tablet server2
FLUSS_CLUSTER_EXTENSION.startTabletServer(2);
retry(
Duration.ofMinutes(1),
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/metadata/SchemaMetadataManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
similarity index 73%
rename from fluss-server/src/test/java/org/apache/fluss/server/metadata/SchemaMetadataManagerTest.java
rename to fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
index 672910b04..c42c7b296 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/metadata/SchemaMetadataManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/metadata/ServerSchemaCacheTest.java
@@ -44,40 +44,37 @@ import static org.apache.fluss.record.TestData.DATA2_TABLE_DESCRIPTOR;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
-/** Test for {@link SchemaMetadataManager}. */
-public class SchemaMetadataManagerTest {
+/** Test for {@link ServerSchemaCache}. */
+public class ServerSchemaCacheTest {
@Test
- void testPublishAndsubscribeSchemaChange() {
- SchemaMetadataManager manager =
- new SchemaMetadataManager(new TestingMetadataManager(Collections.emptyList()));
+ void testPublishAndSubscribeSchemaChange() {
+ ServerSchemaCache manager =
+ new ServerSchemaCache(new TestingMetadataManager(Collections.emptyList()));
SchemaGetter subscriber1 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
+ 1L, new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
SchemaGetter subscriber2 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
+ 2L, new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA1_SCHEMA, 1));
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA1_SCHEMA, 1));
- manager.updateLatestSchema(
- new TablePath("test_tb", "test_table_2"), (short) 2, DATA2_SCHEMA);
+ manager.updateLatestSchema(2L, (short) 2, DATA2_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA1_SCHEMA, 1));
assertThat(subscriber2.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber2.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
- manager.updateLatestSchema(
- new TablePath("test_tb", "test_table_1"), (short) 2, DATA2_SCHEMA);
+ manager.updateLatestSchema(1L, (short) 2, DATA2_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
assertThat(subscriber2.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber2.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
// one more redundant schema change.
- manager.updateLatestSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA2_SCHEMA);
+ manager.updateLatestSchema(1L, (short) 1, DATA2_SCHEMA);
assertThat(subscriber1.getLatestSchemaInfo()).isNotNull();
assertThat(subscriber1.getLatestSchemaInfo()).isEqualTo(new SchemaInfo(DATA2_SCHEMA, 2));
assertThat(subscriber2.getLatestSchemaInfo()).isNotNull();
@@ -94,11 +91,12 @@ public class SchemaMetadataManagerTest {
DATA2_TABLE_DESCRIPTOR,
System.currentTimeMillis(),
System.currentTimeMillis());
- SchemaMetadataManager manager =
- new SchemaMetadataManager(
+ ServerSchemaCache manager =
+ new ServerSchemaCache(
new TestingMetadataManager(Arrays.asList(DATA1_TABLE_INFO, tableInfo2)));
SchemaGetter schemaGetter =
- manager.subscribeWithInitialSchema(DATA1_TABLE_PATH, (short) 2, DATA2_SCHEMA);
+ manager.subscribeWithInitialSchema(
+ DATA1_TABLE_ID, DATA1_TABLE_PATH, (short) 2, DATA2_SCHEMA);
assertThat(schemaGetter.getSchema(2)).isEqualTo(DATA2_SCHEMA);
assertThat(schemaGetter.getSchema(1)).isEqualTo(DATA1_SCHEMA);
assertThatThrownBy(() -> schemaGetter.getSchema(3))
@@ -107,43 +105,39 @@ public class SchemaMetadataManagerTest {
@Test
void testUnsubscribeSchemaChange() {
- SchemaMetadataManager manager =
- new SchemaMetadataManager(new TestingMetadataManager(Collections.emptyList()));
+ ServerSchemaCache manager =
+ new ServerSchemaCache(new TestingMetadataManager(Collections.emptyList()));
SchemaGetter subscriber1 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
+ 1L, new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
SchemaGetter subscriber2 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
+ 1L, new TablePath("test_tb", "test_table_1"), (short) 1, DATA1_SCHEMA);
SchemaGetter subscriber3 =
manager.subscribeWithInitialSchema(
- new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(2);
+ 2L, new TablePath("test_tb", "test_table_2"), (short) 1, DATA1_SCHEMA);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(2);
assertThat(manager.getSubscriberCounters()).hasSize(2);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isEqualTo(2);
+ assertThat(manager.getSubscriberCounters().get(1L)).isEqualTo(2);
subscriber1.release();
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(2);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(2);
assertThat(manager.getSubscriberCounters()).hasSize(2);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isEqualTo(1);
+ assertThat(manager.getSubscriberCounters().get(1L)).isEqualTo(1);
// one more redundant unsubscribe.
subscriber1.release();
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(2);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(2);
assertThat(manager.getSubscriberCounters()).hasSize(2);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isEqualTo(1);
+ assertThat(manager.getSubscriberCounters().get(1L)).isEqualTo(1);
subscriber2.release();
- assertThat(manager.getLatestSchemaByTablePath()).hasSize(1);
+ assertThat(manager.getLatestSchemaByTableId()).hasSize(1);
assertThat(manager.getSubscriberCounters()).hasSize(1);
- assertThat(manager.getSubscriberCounters().get(new TablePath("test_tb", "test_table_1")))
- .isNull();
+ assertThat(manager.getSubscriberCounters().get(1L)).isNull();
subscriber3.release();
- assertThat(manager.getLatestSchemaByTablePath()).isEmpty();
+ assertThat(manager.getLatestSchemaByTableId()).isEmpty();
assertThat(manager.getSubscriberCounters()).isEmpty();
subscriber3.release();
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java
index bb2138881..933abd231 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrITCase.java
@@ -140,7 +140,7 @@ public class AdjustIsrITCase {
isr.add(stopFollower);
FLUSS_CLUSTER_EXTENSION.notifyLeaderAndIsr(
stopFollower, DATA1_TABLE_PATH, tb, newLeaderAndIsr, isr);
- // retry until the stop follower addColumn back to ISR.
+ // retry until the stop follower add back to ISR.
retry(
Duration.ofMinutes(1),
() ->
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java
index 75fb9e248..3f8623569 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/AdjustIsrManagerTest.java
@@ -32,7 +32,7 @@ import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link AdjustIsrManager}. */
class AdjustIsrManagerTest {
- // TODO addColumn more test refer to kafka AlterPartitionManagerTest, See: FLUSS-56278513
+ // TODO add more test refer to kafka AlterPartitionManagerTest, See: FLUSS-56278513
@Test
void testSubmitShrinkIsr() throws Exception {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java
index b089efd30..b093787f0 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/HighWatermarkPersistenceTest.java
@@ -92,7 +92,7 @@ final class HighWatermarkPersistenceTest extends ReplicaTestBase {
assertThat(highWatermark0).isEqualTo(10L);
assertThat(replica0.getLogTablet().getHighWatermark()).isEqualTo(10L);
- // addColumn another replica and set highWatermark.
+ // add another replica and set highWatermark.
TableBucket tableBucket1 = new TableBucket(DATA2_TABLE_ID, 0);
replicaManager.becomeLeaderOrFollower(
INITIAL_COORDINATOR_EPOCH,
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java
index a7c65acda..9e13b1688 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/RemoveOfflineReplicaFromIsrITCase.java
@@ -130,7 +130,7 @@ class RemoveOfflineReplicaFromIsrITCase {
FLUSS_CLUSTER_EXTENSION.startTabletServer(follower);
isr.add(follower);
- // make sure the stopped follower can addColumn back to isr after restart
+ // make sure the stopped follower can add back to isr after restart
retry(
Duration.ofMinutes(1),
() ->
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java
index 92c24f871..6ec71bc79 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaManagerTest.java
@@ -193,7 +193,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
void testFetchLog() throws Exception {
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
makeLogTableAsLeader(tb.getBucket());
@@ -309,7 +309,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
void testFetchLogWithMaxBytesLimit() throws Exception {
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, DEFAULT_SCHEMA_ID, DATA1_SCHEMA);
TableBucket tb = new TableBucket(DATA1_TABLE_ID, 1);
makeLogTableAsLeader(tb.getBucket());
@@ -432,7 +432,8 @@ class ReplicaManagerTest extends ReplicaTestBase {
assertThat(records1).isNotNull();
assertThat(records2).isNotNull();
SchemaGetter schemaGetter =
- serverMetadataCache.subscribeWithInitialSchema(DATA1_TABLE_PATH, 1, DATA1_SCHEMA);
+ serverMetadataCache.subscribeWithInitialSchema(
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA);
if (records1.sizeInBytes() == 0) {
assertThat(records2.sizeInBytes() > 0).isTrue();
assertMemoryRecordsEquals(
@@ -498,7 +499,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
makeKvTableAsLeader(DATA1_TABLE_ID_PK, DATA1_TABLE_PATH_PK, tb.getBucket());
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH_PK, 1, DATA1_SCHEMA_PK);
+ DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK, 1, DATA1_SCHEMA_PK);
// 1. put kv records to kv store.
List<Tuple2<Object[], Object[]>> data1 =
@@ -628,7 +629,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
makeKvTableAsLeader(DATA1_TABLE_ID_PK, DATA1_TABLE_PATH_PK, tb.getBucket());
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH_PK, 1, DATA1_SCHEMA_PK);
+ DATA1_TABLE_PATH_PK, DATA1_TABLE_ID_PK, 1, DATA1_SCHEMA_PK);
// put 10 batches delete non-exists key batch to kv store.
CompletableFuture<List<PutKvResultForBucket>> future;
@@ -906,7 +907,8 @@ class ReplicaManagerTest extends ReplicaTestBase {
CompletableFuture<LimitScanResultForBucket> limitFuture = new CompletableFuture<>();
replicaManager.limitScan(tb, 10, limitFuture::complete);
SchemaGetter schemaGetter =
- serverMetadataCache.subscribeWithInitialSchema(DATA1_TABLE_PATH, 1, DATA1_SCHEMA);
+ serverMetadataCache.subscribeWithInitialSchema(
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA);
assertMemoryRecordsEquals(
DATA1_ROW_TYPE,
schemaGetter,
@@ -1463,7 +1465,7 @@ class ReplicaManagerTest extends ReplicaTestBase {
FetchLogResultForBucket r = r1.getValue();
SchemaGetter schemaGetter =
serverMetadataCache.subscribeWithInitialSchema(
- DATA1_TABLE_PATH, 1, DATA1_SCHEMA);
+ DATA1_TABLE_PATH, DATA1_TABLE_ID, 1, DATA1_SCHEMA);
assertLogRecordsEqualsWithRowKind(
DEFAULT_SCHEMA_ID,
DATA1_ROW_TYPE,
@@ -1591,11 +1593,12 @@ class ReplicaManagerTest extends ReplicaTestBase {
zkClient.registerTable(
nonePartitionTablePath,
TableRegistration.newTable(nonePartitionTableId, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(nonePartitionTablePath, DATA1_TABLE_DESCRIPTOR.getSchema());
+ zkClient.registerFirstSchema(nonePartitionTablePath, DATA1_TABLE_DESCRIPTOR.getSchema());
zkClient.registerTable(
partitionTablePath,
TableRegistration.newTable(partitionTableId, DATA1_PARTITIONED_TABLE_DESCRIPTOR));
- zkClient.registerSchema(partitionTablePath, DATA1_PARTITIONED_TABLE_DESCRIPTOR.getSchema());
+ zkClient.registerFirstSchema(
+ partitionTablePath, DATA1_PARTITIONED_TABLE_DESCRIPTOR.getSchema());
expectedCoordinatorServer.put(
"INTERNAL", new ServerNode(0, "localhost", 1235, ServerType.COORDINATOR));
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
index e45a5557c..c8e26ea7b 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTest.java
@@ -31,6 +31,7 @@ import org.apache.fluss.record.KvRecordTestUtils;
import org.apache.fluss.record.LogRecordBatch;
import org.apache.fluss.record.LogRecords;
import org.apache.fluss.record.MemoryLogRecords;
+import org.apache.fluss.record.ProjectionPushdownCache;
import org.apache.fluss.server.entity.NotifyLeaderAndIsrData;
import org.apache.fluss.server.kv.KvTablet;
import org.apache.fluss.server.kv.snapshot.CompletedSnapshot;
@@ -94,8 +95,8 @@ import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Test for {@link Replica}. */
final class ReplicaTest extends ReplicaTestBase {
- // TODO addColumn more tests refer to kafka's PartitionTest.
- // TODO addColumn more tests to cover partition table
+ // TODO add more tests refer to kafka's PartitionTest.
+ // TODO add more tests to cover partition table
@Test
void testMakeLeader() throws Exception {
@@ -137,15 +138,22 @@ final class ReplicaTest extends ReplicaTestBase {
(int)
conf.get(ConfigOptions.CLIENT_SCANNER_LOG_FETCH_MAX_BYTES)
.getBytes());
+ ProjectionPushdownCache projectionCache = new ProjectionPushdownCache();
fetchParams.setCurrentFetch(
- DATA1_TABLE_ID, 0, Integer.MAX_VALUE, schemaGetter, DEFAULT_COMPRESSION, null);
+ DATA1_TABLE_ID,
+ 0,
+ Integer.MAX_VALUE,
+ schemaGetter,
+ DEFAULT_COMPRESSION,
+ null,
+ projectionCache);
LogReadInfo logReadInfo = logReplica.fetchRecords(fetchParams);
assertLogRecordsEquals(
DATA1_ROW_TYPE, logReadInfo.getFetchedData().getRecords(), DATA1, schemaGetter);
// schema evolution.
serverMetadataCache.updateLatestSchema(
- DATA1_TABLE_PATH, new SchemaInfo(DATA2_SCHEMA, (short) 2));
+ DATA1_TABLE_ID, new SchemaInfo(DATA2_SCHEMA, (short) 2));
mr =
createRecordsWithoutBaseLogOffset(
DATA2_ROW_TYPE,
@@ -163,7 +171,8 @@ final class ReplicaTest extends ReplicaTestBase {
Integer.MAX_VALUE,
schemaGetter,
DEFAULT_COMPRESSION,
- null);
+ null,
+ projectionCache);
logReadInfo = logReplica.fetchRecords(fetchParams);
assertLogRecordsEquals(
2, DATA2_ROW_TYPE, logReadInfo.getFetchedData().getRecords(), DATA2, schemaGetter);
@@ -393,7 +402,7 @@ final class ReplicaTest extends ReplicaTestBase {
currentOffset += 2;
short newSchemaId = 2;
serverMetadataCache.updateLatestSchema(
- DATA1_TABLE_PATH_PK, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
+ DATA1_TABLE_ID, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
KvRecordTestUtils.KvRecordBatchFactory kvRecordBatchFactory2 =
KvRecordTestUtils.KvRecordBatchFactory.of(newSchemaId);
KvRecordTestUtils.KvRecordFactory kvRecordFactory2 =
@@ -718,7 +727,7 @@ final class ReplicaTest extends ReplicaTestBase {
// update schema.
zkClient.registerSchema(DATA1_TABLE_PATH_PK, DATA2_SCHEMA, newSchemaId);
serverMetadataCache.updateLatestSchema(
- DATA1_TABLE_PATH_PK, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
+ DATA1_TABLE_ID, new SchemaInfo(DATA2_SCHEMA, newSchemaId));
// write data with new schema
putRecordsToLeader(
kvReplica,
@@ -808,7 +817,8 @@ final class ReplicaTest extends ReplicaTestBase {
Integer.MAX_VALUE,
replica.getSchemaGetter(),
DEFAULT_COMPRESSION,
- null);
+ null,
+ new ProjectionPushdownCache());
LogReadInfo logReadInfo = replica.fetchRecords(fetchParams);
return logReadInfo.getFetchedData().getRecords();
}
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java
index c438da529..ae7e14a87 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/ReplicaTestBase.java
@@ -254,16 +254,16 @@ public class ReplicaTestBase {
zkClient.registerTable(
DATA1_TABLE_PATH,
TableRegistration.newTable(DATA1_TABLE_ID, data1NonPkTableDescriptor));
- zkClient.registerSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
zkClient.registerTable(
DATA1_TABLE_PATH_PK,
TableRegistration.newTable(DATA1_TABLE_ID_PK, DATA1_TABLE_DESCRIPTOR_PK));
- zkClient.registerSchema(DATA1_TABLE_PATH_PK, DATA1_SCHEMA_PK);
+ zkClient.registerFirstSchema(DATA1_TABLE_PATH_PK, DATA1_SCHEMA_PK);
zkClient.registerTable(
DATA2_TABLE_PATH,
TableRegistration.newTable(DATA2_TABLE_ID, DATA2_TABLE_DESCRIPTOR));
- zkClient.registerSchema(DATA2_TABLE_PATH, DATA2_SCHEMA);
+ zkClient.registerFirstSchema(DATA2_TABLE_PATH, DATA2_SCHEMA);
}
protected long registerTableInZkClient(
@@ -282,7 +282,7 @@ public class ReplicaTestBase {
zkClient.deleteTable(tablePath);
}
zkClient.registerTable(tablePath, TableRegistration.newTable(tableId, tableDescriptor));
- zkClient.registerSchema(tablePath, schema);
+ zkClient.registerFirstSchema(tablePath, schema);
return tableId;
}
@@ -342,7 +342,7 @@ public class ReplicaTestBase {
}
// TODO this is only for single tablet server unit test.
- // TODO addColumn more test cases for partition table which make leader by this method.
+ // TODO add more test cases for partition table which make leader by this method.
protected void makeLogTableAsLeader(int bucketId) {
makeLogTableAsLeader(new TableBucket(DATA1_TABLE_ID, bucketId), false);
}
@@ -375,7 +375,7 @@ public class ReplicaTestBase {
}
// TODO this is only for single tablet server unit test.
- // TODO addColumn more test cases for partition table which make leader by this method.
+ // TODO add more test cases for partition table which make leader by this method.
protected void makeKvTableAsLeader(long tableId, TablePath tablePath, int bucketId) {
makeKvTableAsLeader(
new TableBucket(tableId, bucketId), tablePath, INITIAL_LEADER_EPOCH, false);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java
index ab8bde9df..b1fa9067e 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/replica/fetcher/ReplicaFetcherThreadTest.java
@@ -356,7 +356,7 @@ public class ReplicaFetcherThreadTest {
zkClient.registerTable(
DATA1_TABLE_PATH,
TableRegistration.newTable(DATA1_TABLE_ID, DATA1_TABLE_DESCRIPTOR));
- zkClient.registerSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
+ zkClient.registerFirstSchema(DATA1_TABLE_PATH, DATA1_SCHEMA);
}
private void makeLeaderAndFollower() {
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
index 5a5910d1e..01cf2701a 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/tablet/TabletServiceITCase.java
@@ -177,7 +177,7 @@ public class TabletServiceITCase {
}
@Test
- @Disabled("TODO: addColumn back in https://github.com/apache/fluss/issues/771")
+ @Disabled("TODO: add back in https://github.com/apache/fluss/issues/771")
void testProduceLogResponseReturnInOrder() throws Exception {
long tableId =
createTable(
@@ -330,11 +330,11 @@ public class TabletServiceITCase {
tableId,
0,
Errors.INVALID_COLUMN_PROJECTION.code(),
- "Projected field id 2 is not contains in [0, 1]");
+ "Projected field id 2 is not contained in [0, 1]");
}
@Test
- @Disabled("TODO: addColumn back in https://github.com/apache/fluss/issues/777")
+ @Disabled("TODO: add back in https://github.com/apache/fluss/issues/777")
void testFetchLogWithMinFetchSizeAndTimeout() throws Exception {
long tableId =
createTable(FLUSS_CLUSTER_EXTENSION, DATA1_TABLE_PATH, DATA1_TABLE_DESCRIPTOR);
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
index 37f3dd6cb..a1ee83fa6 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/testutils/FlussClusterExtension.java
@@ -51,8 +51,8 @@ import org.apache.fluss.server.coordinator.MetadataManager;
import org.apache.fluss.server.entity.NotifyLeaderAndIsrData;
import org.apache.fluss.server.kv.snapshot.CompletedSnapshot;
import org.apache.fluss.server.kv.snapshot.CompletedSnapshotHandle;
-import org.apache.fluss.server.metadata.SchemaMetadataManager;
import org.apache.fluss.server.metadata.ServerInfo;
+import org.apache.fluss.server.metadata.ServerSchemaCache;
import org.apache.fluss.server.metadata.TabletServerMetadataCache;
import org.apache.fluss.server.replica.Replica;
import org.apache.fluss.server.replica.ReplicaManager;
@@ -286,7 +286,7 @@ public final class FlussClusterExtension
}
private void startTabletServers() throws Exception {
- // addColumn tablet server to make generate assignment for table possible
+ // add tablet server to make generate assignment for table possible
for (int i = 0; i < initialNumOfTabletServers; i++) {
startTabletServer(i);
}
@@ -670,20 +670,20 @@ public final class FlussClusterExtension
() -> {
TableRegistration tableRegistration = zkClient.getTable(tablePath).get();
int bucketCount = tableRegistration.bucketCount;
+ long tableId = tableRegistration.tableId;
for (int bucketId = 0; bucketId < bucketCount; bucketId++) {
- TableBucket tableBucket =
- new TableBucket(tableRegistration.tableId, bucketId);
+ TableBucket tableBucket = new TableBucket(tableId, bucketId);
Optional<LeaderAndIsr> leaderAndIsrOpt =
zkClient.getLeaderAndIsr(tableBucket);
assertThat(leaderAndIsrOpt).isPresent();
int leader = leaderAndIsrOpt.get().leader();
TabletServer tabletServer = getTabletServerById(leader);
- SchemaMetadataManager schemaMetadataManager =
- tabletServer.getMetadataCache().getSchemaMetadataManager();
- Map<TablePath, SchemaInfo> latestSchemaByTablePath =
- schemaMetadataManager.getLatestSchemaByTablePath();
- assertThat(latestSchemaByTablePath).containsKey(tablePath);
- assertThat(latestSchemaByTablePath.get(tablePath).getSchemaId())
+ ServerSchemaCache serverSchemaCache =
+ tabletServer.getMetadataCache().getServerSchemaCache();
+ Map<Long, SchemaInfo> latestSchemaByTablePath =
+ serverSchemaCache.getLatestSchemaByTableId();
+ assertThat(latestSchemaByTablePath).containsKey(tableId);
+ assertThat(latestSchemaByTablePath.get(tableId).getSchemaId())
.isEqualTo(schemaId);
}
});
diff --git a/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java b/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java
index b87ed418e..f42d381fa 100644
--- a/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java
+++ b/fluss-server/src/test/java/org/apache/fluss/server/zk/ZooKeeperClientTest.java
@@ -387,7 +387,7 @@ class ZooKeeperClientTest {
.withComment("c is third column")
.primaryKey("a")
.build();
- int registeredSchemaId = zookeeperClient.registerSchema(tablePath, schema);
+ int registeredSchemaId = zookeeperClient.registerFirstSchema(tablePath, schema);
assertThat(registeredSchemaId).isEqualTo(schemaId);
assertThat(zookeeperClient.getCurrentSchemaId(tablePath)).isEqualTo(schemaId);
@@ -406,7 +406,7 @@ class ZooKeeperClientTest {
.withComment("b is second column")
.primaryKey("a")
.build();
- registeredSchemaId = zookeeperClient.registerSchema(tablePath, schema2);
+ registeredSchemaId = zookeeperClient.registerSchema(tablePath, schema2, 2);
assertThat(registeredSchemaId).isEqualTo(2);
assertThat(zookeeperClient.getCurrentSchemaId(tablePath)).isEqualTo(2);
diff --git a/fluss-server/src/test/resources/log4j2-test.properties b/fluss-server/src/test/resources/log4j2-test.properties
index 3db036fea..7acf6e146 100644
--- a/fluss-server/src/test/resources/log4j2-test.properties
+++ b/fluss-server/src/test/resources/log4j2-test.properties
@@ -18,7 +18,7 @@
# Set root logger level to OFF to not flood build logs
# set manually to INFO for debugging purposes
-rootLogger.level = INFO
+rootLogger.level = OFF
rootLogger.appenderRef.test.ref = TestLogger
appender.testlogger.name = TestLogger
diff --git a/fluss-test-coverage/pom.xml b/fluss-test-coverage/pom.xml
index ca167c2d9..0c6f54cd5 100644
--- a/fluss-test-coverage/pom.xml
+++ b/fluss-test-coverage/pom.xml
@@ -164,7 +164,7 @@
<excludes>
<exclude>fluss-test-coverage/**</exclude>
<exclude>fluss-test-utils/**</exclude>
- <!-- exclude adapter classes to avoid Jacoco error: "Can'tadd different class with same name" -->
+ <!-- exclude adapter classes to avoid Jacoco error: "Can't add different class with same name" -->
<exclude>fluss-flink/**/target/classes/org/apache/fluss/flink/adapter/**</exclude>
</excludes>
</resource>
@@ -206,7 +206,7 @@
<excludes>
<exclude>fluss-test-coverage/**</exclude>
<exclude>fluss-test-utils/**</exclude>
- <!-- exclude adapter classes to avoid Jacoco error: "Can'tadd different class with same name" -->
+ <!-- exclude adapter classes to avoid Jacoco error: "Can't add different class with same name" -->
<exclude>fluss-flink/**/target/classes/org/apache/fluss/flink/adapter/**</exclude>
</excludes>
</resource>