You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2026/10/20 06:46:39 UTC
(paimon) branch master updated: Revert last 10 commits on master
This is an automated email from the ASF dual-hosted git repository.
JingsongLi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/paimon.git
The following commit(s) were added to refs/heads/master by this push:
new 1489412cf6 Revert last 10 commits on master
1489412cf6 is described below
commit 1489412cf67121141cd735f7749f52b1456b5cce
Author: JingsongLi <ji...@gmail.com>
AuthorDate: Tue Oct 20 14:45:56 2026 +0800
Revert last 10 commits on master
Revert #10677 through #10686 as a single commit.
Reverted commits, newest first:
44c5113b8b [core] Extract injectable data evolution commit preparation (#10686)
1da2ac6155 [core] Share rewrite options for compaction and clustering (#10685)
6528cb5d3a [core] Compact Data Evolution column versions alongside clustering (#10684)
a8010b4ca5 [core] Preserve clustering levels for non-key Data Evolution updates (#10683)
cdb3e502b3 [flink] Maintain scalar indexes during Data Evolution clustering (#10682)
209a1ba679 [core][flink] Record row ID mappings for clustering output (#10681)
a6e6647c5b [core] Support incremental clustering for Data Evolution tables (#10680)
c638a4d427 [core] Extract the shared incremental clustering rewriter (#10679)
c2673f63d3 [core] Support row ID rewriting in scalar global indexes (#10678)
36ce7c97ee [flink] Preserve vector layouts in schema-aware row conversion (#10677)
Restore the tracked tree of e7ef0cbfad28f9e6d3ca40d49386fce46ca82305 (#10676).
---
docs/docs/append-table/incremental-clustering.mdx | 11 +-
.../data-evolution-maintenance.mdx | 71 -
docs/docs/multimodal-table/data-evolution.mdx | 4 +-
.../apache/paimon/globalindex/GlobalIndexer.java | 21 -
.../globalindex/bitmap/BitmapGlobalIndexer.java | 111 -
.../globalindex/btree/BTreeGlobalIndexer.java | 82 -
.../globalindex/GlobalIndexRowIdRewriteTest.java | 213 --
.../cluster/DataEvolutionClusterCompaction.java | 112 -
.../DataEvolutionIncrementalClusterRewriter.java | 217 --
.../append/cluster/HistoryPartitionCluster.java | 15 +-
.../append/cluster/IncrementalClusterManager.java | 105 +-
.../append/cluster/IncrementalClusterRewriter.java | 125 --
.../ClusteringCommitChangesRewriter.java | 59 -
.../dataevolution/ClusteringRowIdMapping.java | 113 -
.../DataEvolutionBlobCompactTask.java | 3 +-
.../DataEvolutionCommitChangesRewriter.java | 125 --
.../dataevolution/DataEvolutionCompactTask.java | 5 +-
.../DataEvolutionCompactTaskSerializer.java | 8 +-
.../dataevolution/DataEvolutionIndexRewriter.java | 474 ----
.../DataEvolutionNormalCompactTask.java | 27 +-
...ializeDeletionVectorsCommitChangesRewriter.java | 42 -
.../org/apache/paimon/compact/CompactOptions.java | 44 -
.../apache/paimon/operation/FileStoreCommit.java | 4 -
.../paimon/operation/FileStoreCommitImpl.java | 120 +-
.../operation/commit/CommitChangesRewriter.java | 69 -
.../commit/DataEvolutionConflictDetection.java | 19 +-
.../commit/ReassignCompactChangesProvider.java | 21 -
.../commit/RowIdRangeConflictChecker.java | 20 +-
.../org/apache/paimon/schema/SchemaValidation.java | 13 +-
.../apache/paimon/table/sink/TableCommitImpl.java | 6 -
.../apache/paimon/utils/DataEvolutionUtils.java | 66 -
.../DataEvolutionClusterCompactionTest.java | 277 ---
...ataEvolutionIncrementalClusterRewriterTest.java | 282 ---
.../cluster/HistoryPartitionClusterTest.java | 10 +-
.../cluster/IncrementalClusterManagerTest.java | 67 +-
.../dataevolution/ClusteringRowIdMappingTest.java | 139 --
.../dataevolution/DataEvolutionClusterTest.java | 2343 --------------------
.../DataEvolutionCompactTaskSerializerTest.java | 100 -
.../DataEvolutionNormalCompactTaskTest.java | 80 -
.../operation/FileStoreCommitRewriterTest.java | 453 ----
.../paimon/operation/FileStoreCommitTest.java | 56 +-
.../apache/paimon/schema/SchemaValidationTest.java | 87 -
.../table/DataEvolutionDeletionVectorTest.java | 135 +-
.../paimon/utils/DataEvolutionUtilsTest.java | 100 -
.../java/org/apache/paimon/flink/FlinkRowData.java | 76 +-
.../apache/paimon/flink/action/CompactAction.java | 11 +-
.../cluster/DataEvolutionClusterWriteOperator.java | 240 --
.../cluster/IncrementalClusterSplitSource.java | 16 +-
.../flink/compact/IncrementalClusterCompact.java | 113 +-
.../flink/sink/DataEvolutionClusterSink.java | 49 -
.../DataEvolutionCommitPreparationOperator.java | 31 +-
...DataEvolutionDeletionVectorMaterializeSink.java | 3 -
.../org/apache/paimon/flink/sorter/SortUtils.java | 2 +-
.../paimon/flink/DataEvolutionDeleteSqlITCase.java | 734 +-----
.../paimon/flink/FlinkRowDataWithVectorTest.java | 75 -
.../action/IncrementalClusterActionITCase.java | 13 -
.../DataEvolutionClusterWriteOperatorTest.java | 250 ---
.../paimon/spark/procedure/CompactProcedure.java | 181 +-
.../procedure/DataEvolutionRewriteExecutor.java | 111 +-
.../MaterializeDeletionVectorsProcedure.java | 3 -
.../spark/procedure/CompactProcedureTestBase.scala | 406 +---
61 files changed, 388 insertions(+), 8380 deletions(-)
diff --git a/docs/docs/append-table/incremental-clustering.mdx b/docs/docs/append-table/incremental-clustering.mdx
index d0152cde9e..25249a8f16 100644
--- a/docs/docs/append-table/incremental-clustering.mdx
+++ b/docs/docs/append-table/incremental-clustering.mdx
@@ -48,10 +48,8 @@ by a query, and does not replace SQL `ORDER BY`.
| Global/local sort mode | Configurable for batch clustering jobs. | Clustering is performed within each partition and bucket; the global/local option does not select this path. |
| Historical-partition auto-clustering | Supported. | `clustering.history-partition.*` does not apply. |
-Data Evolution tables support incremental clustering without dedicated BLOB or VECTOR files; see
-[Data Evolution clustering](../multimodal-table/data-evolution-maintenance#incremental-clustering) for its execution
-and row-ID behavior. If streaming consumers require ordered append reads from a bucketed table, keep that ordering
-and do not enable clustering.
+Data Evolution tables cannot enable incremental clustering. If streaming consumers require ordered append reads from a
+bucketed table, keep that ordering and do not enable clustering.
## Enable Incremental Clustering
@@ -136,11 +134,6 @@ For unaware-bucket tables, the mode controls how the **selected files in each pa
Here, “global” refers to the selected clustering work within a partition. It does not imply that all existing files or
all table partitions become globally ordered after an incremental run.
-Data Evolution uses the same sort modes. Its readers merge the latest column values and apply deletion vectors
-for complete selected file groups before sorting. The rewritten rows receive new row IDs at commit time.
-Already-clustered groups with accumulated non-clustering column versions can instead be compacted in the same job,
-preserving their row IDs and logical deletes; see [Data Evolution clustering](../multimodal-table/data-evolution-maintenance#incremental-clustering).
-
## Run Incremental Clustering
Run explicit compact jobs in batch mode. Table options supply the clustering columns and strategy. The examples below
diff --git a/docs/docs/multimodal-table/data-evolution-maintenance.mdx b/docs/docs/multimodal-table/data-evolution-maintenance.mdx
index 64d6bedb77..7666899d42 100644
--- a/docs/docs/multimodal-table/data-evolution-maintenance.mdx
+++ b/docs/docs/multimodal-table/data-evolution-maintenance.mdx
@@ -33,7 +33,6 @@ have different effects on row IDs and dependent indexes.
| Reduce column-file fragmentation | [Ordinary compaction](#ordinary-compaction) | Preserved |
| Resize large normal files | [Split during compaction](#splitting-large-files-during-compaction) | Preserved |
| Remove logically deleted positions | [Materialize deletion vectors](#materialize-deletion-vectors) | Reassigned in rewritten ranges |
-| Sort ordinary columns for pruning | [Incremental clustering](#incremental-clustering) | Reassigned in rewritten groups |
For updates and logical deletes, see [Data Evolution](./data-evolution).
For the normal/dedicated range contract, see [File Layout and Reads](./data-evolution-file-layout).
@@ -170,75 +169,6 @@ expiration to remove the files. The legacy
`data-evolution.compaction.rewrite-row-ids = true` setting is rejected; use the
materialization procedure instead.
-### Incremental Clustering
-
-Data Evolution tables can enable `clustering.incremental = true` with nonempty
-`clustering.columns` when they have no dedicated BLOB or VECTOR files. BLOB
-descriptor and view fields stored inline, and VECTOR columns stored in normal
-files, do not require dedicated files and remain supported.
-
-The existing [clustering strategies and file selection](../append-table/incremental-clustering)
-choose the work. Selection includes every column version in each selected
-row-ID group. Readers merge the latest column values and apply logical deletes
-before the existing Spark or Flink clustering pipeline sorts the surviving rows
-and writes new normal files with new row IDs. Unselected groups keep their row IDs.
-The output records the clustering columns and level for subsequent incremental
-selection. Updating only non-clustering columns does not make an already clustered
-group require sorting again: its row order remains valid. Unselected groups with
-at least `compaction.min.file-num` normal files (default 5) are instead compacted
-within the same job and snapshot. This merges their latest column values while
-preserving row IDs, clustering level, logical deletes, and indexes on unchanged
-columns. If minor selection skips a group whose sorting order is no longer
-valid, column-version compaction keeps it at level 0 for later clustering.
-Each such task handles one complete row-ID group; adjacent groups are
-not combined because row-ID order alone does not prove clustering-key order.
-Groups below the threshold can be skipped even by a full clustering call.
-Updating a clustering column
-makes the group eligible again. When a group is selected, all its column versions
-are rewritten together, including newer values of non-clustering columns.
-`clustering.incremental.mode` selects the same distributed global sort
-or task-local sort used by ordinary append-table clustering.
-
-For example, enable clustering with Spark SQL and run an explicit batch compact:
-
-```sql
-ALTER TABLE default.target_table SET TBLPROPERTIES (
- 'clustering.incremental' = 'true',
- 'clustering.columns' = 'id'
-);
-CALL sys.compact(table => 'default.target_table', compact_strategy => 'minor');
-```
-
-Use the corresponding Flink `ALTER TABLE ... SET` syntax and run the compact job
-in batch mode. Explicit compact jobs combine clustering and column-version
-compaction while this option is enabled. A group selected for clustering is
-rewritten only once. To run ordinary Data Evolution compaction across adjacent
-row-ID groups, disable clustering for the invocation with
-`options => 'clustering.incremental=false'`.
-
-The sorting path removes the applied deletion vectors; column-version compaction
-retains logical deletes and moves their vectors to the replacement files.
-Flink clustering also rewrites
-B-tree and bitmap index files with the new row IDs, reusing their stored keys.
-Indexes outside the rewritten row-ID ranges are preserved, including those in
-the same partition. Partially indexed data retains its exact coverage after sorting.
-The current contiguous-range index protocol limits rewriting to 64 new coverage
-fragments per index definition, and 64 retained plus new fragments per source file.
-If sorting would exceed either bound, clustering fails before committing; rebuild
-complete indexes with smaller shards first.
-The job records the old row IDs alongside each output file so that concurrent
-index creation and optimistic commit retries use the final assigned row IDs.
-
-Indexes with stale keys after indexed-column updates, and affected index types
-without RowID rewrite support, are dropped. Spark clustering drops index files
-whose coverage intersects the row-ID ranges selected for sorting. Rebuild dropped indexes, or use
-`scalar-index.search-mode = full` for complete scalar query results while part
-of the table remains unindexed.
-Concurrent updates or deletes that overlap the selected row-ID groups cause
-commit to fail, preventing stale rewrites from replacing newer values.
-Historical snapshots and tags retain
-the previous files until expiration.
-
### Row-ID Lifetime
| Operation | Effect on row IDs |
@@ -248,7 +178,6 @@ the previous files until expiration.
| Logical delete | Hides IDs without shifting the remaining rows. |
| Ordinary Data Evolution compaction | Preserves IDs, but may change file-group boundaries. |
| Deletion-vector materialization | Assigns new IDs to surviving rows in the rewritten ranges. |
-| Data Evolution incremental clustering | Assigns new IDs to surviving rows in groups selected for sorting; column-version compaction preserves IDs. |
| Explicit `reassign_row_id` maintenance | Changes IDs by remapping file metadata. |
| Overwrite or partition removal | Removes the replaced rows from the current table state; do not assume their IDs remain usable. |
diff --git a/docs/docs/multimodal-table/data-evolution.mdx b/docs/docs/multimodal-table/data-evolution.mdx
index fbe2d4ac88..c57b95a065 100644
--- a/docs/docs/multimodal-table/data-evolution.mdx
+++ b/docs/docs/multimodal-table/data-evolution.mdx
@@ -50,9 +50,7 @@ read work until compaction consolidates them.
Set both `row-tracking.enabled` and `data-evolution.enabled` when creating the
table. These options are immutable. Data Evolution requires an append table
with no primary key and `bucket = -1` (the default for an append table).
-[Incremental clustering](./data-evolution-maintenance#incremental-clustering) is
-supported for tables without dedicated BLOB or VECTOR files. Clustering rewrites
-the selected groups and assigns new row IDs.
+`clustering.incremental` cannot be enabled with Data Evolution.
The following examples assume a configured Paimon catalog and the `default`
database. Deletion vectors are enabled so the same table can also run the
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java
index beb08f07c0..9792602393 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/GlobalIndexer.java
@@ -29,7 +29,6 @@ import javax.annotation.Nullable;
import java.io.IOException;
import java.util.List;
import java.util.concurrent.ExecutorService;
-import java.util.function.LongUnaryOperator;
/** Abstract base class for global indexers. */
public interface GlobalIndexer {
@@ -50,26 +49,6 @@ public interface GlobalIndexer {
@Nullable List<Range> rowRanges,
ExecutorService executor);
- /**
- * Rewrites stored index keys with mapped, relative row IDs without reading source data.
- *
- * <p>The input files share the relative row-ID domain {@code [0, sourceRowCount)}. The mapping
- * returns a row ID in {@code [0, targetRowCount)}, or {@code -1} to exclude a source row.
- * Output files share the target domain and may need to be read together, just like the input
- * files.
- */
- default List<ResultEntry> rewriteRowIds(
- GlobalIndexFileReader reader,
- List<GlobalIndexIOMeta> files,
- long sourceRowCount,
- long targetRowCount,
- LongUnaryOperator mapping,
- GlobalIndexFileWriter writer)
- throws IOException {
- throw new UnsupportedOperationException(
- "This global index does not support rewriting stored row IDs.");
- }
-
static GlobalIndexer create(String type, List<DataField> indexFields, Options options) {
return GlobalIndexerFactoryUtils.load(type).create(indexFields, options);
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexer.java
index a5711abcfc..c63ded522f 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexer.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/bitmap/BitmapGlobalIndexer.java
@@ -20,33 +20,23 @@ package org.apache.paimon.globalindex.bitmap;
import org.apache.paimon.compression.BlockCompressionFactory;
import org.apache.paimon.compression.CompressOptions;
-import org.apache.paimon.fs.PositionOutputStream;
-import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.globalindex.GlobalIndexIOMeta;
import org.apache.paimon.globalindex.GlobalIndexKeyExtractor;
import org.apache.paimon.globalindex.GlobalIndexReader;
import org.apache.paimon.globalindex.GlobalIndexer;
import org.apache.paimon.globalindex.KeySerializer;
-import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.SortedGlobalIndexer;
-import org.apache.paimon.globalindex.SortedIndexFileMeta;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
import org.apache.paimon.options.Options;
import org.apache.paimon.types.DataField;
-import org.apache.paimon.utils.IOUtils;
import org.apache.paimon.utils.Range;
-import org.apache.paimon.utils.RoaringNavigableMap64;
import javax.annotation.Nullable;
import java.io.IOException;
-import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
-import java.util.function.LongUnaryOperator;
-
-import static org.apache.paimon.utils.Preconditions.checkArgument;
/** The {@link GlobalIndexer} for bitmap index. */
public class BitmapGlobalIndexer implements SortedGlobalIndexer {
@@ -96,105 +86,4 @@ public class BitmapGlobalIndexer implements SortedGlobalIndexer {
return new LazyFilteredBitmapReader(
fileReader, files, keySerializer, fallbackScanMaxSize, totalRowCount, executor);
}
-
- @Override
- public List<ResultEntry> rewriteRowIds(
- GlobalIndexFileReader fileReader,
- List<GlobalIndexIOMeta> files,
- long sourceRowCount,
- long targetRowCount,
- LongUnaryOperator mapping,
- GlobalIndexFileWriter fileWriter)
- throws IOException {
- checkArgument(sourceRowCount >= 0 && targetRowCount >= 0, "Invalid row count.");
- List<ResultEntry> results = new ArrayList<>();
- if (targetRowCount == 0) {
- return results;
- }
- for (GlobalIndexIOMeta file : files) {
- try (SeekableInputStream input = fileReader.getInputStream(file)) {
- BitmapGlobalIndexFormat.SeekableReader reader =
- (offset, length) -> {
- input.seek(offset);
- byte[] bytes = new byte[length];
- IOUtils.readFully(input, bytes);
- return bytes;
- };
- BitmapGlobalIndexFormat.Footer footer =
- BitmapGlobalIndexFormat.readFooter(reader, file.fileSize());
- RoaringNavigableMap64 nullRows =
- mapRowIds(
- BitmapGlobalIndexFormat.readBitmap(reader, footer.nullRowsBlock),
- sourceRowCount,
- targetRowCount,
- mapping);
- RoaringNavigableMap64 nonNullRows =
- mapRowIds(
- BitmapGlobalIndexFormat.readBitmap(reader, footer.nonNullRowsBlock),
- sourceRowCount,
- targetRowCount,
- mapping);
- if (nullRows.isEmpty() && nonNullRows.isEmpty()) {
- continue;
- }
- String fileName = fileWriter.newFileName(BitmapGlobalIndexerFactory.IDENTIFIER);
- byte[] firstKey = null;
- byte[] lastKey = null;
- try (PositionOutputStream output = fileWriter.newOutputStream(fileName)) {
- BitmapGlobalIndexFormat.StreamingWriter writer =
- new BitmapGlobalIndexFormat.StreamingWriter(
- output, dictionaryBlockSize, compressionFactory);
- for (BitmapGlobalIndexFormat.DictionaryBlockMeta block :
- BitmapGlobalIndexFormat.readIndexBlock(reader, footer.indexBlock)) {
- for (BitmapGlobalIndexFormat.DictionaryEntry entry :
- BitmapGlobalIndexFormat.readDictionaryBlock(reader, block)
- .entries()) {
- RoaringNavigableMap64 rowIds =
- mapRowIds(
- BitmapGlobalIndexFormat.readBitmap(
- reader, entry.bitmapBlock()),
- sourceRowCount,
- targetRowCount,
- mapping);
- if (!rowIds.isEmpty()) {
- // Dictionary order is unchanged, so encoded keys need no decoding.
- writer.write(entry.key(), rowIds);
- if (firstKey == null) {
- firstKey = entry.key().bytes();
- }
- lastKey = entry.key().bytes();
- }
- }
- }
- writer.finish(nullRows, nonNullRows);
- }
- results.add(
- new ResultEntry(
- fileName,
- nullRows.getLongCardinality() + nonNullRows.getLongCardinality(),
- new SortedIndexFileMeta(firstKey, lastKey, !nullRows.isEmpty())
- .serialize()));
- }
- }
- return results;
- }
-
- private static RoaringNavigableMap64 mapRowIds(
- RoaringNavigableMap64 rows,
- long sourceRowCount,
- long targetRowCount,
- LongUnaryOperator mapping) {
- RoaringNavigableMap64 result = new RoaringNavigableMap64();
- for (long source : rows) {
- checkArgument(
- source >= 0 && source < sourceRowCount, "Invalid source row ID: %s", source);
- long target = mapping.applyAsLong(source);
- checkArgument(
- target >= -1 && target < targetRowCount, "Invalid target row ID: %s", target);
- if (target >= 0) {
- result.add(target);
- }
- }
- return result;
- }
}
diff --git a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java
index 8733bc2e10..e8db64056d 100644
--- a/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java
+++ b/paimon-common/src/main/java/org/apache/paimon/globalindex/btree/BTreeGlobalIndexer.java
@@ -26,7 +26,6 @@ import org.apache.paimon.globalindex.GlobalIndexKeyExtractor;
import org.apache.paimon.globalindex.GlobalIndexReader;
import org.apache.paimon.globalindex.GlobalIndexer;
import org.apache.paimon.globalindex.KeySerializer;
-import org.apache.paimon.globalindex.ResultEntry;
import org.apache.paimon.globalindex.SortedGlobalIndexer;
import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
@@ -38,7 +37,6 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.BloomFilter;
import org.apache.paimon.utils.LazyField;
import org.apache.paimon.utils.Range;
-import org.apache.paimon.utils.RoaringNavigableMap64;
import javax.annotation.Nullable;
@@ -46,7 +44,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
-import java.util.function.LongUnaryOperator;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -155,83 +152,4 @@ public class BTreeGlobalIndexer implements SortedGlobalIndexer {
rowRanges,
executor);
}
-
- @Override
- public List<ResultEntry> rewriteRowIds(
- GlobalIndexFileReader fileReader,
- List<GlobalIndexIOMeta> files,
- long sourceRowCount,
- long targetRowCount,
- LongUnaryOperator mapping,
- GlobalIndexFileWriter fileWriter)
- throws IOException {
- checkArgument(sourceRowCount >= 0 && targetRowCount >= 0, "Invalid row count.");
- List<ResultEntry> results = new ArrayList<>();
- if (targetRowCount == 0) {
- return results;
- }
- for (GlobalIndexIOMeta file : files) {
- try (BTreeIndexReader reader =
- new BTreeIndexReader(
- keySerializer, fileReader, file, cacheManager.get(), null)) {
- RoaringNavigableMap64 nullRows = new RoaringNavigableMap64();
- reader.scanNullRowIds(
- rowId ->
- addMappedRowId(
- nullRows, rowId, sourceRowCount, targetRowCount, mapping));
- BTreeIndexWriter writer = null;
- try {
- if (!nullRows.isEmpty()) {
- writer = createWriter(fileWriter);
- for (long rowId : nullRows) {
- writer.write(null, rowId);
- }
- }
- BTreeIndexReader.EntryIterator entries = reader.entryIterator();
- while (entries.hasNext()) {
- BTreeIndexReader.KeyRowIds entry = entries.next();
- RoaringNavigableMap64 rowIds = new RoaringNavigableMap64();
- for (long rowId : entry.rowIds()) {
- addMappedRowId(rowIds, rowId, sourceRowCount, targetRowCount, mapping);
- }
- if (!rowIds.isEmpty()) {
- if (writer == null) {
- writer = createWriter(fileWriter);
- }
- // A permutation can reverse a posting list; v2 requires sorted,
- // unique IDs.
- for (long rowId : rowIds) {
- writer.write(entry.key(), rowId);
- }
- }
- }
- if (writer != null) {
- results.addAll(writer.finish());
- }
- } finally {
- if (writer != null) {
- writer.close();
- }
- }
- }
- }
- return results;
- }
-
- private static void addMappedRowId(
- RoaringNavigableMap64 result,
- long sourceRowId,
- long sourceRowCount,
- long targetRowCount,
- LongUnaryOperator mapping) {
- checkArgument(
- sourceRowId >= 0 && sourceRowId < sourceRowCount,
- "Invalid source row ID: %s",
- sourceRowId);
- long target = mapping.applyAsLong(sourceRowId);
- checkArgument(target >= -1 && target < targetRowCount, "Invalid target row ID: %s", target);
- if (target >= 0) {
- result.add(target);
- }
- }
}
diff --git a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexRowIdRewriteTest.java b/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexRowIdRewriteTest.java
deleted file mode 100644
index 84cf6b9963..0000000000
--- a/paimon-common/src/test/java/org/apache/paimon/globalindex/GlobalIndexRowIdRewriteTest.java
+++ /dev/null
@@ -1,213 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.globalindex;
-
-import org.apache.paimon.fs.FileIO;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.PositionOutputStream;
-import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.globalindex.bitmap.BitmapGlobalIndexer;
-import org.apache.paimon.globalindex.btree.BTreeGlobalIndexer;
-import org.apache.paimon.globalindex.btree.BTreeIndexOptions;
-import org.apache.paimon.globalindex.io.GlobalIndexFileReader;
-import org.apache.paimon.globalindex.io.GlobalIndexFileWriter;
-import org.apache.paimon.options.Options;
-import org.apache.paimon.predicate.FieldRef;
-import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.IntType;
-import org.apache.paimon.utils.Pair;
-
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Optional;
-import java.util.UUID;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.apache.paimon.shade.guava30.com.google.common.util.concurrent.MoreExecutors.newDirectExecutorService;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests scalar index payload rewrites through query results. */
-class GlobalIndexRowIdRewriteTest {
-
- private static final DataField FIELD = new DataField(0, "value", new IntType());
- private static final FieldRef REF = new FieldRef(0, "value", new IntType());
-
- @TempDir java.nio.file.Path tempPath;
-
- private FileIO fileIO;
- private Path basePath;
- private GlobalIndexFileReader fileReader;
- private GlobalIndexFileWriter fileWriter;
- private AtomicInteger outputFiles;
-
- @BeforeEach
- void setUp() {
- fileIO = LocalFileIO.create();
- basePath = new Path(tempPath.toUri());
- outputFiles = new AtomicInteger();
- fileReader = meta -> fileIO.newInputStream(meta.filePath());
- fileWriter =
- new GlobalIndexFileWriter() {
- @Override
- public String newFileName(String prefix) {
- return prefix + "-" + UUID.randomUUID();
- }
-
- @Override
- public PositionOutputStream newOutputStream(String fileName)
- throws IOException {
- outputFiles.incrementAndGet();
- return fileIO.newOutputStream(new Path(basePath, fileName), false);
- }
- };
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree-v1", "btree-v2", "bitmap"})
- void testPermutationAndRemovalAcrossPayloadFiles(String type) throws Exception {
- GlobalIndexer indexer = indexer(type);
- List<GlobalIndexIOMeta> source = new ArrayList<>();
- source.addAll(write(indexer, 0, 2, null, 1, 2));
- source.addAll(write(indexer, 4, -3, 4, null, 2));
- long[] mapping = {3, 1, -1, 0, 4, -1, 2, 5};
- List<ResultEntry> rewritten =
- indexer.rewriteRowIds(
- fileReader, source, 8, 6, id -> mapping[(int) id], fileWriter);
- assertThat(rewritten).hasSize(2);
- assertThat(rewritten.stream().mapToLong(ResultEntry::rowCount).sum()).isEqualTo(6);
- try (GlobalIndexReader reader = reader(indexer, rewritten, 6)) {
- assertRows(reader.visitEqual(REF, 2).join(), 0, 3, 5);
- assertRows(reader.visitEqual(REF, -3).join(), 4);
- assertRows(reader.visitEqual(REF, 1).join());
- assertRows(reader.visitEqual(REF, 4).join());
- assertRows(reader.visitIsNull(REF).join(), 1, 2);
- assertRows(reader.visitIsNotNull(REF).join(), 0, 3, 4, 5);
- assertRows(reader.visitNotEqual(REF, 2).join(), 4);
- assertRows(reader.visitLessThan(REF, 0).join(), 4);
- }
- // A rewrite must leave the old payload usable by readers pinned to its snapshot.
- try (GlobalIndexReader reader =
- indexer.createReader(fileReader, source, 8, null, newDirectExecutorService())) {
- assertRows(reader.visitEqual(REF, 2).join(), 0, 3, 7);
- assertRows(reader.visitIsNull(REF).join(), 1, 6);
- }
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree-v1", "btree-v2", "bitmap"})
- void testMappedPostingIdsAreSortedAndUnique(String type) throws Exception {
- GlobalIndexer indexer = indexer(type);
- List<GlobalIndexIOMeta> source = write(indexer, 0, 7, 7, null, 7);
- long[] mapping = {2, 1, 0, 1};
- List<ResultEntry> rewritten =
- indexer.rewriteRowIds(
- fileReader, source, 4, 3, id -> mapping[(int) id], fileWriter);
- assertThat(rewritten).hasSize(1);
- assertThat(rewritten.get(0).rowCount()).isEqualTo(3);
- try (GlobalIndexReader reader = reader(indexer, rewritten, 3)) {
- assertRows(reader.visitEqual(REF, 7).join(), 1, 2);
- assertRows(reader.visitIsNull(REF).join(), 0);
- assertRows(reader.visitNotEqual(REF, 8).join(), 1, 2);
- }
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree-v1", "btree-v2", "bitmap"})
- void testNullOnlyAndFullyExcludedPayloads(String type) throws Exception {
- GlobalIndexer indexer = indexer(type);
- List<GlobalIndexIOMeta> source = write(indexer, 0, null, null);
- List<ResultEntry> rewritten =
- indexer.rewriteRowIds(fileReader, source, 2, 2, id -> 1 - id, fileWriter);
- try (GlobalIndexReader reader = reader(indexer, rewritten, 2)) {
- assertRows(reader.visitIsNull(REF).join(), 0, 1);
- assertRows(reader.visitIsNotNull(REF).join());
- }
- int before = outputFiles.get();
- assertThat(indexer.rewriteRowIds(fileReader, source, 2, 1, id -> -1, fileWriter)).isEmpty();
- assertThat(indexer.rewriteRowIds(fileReader, source, 2, 0, id -> -1, fileWriter)).isEmpty();
- assertThat(outputFiles).hasValue(before);
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree-v1", "btree-v2", "bitmap"})
- void testRejectsMappingOutsideTargetDomain(String type) throws Exception {
- GlobalIndexer indexer = indexer(type);
- List<GlobalIndexIOMeta> source = write(indexer, 0, 7);
- assertThatThrownBy(
- () -> indexer.rewriteRowIds(fileReader, source, 1, 1, id -> 1, fileWriter))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Invalid target row ID");
- }
-
- private GlobalIndexer indexer(String type) {
- Options options = new Options();
- if (type.equals("bitmap")) {
- return new BitmapGlobalIndexer(FIELD, options);
- }
- options.set(BTreeIndexOptions.BTREE_INDEX_FILE_VERSION, type.equals("btree-v1") ? 1 : 2);
- return new BTreeGlobalIndexer(Collections.singletonList(FIELD), options);
- }
-
- private List<GlobalIndexIOMeta> write(GlobalIndexer indexer, long start, Integer... values)
- throws IOException {
- List<Pair<Integer, Long>> rows = new ArrayList<>();
- for (int i = 0; i < values.length; i++) {
- rows.add(Pair.of(values[i], start + i));
- }
- Comparator<Integer> comparator = Comparator.nullsFirst(Integer::compareTo);
- rows.sort((left, right) -> comparator.compare(left.getKey(), right.getKey()));
- GlobalIndexSingleColumnWriter writer =
- (GlobalIndexSingleColumnWriter) indexer.createWriter(fileWriter);
- for (Pair<Integer, Long> row : rows) {
- writer.write(row.getKey(), row.getValue());
- }
- return toMetas(writer.finish());
- }
-
- private GlobalIndexReader reader(
- GlobalIndexer indexer, List<ResultEntry> entries, long rowCount) throws IOException {
- return indexer.createReader(
- fileReader, toMetas(entries), rowCount, null, newDirectExecutorService());
- }
-
- private List<GlobalIndexIOMeta> toMetas(List<ResultEntry> entries) throws IOException {
- List<GlobalIndexIOMeta> result = new ArrayList<>();
- for (ResultEntry entry : entries) {
- Path path = new Path(basePath, entry.fileName());
- result.add(
- new GlobalIndexIOMeta(
- path, fileIO.getFileSize(path), entry.rowCount(), entry.meta()));
- }
- return result;
- }
-
- private void assertRows(Optional<GlobalIndexResult> result, long... expected) {
- assertThat(result).isPresent();
- assertThat(result.get().results().toArray(Integer.MAX_VALUE)).containsExactly(expected);
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/DataEvolutionClusterCompaction.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/DataEvolutionClusterCompaction.java
deleted file mode 100644
index 58ee72a1ff..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/DataEvolutionClusterCompaction.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.cluster;
-
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
-import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask;
-import org.apache.paimon.compact.CompactUnit;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.partition.PartitionPredicate;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.utils.RangeHelper;
-
-import javax.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
-import static org.apache.paimon.types.VectorType.isVectorStoreFile;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-
-/** Compacts column versions left outside the incremental clustering selection. */
-public final class DataEvolutionClusterCompaction {
-
- private DataEvolutionClusterCompaction() {}
-
- public static List<DataEvolutionCompactTask> plan(
- FileStoreTable table,
- Snapshot snapshot,
- @Nullable PartitionPredicate predicate,
- Map<BinaryRow, CompactUnit> clusteringUnits) {
- if (!table.coreOptions().dataEvolutionEnabled()) {
- return Collections.emptyList();
- }
- Map<BinaryRow, List<DataFileMeta>> partitions = new HashMap<>();
- for (DataSplit split :
- table.newSnapshotReader()
- .dropStats()
- .withSnapshot(snapshot)
- .withPartitionFilter(predicate)
- .read()
- .dataSplits()) {
- partitions
- .computeIfAbsent(split.partition(), ignored -> new ArrayList<>())
- .addAll(split.dataFiles());
- }
- DataEvolutionIncrementalClusterRewriter rewriter =
- new DataEvolutionIncrementalClusterRewriter(table, snapshot);
- RangeHelper<DataFileMeta> helper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange);
- List<DataEvolutionCompactTask> tasks = new ArrayList<>();
- partitions.forEach(
- (partition, files) -> {
- CompactUnit clustering = clusteringUnits.get(partition);
- Set<String> selected =
- clustering == null
- ? Collections.emptySet()
- : clustering.files().stream()
- .map(DataFileMeta::fileName)
- .collect(Collectors.toSet());
- List<DataFileMeta> normalFiles =
- files.stream()
- .filter(
- file ->
- !isBlobFile(file.fileName())
- && !isVectorStoreFile(file.fileName()))
- .collect(Collectors.toList());
- for (List<DataFileMeta> group : helper.mergeOverlappingRanges(normalFiles)) {
- checkArgument(
- helper.areAllRangesSame(group),
- "Normal files must have equal row-id ranges.");
- if (group.size() < Math.max(2, table.coreOptions().compactionMinFileNum())
- || group.stream()
- .anyMatch(file -> selected.contains(file.fileName()))) {
- continue;
- }
- // A single group preserves even LOCAL_SORT order when its level is trusted.
- // Dirty groups still compact at L0 without claiming that they are sorted.
- tasks.add(
- new DataEvolutionNormalCompactTask(
- partition,
- group,
- Collections.emptyList(),
- rewriter.groupLevel(group)));
- }
- });
- return tasks;
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/DataEvolutionIncrementalClusterRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/DataEvolutionIncrementalClusterRewriter.java
deleted file mode 100644
index 90269cd4bb..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/DataEvolutionIncrementalClusterRewriter.java
+++ /dev/null
@@ -1,217 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.cluster;
-
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.dataevolution.DataEvolutionCompactDeletionVectorRewriter;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
-import org.apache.paimon.io.CompactIncrement;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataIncrement;
-import org.apache.paimon.mergetree.LevelSortedRun;
-import org.apache.paimon.mergetree.SortedRun;
-import org.apache.paimon.schema.TableSchema;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.table.source.SplitGenerator;
-import org.apache.paimon.utils.Pair;
-import org.apache.paimon.utils.RangeHelper;
-
-import javax.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
-import static org.apache.paimon.types.VectorType.isVectorStoreFile;
-import static org.apache.paimon.utils.DataEvolutionUtils.fileFieldIds;
-import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-
-/** Preserves complete row-id groups and materializes deletion vectors for clustering. */
-class DataEvolutionIncrementalClusterRewriter extends IncrementalClusterRewriter {
-
- private final FileStoreTable table;
- private final Snapshot snapshot;
- private final List<String> clusterKeys;
- private final Set<Integer> clusterFieldIds;
- private final Map<Long, TableSchema> schemas = new HashMap<>();
- private final Map<Pair<Long, List<String>>, Set<Integer>> writtenFieldIds = new HashMap<>();
-
- DataEvolutionIncrementalClusterRewriter(FileStoreTable table, Snapshot snapshot) {
- this.table = table;
- this.snapshot = snapshot;
- this.clusterKeys = table.coreOptions().clusteringColumns();
- this.clusterFieldIds =
- clusterKeys.stream()
- // A partition key is constant within a group, including partial writes.
- .filter(key -> !table.partitionKeys().contains(key))
- .map(key -> table.rowType().getField(key).id())
- .collect(Collectors.toSet());
- }
-
- @Override
- List<LevelSortedRun> constructPartitionLevels(List<DataFileMeta> files) {
- checkArgument(
- files.stream()
- .noneMatch(
- file ->
- isBlobFile(file.fileName())
- || isVectorStoreFile(file.fileName())),
- "Data evolution incremental clustering does not support dedicated BLOB/VECTOR files.");
- RangeHelper<DataFileMeta> helper = new RangeHelper<>(DataFileMeta::nonNullRowIdRange);
- List<LevelSortedRun> runs = new ArrayList<>();
- Map<Integer, List<DataFileMeta>> higherLevels = new HashMap<>();
- for (List<DataFileMeta> group : helper.mergeOverlappingRanges(files)) {
- checkArgument(
- helper.areAllRangesSame(group), "Normal files must have equal row-id ranges.");
- DataFileMeta anchor = retrieveAnchorFile(group, file -> file);
- // The shared full planner reads the first file's clustering schema.
- Collections.swap(group, 0, group.indexOf(anchor));
- int level = groupLevel(group, anchor);
- if (level == 0) {
- runs.add(new LevelSortedRun(0, SortedRun.fromSorted(group)));
- } else {
- higherLevels.computeIfAbsent(level, ignored -> new ArrayList<>()).addAll(group);
- }
- }
- higherLevels.forEach(
- (level, group) -> runs.add(new LevelSortedRun(level, SortedRun.fromSorted(group))));
- runs.sort(Comparator.comparingInt(LevelSortedRun::level));
- return runs;
- }
-
- int groupLevel(List<DataFileMeta> group) {
- return groupLevel(group, retrieveAnchorFile(group, file -> file));
- }
-
- private int groupLevel(List<DataFileMeta> group, DataFileMeta anchor) {
- int level = anchor.level();
- CoreOptions anchorOptions = CoreOptions.fromMap(schema(anchor.schemaId()).options());
- if (!fieldIds(anchor).containsAll(clusterFieldIds)
- || !anchorOptions.clusteringColumns().equals(clusterKeys)
- || anchorOptions.clusteringStrategy(clusterKeys.size())
- != table.coreOptions().clusteringStrategy(clusterKeys.size())) {
- // The shared full planner checks only the first file's schema. Do not let a newer
- // non-key update or another group hide an incompatible clustering definition.
- return 0;
- }
- for (DataFileMeta file : group) {
- if (file.level() < level && !Collections.disjoint(fieldIds(file), clusterFieldIds)) {
- // Keep every column version in the run, but only key writes lower its level.
- level = file.level();
- }
- }
- return level;
- }
-
- private Set<Integer> fieldIds(DataFileMeta file) {
- return writtenFieldIds.computeIfAbsent(
- Pair.of(file.schemaId(), file.writeCols()),
- ignored -> fileFieldIds(schema(file.schemaId()), file));
- }
-
- private TableSchema schema(long schemaId) {
- return schemas.computeIfAbsent(schemaId, id -> table.schemaManager().schema(id));
- }
-
- @Override
- void configureSplit(
- DataSplit.Builder builder,
- SplitGenerator.SplitGroup splitGroup,
- @Nullable AppendDeleteFileMaintainer dvMaintainer) {
- builder.rawConvertible(false);
- if (dvMaintainer != null) {
- builder.withDataDeletionFiles(
- splitGroup.files.stream()
- .map(file -> dvMaintainer.getDeletionFile(file.fileName()))
- .collect(Collectors.toList()));
- }
- }
-
- @Override
- @Nullable
- CommitMessage finishPartition(
- @Nullable AppendDeleteFileMaintainer dvMaintainer, int totalBuckets) {
- // Keep source deletion vectors intact until the rewrite commits.
- return null;
- }
-
- @Override
- public List<CommitMessage> prepareCommitMessages(List<CommitMessage> messages) {
- Map<BinaryRow, CompactIncrement> increments = new HashMap<>();
- List<CommitMessage> prepared = new ArrayList<>();
- for (CommitMessage message : messages) {
- CommitMessageImpl impl = (CommitMessageImpl) message;
- checkArgument(
- impl.bucket() == 0 && impl.newFilesIncrement().isEmpty(),
- "Data evolution clustering requires compact messages for unaware bucket 0.");
- CompactIncrement increment = impl.compactIncrement();
- if (!increment.compactAfter().isEmpty()
- && increment.compactAfter().stream()
- .allMatch(file -> file.firstRowId() != null)) {
- // Keep row-preserving tasks separate: their DVs move to the replacement anchor,
- // whereas clustering outputs have already applied DVs and need fresh row IDs.
- prepared.add(message);
- continue;
- }
- CompactIncrement merged =
- increments.computeIfAbsent(
- message.partition(),
- ignored ->
- new CompactIncrement(
- new ArrayList<>(),
- new ArrayList<>(),
- new ArrayList<>(),
- new ArrayList<>(),
- new ArrayList<>()));
- merged.compactBefore().addAll(increment.compactBefore());
- merged.compactAfter().addAll(increment.compactAfter());
- merged.changelogFiles().addAll(increment.changelogFiles());
- merged.newIndexFiles().addAll(increment.newIndexFiles());
- merged.deletedIndexFiles().addAll(increment.deletedIndexFiles());
- }
- increments.forEach(
- (partition, increment) -> {
- checkArgument(
- !increment.compactBefore().isEmpty(),
- "Data evolution clustering output must include its source files.");
- prepared.add(
- new CommitMessageImpl(
- partition, 0, null, DataIncrement.emptyIncrement(), increment));
- });
- // The committer maintains global indexes against the latest snapshot on every attempt.
- prepared.addAll(
- new DataEvolutionCompactDeletionVectorRewriter(table, snapshot)
- .rewriteDeletionVectors(prepared));
- return prepared.stream()
- .map(message -> ((CommitMessageImpl) message).withCheckFromSnapshot(snapshot.id()))
- .collect(Collectors.toList());
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java
index d7d5081366..8acbb8e778 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/cluster/HistoryPartitionCluster.java
@@ -44,9 +44,9 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
-import java.util.function.Function;
import java.util.stream.Collectors;
+import static org.apache.paimon.append.cluster.IncrementalClusterManager.constructPartitionLevels;
import static org.apache.paimon.append.cluster.IncrementalClusterManager.logForPartitionLevel;
/** Handle historical partition for full clustering. */
@@ -62,7 +62,6 @@ public class HistoryPartitionCluster {
private final Duration historyPartitionIdleTime;
private final int historyPartitionLimit;
private final int maxLevel;
- private final Function<List<DataFileMeta>, List<LevelSortedRun>> levelConstructor;
public HistoryPartitionCluster(
FileStoreTable table,
@@ -71,8 +70,7 @@ public class HistoryPartitionCluster {
InternalRowPartitionComputer partitionComputer,
PartitionPredicate specifiedPartitions,
Duration historyPartitionIdleTime,
- int historyPartitionLimit,
- Function<List<DataFileMeta>, List<LevelSortedRun>> levelConstructor) {
+ int historyPartitionLimit) {
this.table = table;
this.snapshot = snapshot;
this.incrementalClusterStrategy = incrementalClusterStrategy;
@@ -81,7 +79,6 @@ public class HistoryPartitionCluster {
this.historyPartitionIdleTime = historyPartitionIdleTime;
this.historyPartitionLimit = historyPartitionLimit;
this.maxLevel = table.coreOptions().numLevels() - 1;
- this.levelConstructor = levelConstructor;
}
@Nullable
@@ -90,8 +87,7 @@ public class HistoryPartitionCluster {
Snapshot snapshot,
IncrementalClusterStrategy incrementalClusterStrategy,
InternalRowPartitionComputer partitionComputer,
- @Nullable PartitionPredicate specifiedPartitions,
- Function<List<DataFileMeta>, List<LevelSortedRun>> levelConstructor) {
+ @Nullable PartitionPredicate specifiedPartitions) {
if (table.schema().partitionKeys().isEmpty()) {
return null;
}
@@ -112,8 +108,7 @@ public class HistoryPartitionCluster {
partitionComputer,
specifiedPartitions,
idleTime,
- limit,
- levelConstructor);
+ limit);
}
public Map<BinaryRow, CompactUnit> pickForHistoryPartitions() {
@@ -168,7 +163,7 @@ public class HistoryPartitionCluster {
.collect(
Collectors.toMap(
Map.Entry::getKey,
- entry -> levelConstructor.apply(entry.getValue())));
+ entry -> constructPartitionLevels(entry.getValue())));
}
private Map<BinaryRow, List<DataFileMeta>> filterPartitions(
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
index 6c196c3b04..f5642f6648 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterManager.java
@@ -25,13 +25,21 @@ import org.apache.paimon.compact.CompactUnit;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
+import org.apache.paimon.index.IndexFileMeta;
+import org.apache.paimon.io.CompactIncrement;
import org.apache.paimon.io.DataFileMeta;
+import org.apache.paimon.io.DataIncrement;
+import org.apache.paimon.manifest.FileKind;
+import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.mergetree.LevelSortedRun;
+import org.apache.paimon.mergetree.SortedRun;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.table.BucketMode;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.sink.CommitMessage;
+import org.apache.paimon.table.sink.CommitMessageImpl;
import org.apache.paimon.table.source.DataSplit;
+import org.apache.paimon.table.source.DeletionFile;
import org.apache.paimon.table.source.SplitGenerator;
import org.apache.paimon.table.source.snapshot.SnapshotReader;
import org.apache.paimon.utils.InternalRowPartitionComputer;
@@ -44,6 +52,7 @@ import javax.annotation.Nullable;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.Comparator;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -62,17 +71,18 @@ public class IncrementalClusterManager {
private final InternalRowPartitionComputer partitionComputer;
private final Snapshot snapshot;
private final SnapshotReader snapshotReader;
- private final IncrementalClusterRewriter rewriter;
private final IncrementalClusterStrategy incrementalClusterStrategy;
private final CoreOptions.OrderType clusterCurve;
private final List<String> clusterKeys;
private final int numLevels;
private final @Nullable HistoryPartitionCluster historyPartitionCluster;
+ public IncrementalClusterManager(FileStoreTable table) {
+ this(table, null);
+ }
+
public IncrementalClusterManager(
- FileStoreTable table,
- @Nullable PartitionPredicate specifiedPartitions,
- Snapshot snapshot) {
+ FileStoreTable table, @Nullable PartitionPredicate specifiedPartitions) {
checkArgument(
table.bucketMode() == BucketMode.BUCKET_UNAWARE,
"only append unaware-bucket table support incremental clustering.");
@@ -89,8 +99,7 @@ public class IncrementalClusterManager {
table.store().partitionType(),
table.partitionKeys().toArray(new String[0]),
table.coreOptions().legacyPartitionName());
- this.snapshot = snapshot;
- this.rewriter = IncrementalClusterRewriter.create(table, snapshot);
+ this.snapshot = table.snapshotManager().latestSnapshot();
this.snapshotReader =
table.newSnapshotReader()
.dropStats()
@@ -111,15 +120,10 @@ public class IncrementalClusterManager {
snapshot,
incrementalClusterStrategy,
partitionComputer,
- specifiedPartitions,
- rewriter::constructPartitionLevels);
+ specifiedPartitions);
}
public Map<BinaryRow, CompactUnit> createCompactUnits(boolean fullCompaction) {
- if (snapshot == null) {
- return Collections.emptyMap();
- }
-
// 1. construct LSM structure for each partition
Map<BinaryRow, List<LevelSortedRun>> partitionLevels = constructLevels();
logForPartitionLevel(partitionLevels, partitionComputer);
@@ -173,7 +177,33 @@ public class IncrementalClusterManager {
.collect(
Collectors.toMap(
Map.Entry::getKey,
- entry -> rewriter.constructPartitionLevels(entry.getValue())));
+ entry -> constructPartitionLevels(entry.getValue())));
+ }
+
+ public static List<LevelSortedRun> constructPartitionLevels(List<DataFileMeta> partitionFiles) {
+ List<LevelSortedRun> partitionLevels = new ArrayList<>();
+ Map<Integer, List<DataFileMeta>> levelMap =
+ partitionFiles.stream().collect(Collectors.groupingBy(DataFileMeta::level));
+
+ for (Map.Entry<Integer, List<DataFileMeta>> entry : levelMap.entrySet()) {
+ int level = entry.getKey();
+ if (level == 0) {
+ for (DataFileMeta level0File : entry.getValue()) {
+ partitionLevels.add(
+ new LevelSortedRun(level, SortedRun.fromSingle(level0File)));
+ }
+ } else {
+ // don't need to guarantee that the files within the same sorted run are
+ // non-overlapping here, so we call SortedRun.fromSorted() to avoid sorting and
+ // validation
+ partitionLevels.add(
+ new LevelSortedRun(level, SortedRun.fromSorted(entry.getValue())));
+ }
+ }
+
+ // sort by level
+ partitionLevels.sort(Comparator.comparing(LevelSortedRun::level));
+ return partitionLevels;
}
public Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> toSplitsAndRewriteDvFiles(
@@ -191,7 +221,6 @@ public class IncrementalClusterManager {
DataSplit.Builder builder =
DataSplit.builder()
- .withSnapshot(snapshot.id())
.withPartition(partition)
.withBucket(0)
.withTotalBuckets(1)
@@ -206,14 +235,48 @@ public class IncrementalClusterManager {
String bucketPath =
snapshotReader.pathFactory().bucketPath(partition, 0).toString();
- builder.withDataFiles(dataFiles).withBucketPath(bucketPath);
-
- rewriter.configureSplit(builder, splitGroup, dvMaintainer);
+ builder.withDataFiles(dataFiles)
+ .rawConvertible(splitGroup.rawConvertible)
+ .withBucketPath(bucketPath);
+
+ if (dvMaintainer != null) {
+ List<DeletionFile> dataDeletionFiles = new ArrayList<>();
+ for (DataFileMeta file : dataFiles) {
+ DeletionFile deletionFile =
+ dvMaintainer.notifyRemovedDeletionVector(file.fileName());
+ dataDeletionFiles.add(deletionFile);
+ }
+ builder.withDataDeletionFiles(dataDeletionFiles);
+ }
splits.add(builder.build());
}
- CommitMessage dvCommitMessage =
- rewriter.finishPartition(dvMaintainer, table.coreOptions().bucket());
+ // generate delete dv index meta
+ CommitMessage dvCommitMessage = null;
+ if (dvMaintainer != null) {
+ List<IndexFileMeta> newIndexFiles = new ArrayList<>();
+ List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
+ List<IndexManifestEntry> indexEntries = dvMaintainer.persist();
+ for (IndexManifestEntry entry : indexEntries) {
+ if (entry.kind() == FileKind.ADD) {
+ newIndexFiles.add(entry.indexFile());
+ } else {
+ deletedIndexFiles.add(entry.indexFile());
+ }
+ }
+ dvCommitMessage =
+ new CommitMessageImpl(
+ dvMaintainer.getPartition(),
+ 0,
+ table.coreOptions().bucket(),
+ DataIncrement.emptyIncrement(),
+ new CompactIncrement(
+ Collections.emptyList(),
+ Collections.emptyList(),
+ Collections.emptyList(),
+ newIndexFiles,
+ deletedIndexFiles));
+ }
result.put(partition, Pair.of(splits, dvCommitMessage));
}
@@ -221,10 +284,6 @@ public class IncrementalClusterManager {
return result;
}
- public List<CommitMessage> prepareCommitMessages(List<CommitMessage> messages) {
- return rewriter.prepareCommitMessages(messages);
- }
-
public static List<DataFileMeta> upgrade(
List<DataFileMeta> filesAfterCluster, int outputLevel) {
return filesAfterCluster.stream()
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterRewriter.java
deleted file mode 100644
index d021e2c815..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/cluster/IncrementalClusterRewriter.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.cluster;
-
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.deletionvectors.append.AppendDeleteFileMaintainer;
-import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.io.CompactIncrement;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataIncrement;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.mergetree.LevelSortedRun;
-import org.apache.paimon.mergetree.SortedRun;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.table.source.SplitGenerator;
-
-import javax.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.List;
-import java.util.Map;
-import java.util.stream.Collectors;
-
-/** Table-specific file and deletion-vector semantics for incremental clustering. */
-public class IncrementalClusterRewriter {
-
- public static IncrementalClusterRewriter create(FileStoreTable table, Snapshot snapshot) {
- return table.coreOptions().dataEvolutionEnabled()
- ? new DataEvolutionIncrementalClusterRewriter(table, snapshot)
- : new IncrementalClusterRewriter();
- }
-
- List<LevelSortedRun> constructPartitionLevels(List<DataFileMeta> partitionFiles) {
- List<LevelSortedRun> partitionLevels = new ArrayList<>();
- Map<Integer, List<DataFileMeta>> levelMap =
- partitionFiles.stream().collect(Collectors.groupingBy(DataFileMeta::level));
-
- for (Map.Entry<Integer, List<DataFileMeta>> entry : levelMap.entrySet()) {
- int level = entry.getKey();
- if (level == 0) {
- for (DataFileMeta level0File : entry.getValue()) {
- partitionLevels.add(
- new LevelSortedRun(level, SortedRun.fromSingle(level0File)));
- }
- } else {
- // don't need to guarantee that the files within the same sorted run are
- // non-overlapping here, so we call SortedRun.fromSorted() to avoid sorting and
- // validation
- partitionLevels.add(
- new LevelSortedRun(level, SortedRun.fromSorted(entry.getValue())));
- }
- }
-
- // sort by level
- partitionLevels.sort(Comparator.comparing(LevelSortedRun::level));
- return partitionLevels;
- }
-
- void configureSplit(
- DataSplit.Builder builder,
- SplitGenerator.SplitGroup splitGroup,
- @Nullable AppendDeleteFileMaintainer dvMaintainer) {
- builder.rawConvertible(splitGroup.rawConvertible);
- if (dvMaintainer != null) {
- builder.withDataDeletionFiles(
- splitGroup.files.stream()
- .map(file -> dvMaintainer.notifyRemovedDeletionVector(file.fileName()))
- .collect(Collectors.toList()));
- }
- }
-
- @Nullable
- CommitMessage finishPartition(
- @Nullable AppendDeleteFileMaintainer dvMaintainer, int totalBuckets) {
- if (dvMaintainer == null) {
- return null;
- }
- List<IndexFileMeta> newIndexFiles = new ArrayList<>();
- List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
- for (IndexManifestEntry entry : dvMaintainer.persist()) {
- if (entry.kind() == FileKind.ADD) {
- newIndexFiles.add(entry.indexFile());
- } else {
- deletedIndexFiles.add(entry.indexFile());
- }
- }
- return new CommitMessageImpl(
- dvMaintainer.getPartition(),
- 0,
- totalBuckets,
- DataIncrement.emptyIncrement(),
- new CompactIncrement(
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- newIndexFiles,
- deletedIndexFiles));
- }
-
- public List<CommitMessage> prepareCommitMessages(List<CommitMessage> messages) {
- return messages;
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/ClusteringCommitChangesRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/ClusteringCommitChangesRewriter.java
deleted file mode 100644
index 932362bb29..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/ClusteringCommitChangesRewriter.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.Snapshot.CommitKind;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.operation.commit.CommitChanges;
-import org.apache.paimon.table.FileStoreTable;
-
-import java.util.Set;
-import java.util.stream.Collectors;
-
-/** Remaps scalar indexes after clustering outputs receive their final row IDs. */
-public final class ClusteringCommitChangesRewriter extends DataEvolutionCommitChangesRewriter {
-
- public ClusteringCommitChangesRewriter(FileStoreTable table) {
- super(table);
- }
-
- @Override
- public Attempt createAttempt(CommitKind kind, CommitChanges changes) {
- if (kind != CommitKind.COMPACT) {
- return Attempt.IDENTITY;
- }
-
- // Side files remain attached to committed data. Only outputs awaiting their first row-ID
- // assignment carry a permutation for this commit; later metadata rewrites must ignore it.
- Set<String> mappedClusteringOutputs =
- changes.tableFiles.stream()
- .filter(entry -> entry.kind() == FileKind.ADD)
- .filter(entry -> entry.file().firstRowId() == null)
- .filter(
- entry ->
- entry.file()
- .extraFiles()
- .contains(
- ClusteringRowIdMapping.fileName(
- entry.file().fileName())))
- .map(entry -> entry.file().fileName())
- .collect(Collectors.toSet());
- return indexRewriteAttempt(changes, mappedClusteringOutputs);
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/ClusteringRowIdMapping.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/ClusteringRowIdMapping.java
deleted file mode 100644
index 09a62c8eba..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/ClusteringRowIdMapping.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.fs.FileIO;
-import org.apache.paimon.fs.Path;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.Closeable;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.IOException;
-
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-
-/** Old row IDs in the physical row order of a clustering output file. */
-public final class ClusteringRowIdMapping {
-
- private static final String PREFIX = "cluster-row-id-";
- private static final String SUFFIX = ".mapping";
- private static final int MAGIC = 0x43524944;
- private static final int VERSION = 1;
-
- private ClusteringRowIdMapping() {}
-
- public static String fileName(String dataFileName) {
- return PREFIX + dataFileName + SUFFIX;
- }
-
- public static boolean isMappingFile(String fileName) {
- return fileName.startsWith(PREFIX) && fileName.endsWith(SUFFIX);
- }
-
- public static Writer createWriter(FileIO fileIO, Path path) throws IOException {
- return new Writer(fileIO, path);
- }
-
- public static void read(FileIO fileIO, Path path, long rowCount, RowIdConsumer consumer)
- throws IOException {
- checkArgument(rowCount >= 0, "Invalid clustering row count: %s", rowCount);
- try (DataInputStream in =
- new DataInputStream(new BufferedInputStream(fileIO.newInputStream(path)))) {
- checkArgument(in.readInt() == MAGIC, "Invalid clustering row ID mapping: %s", path);
- checkArgument(
- in.readInt() == VERSION, "Unsupported clustering row ID mapping: %s", path);
- for (long ordinal = 0; ordinal < rowCount; ordinal++) {
- long oldRowId = in.readLong();
- checkArgument(oldRowId >= 0, "Invalid old row ID: %s", oldRowId);
- consumer.accept(ordinal, oldRowId);
- }
- checkArgument(in.read() == -1, "Unexpected trailing clustering row IDs: %s", path);
- }
- }
-
- /** Streaming consumer; the ordinal is local to the output data file. */
- @FunctionalInterface
- public interface RowIdConsumer {
- void accept(long ordinal, long oldRowId) throws IOException;
- }
-
- /**
- * Streaming writer, also used when splitting a spilled sort output at actual file boundaries.
- */
- public static final class Writer implements Closeable {
-
- private final DataOutputStream out;
-
- private Writer(FileIO fileIO, Path path) throws IOException {
- out =
- new DataOutputStream(
- new BufferedOutputStream(fileIO.newOutputStream(path, false)));
- try {
- out.writeInt(MAGIC);
- out.writeInt(VERSION);
- } catch (IOException | RuntimeException | Error failure) {
- try {
- out.close();
- } catch (IOException closeFailure) {
- failure.addSuppressed(closeFailure);
- }
- fileIO.deleteQuietly(path);
- throw failure;
- }
- }
-
- public void write(long oldRowId) throws IOException {
- checkArgument(oldRowId >= 0, "Invalid old row ID: %s", oldRowId);
- out.writeLong(oldRowId);
- }
-
- @Override
- public void close() throws IOException {
- out.close();
- }
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionBlobCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionBlobCompactTask.java
index 33154ff3d9..54af1361aa 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionBlobCompactTask.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionBlobCompactTask.java
@@ -49,7 +49,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
-import static org.apache.paimon.compact.CompactOptions.REWRITE_OPTIONS;
import static org.apache.paimon.types.BlobType.fieldNamesInBlobFile;
import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -79,7 +78,7 @@ public class DataEvolutionBlobCompactTask extends DataEvolutionCompactTask {
RowType blobWriteType = new RowType(Collections.singletonList(blobField));
- FileStoreTable readTable = table.copy(REWRITE_OPTIONS);
+ FileStoreTable readTable = table.copy(BLOB_COMPACT_READ_OPTIONS);
AppendOnlyFileStore store = (AppendOnlyFileStore) readTable.store();
DataFilePathFactory pathFactory =
store.pathFactory().createDataFilePathFactory(partition, 0);
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCommitChangesRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCommitChangesRewriter.java
deleted file mode 100644
index 3adcc63ba5..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCommitChangesRewriter.java
+++ /dev/null
@@ -1,125 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.manifest.IndexManifestFile;
-import org.apache.paimon.operation.commit.CommitChanges;
-import org.apache.paimon.operation.commit.CommitChangesProvider;
-import org.apache.paimon.operation.commit.CommitChangesRewriter;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.utils.Pair;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.stream.Collectors;
-
-import static org.apache.paimon.utils.DataEvolutionUtils.materializedRowIdRanges;
-
-/** Maintains index changes for operations which replace data-evolution row IDs. */
-public abstract class DataEvolutionCommitChangesRewriter implements CommitChangesRewriter {
-
- private final FileStoreTable table;
- private final IndexManifestFile indexManifestFile;
-
- protected DataEvolutionCommitChangesRewriter(FileStoreTable table) {
- this.table = table;
- this.indexManifestFile = table.store().indexManifestFileFactory().create();
- }
-
- @Override
- public CommitChangesProvider compactChangesProvider(
- CommitChanges changes, List<CommitMessage> messages) {
- Set<Pair<BinaryRow, Integer>> materializedBuckets = new HashSet<>();
- for (CommitMessage message : messages) {
- CommitMessageImpl commitMessage = (CommitMessageImpl) message;
- if (!materializedRowIdRanges(
- commitMessage.compactIncrement().compactBefore(),
- commitMessage.compactIncrement().compactAfter())
- .isEmpty()) {
- materializedBuckets.add(Pair.of(commitMessage.partition(), commitMessage.bucket()));
- }
- }
- if (materializedBuckets.isEmpty()) {
- return CommitChangesRewriter.super.compactChangesProvider(changes, messages);
- }
-
- return latestSnapshot -> {
- List<IndexManifestEntry> indexFiles =
- changes.indexFiles.stream()
- // Replace global-index deletions prepared against an older snapshot.
- .filter(
- entry ->
- entry.kind() != FileKind.DELETE
- || entry.indexFile().globalIndexMeta() == null
- || !materializedBuckets.contains(
- Pair.of(
- entry.partition(),
- entry.bucket())))
- .collect(Collectors.toList());
-
- // Each optimistic retry must include concurrently created indexes before assigning
- // final row IDs to this operation's outputs.
- if (latestSnapshot != null && latestSnapshot.indexManifest() != null) {
- for (IndexManifestEntry entry :
- indexManifestFile.read(latestSnapshot.indexManifest())) {
- if (entry.indexFile().globalIndexMeta() != null
- && materializedBuckets.contains(
- Pair.of(entry.partition(), entry.bucket()))) {
- indexFiles.add(entry.toDeleteEntry());
- }
- }
- }
-
- return new CommitChanges(changes.tableFiles, changes.changelogFiles, indexFiles);
- };
- }
-
- protected Attempt indexRewriteAttempt(CommitChanges changes, Set<String> mappedOutputs) {
- // Capture this before assignment makes newly allocated outputs indistinguishable from
- // ordinary compaction outputs which preserve their existing row IDs.
- Set<String> rowIdPreservingOutputs =
- changes.tableFiles.stream()
- .filter(entry -> entry.kind() == FileKind.ADD)
- .filter(entry -> entry.file().firstRowId() != null)
- .map(entry -> entry.file().fileName())
- .collect(Collectors.toSet());
- return (newSnapshotId, assignedFiles, indexFiles, ioManager) -> {
- if (assignedFiles.stream().noneMatch(entry -> entry.kind() == FileKind.DELETE)) {
- return indexFiles;
- }
- return DataEvolutionIndexRewriter.rewrite(
- table.fileIO(),
- table.store().pathFactory(),
- table.schemaManager(),
- table.coreOptions(),
- ioManager,
- assignedFiles,
- indexFiles,
- mappedOutputs,
- rowIdPreservingOutputs,
- newSnapshotId);
- };
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java
index 54e63932e4..6f24a41bcc 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTask.java
@@ -36,7 +36,6 @@ import java.util.Map;
import java.util.Objects;
import static java.util.Comparator.comparingLong;
-import static org.apache.paimon.compact.CompactOptions.REWRITE_OPTIONS;
import static org.apache.paimon.utils.DataEvolutionUtils.checkContiguousRowRange;
import static org.apache.paimon.utils.Preconditions.checkArgument;
@@ -44,6 +43,8 @@ import static org.apache.paimon.utils.Preconditions.checkArgument;
public abstract class DataEvolutionCompactTask {
protected static final Map<String, String> DYNAMIC_WRITE_OPTIONS = dynamicWriteOptions();
+ protected static final Map<String, String> BLOB_COMPACT_READ_OPTIONS =
+ Collections.singletonMap(CoreOptions.BLOB_AS_DESCRIPTOR.key(), "true");
protected final BinaryRow partition;
protected final List<DataFileMeta> compactBefore;
@@ -57,7 +58,7 @@ public abstract class DataEvolutionCompactTask {
}
private static Map<String, String> dynamicWriteOptions() {
- Map<String, String> options = new HashMap<>(REWRITE_OPTIONS);
+ Map<String, String> options = new HashMap<>();
options.put(CoreOptions.TARGET_FILE_SIZE.key(), "99999 G");
options.put(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "99999 G");
// Compaction is size-based; the write-time row limit must not roll its output.
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java
index cd6129a170..5b4e463dcb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializer.java
@@ -43,7 +43,7 @@ import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
public class DataEvolutionCompactTaskSerializer
implements VersionedSerializer<DataEvolutionCompactTask> {
- private static final int CURRENT_VERSION = 5;
+ private static final int CURRENT_VERSION = 4;
private final DataFileMetaSerializer dataFileSerializer;
@@ -83,7 +83,6 @@ public class DataEvolutionCompactTaskSerializer
view.writeLong(range.from);
view.writeLong(range.to);
}
- view.writeInt(((DataEvolutionNormalCompactTask) task).outputLevel());
} else if (task.type() == DataEvolutionCompactTask.TaskType.MATERIALIZE_DELETION) {
DeletionFile.serializeList(
view, ((DataEvolutionMaterializeDeletionCompactTask) task).deletionFiles());
@@ -109,7 +108,7 @@ public class DataEvolutionCompactTaskSerializer
}
private void checkVersion(int version) {
- if (version != 4 && version != CURRENT_VERSION) {
+ if (version != CURRENT_VERSION) {
throw new UnsupportedOperationException(
"Expecting DataEvolutionCompactTask version to be "
+ CURRENT_VERSION
@@ -133,8 +132,7 @@ public class DataEvolutionCompactTaskSerializer
for (int i = 0; i < rangeCount; i++) {
ranges.add(new Range(view.readLong(), view.readLong()));
}
- int outputLevel = version >= 5 ? view.readInt() : 0;
- return new DataEvolutionNormalCompactTask(partition, files, ranges, outputLevel);
+ return new DataEvolutionNormalCompactTask(partition, files, ranges);
case BLOB:
return new DataEvolutionBlobCompactTask(partition, files);
case MATERIALIZE_DELETION:
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionIndexRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionIndexRewriter.java
deleted file mode 100644
index d9ec534abc..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionIndexRewriter.java
+++ /dev/null
@@ -1,474 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.fs.FileIO;
-import org.apache.paimon.globalindex.DataEvolutionGlobalIndexRefreshPlanner;
-import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
-import org.apache.paimon.globalindex.GlobalIndexFileReadWrite;
-import org.apache.paimon.globalindex.GlobalIndexIOMeta;
-import org.apache.paimon.globalindex.GlobalIndexer;
-import org.apache.paimon.globalindex.ResultEntry;
-import org.apache.paimon.index.DataEvolutionIndexSourceMeta;
-import org.apache.paimon.index.GlobalIndexMeta;
-import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.index.IndexPathFactory;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataFilePathFactory;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.manifest.ManifestEntry;
-import org.apache.paimon.options.Options;
-import org.apache.paimon.schema.SchemaManager;
-import org.apache.paimon.types.DataField;
-import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.FileStorePathFactory;
-import org.apache.paimon.utils.Pair;
-import org.apache.paimon.utils.Range;
-import org.apache.paimon.utils.RowRangeIndex;
-
-import javax.annotation.Nullable;
-
-import java.io.Closeable;
-import java.io.File;
-import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.io.UncheckedIOException;
-import java.nio.file.Files;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.Set;
-import java.util.TreeMap;
-import java.util.stream.Collectors;
-
-import static org.apache.paimon.utils.DataEvolutionUtils.materializedRowIdRanges;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-
-/** Maintains scalar index coverage when data evolution replaces row IDs. */
-public final class DataEvolutionIndexRewriter {
-
- // The current metadata protocol represents coverage with contiguous ranges. Bound both
- // tiny-file fan-out and repeated source scans until sparse coverage has its own protocol.
- private static final int MAX_COVERAGE_FRAGMENTS = 64;
-
- private DataEvolutionIndexRewriter() {}
-
- public static List<IndexManifestEntry> rewrite(
- FileIO fileIO,
- FileStorePathFactory pathFactory,
- SchemaManager schemaManager,
- CoreOptions options,
- @Nullable IOManager ioManager,
- List<ManifestEntry> dataFiles,
- List<IndexManifestEntry> indexFiles,
- Set<String> mappedClusteringOutputs,
- Set<String> rowIdPreservingOutputs,
- long newSnapshotId)
- throws IOException {
- if (indexFiles.stream()
- .noneMatch(
- entry ->
- entry.kind() == FileKind.DELETE
- && entry.indexFile().globalIndexMeta() != null)) {
- return indexFiles;
- }
- Map<Pair<BinaryRow, Integer>, List<ManifestEntry>> groups =
- dataFiles.stream()
- .collect(Collectors.groupingBy(e -> Pair.of(e.partition(), e.bucket())));
- Map<Pair<BinaryRow, Integer>, List<IndexManifestEntry>> deletes = new HashMap<>();
- List<IndexManifestEntry> result = new ArrayList<>();
- for (IndexManifestEntry entry : indexFiles) {
- if (entry.kind() == FileKind.DELETE && entry.indexFile().globalIndexMeta() != null) {
- deletes.computeIfAbsent(
- Pair.of(entry.partition(), entry.bucket()), k -> new ArrayList<>())
- .add(entry);
- } else {
- result.add(entry);
- }
- }
- for (Map.Entry<Pair<BinaryRow, Integer>, List<IndexManifestEntry>> group :
- deletes.entrySet()) {
- List<ManifestEntry> files =
- groups.getOrDefault(group.getKey(), Collections.emptyList());
- List<DataFileMeta> after =
- files.stream()
- .filter(e -> e.kind() == FileKind.ADD)
- .map(ManifestEntry::file)
- .collect(Collectors.toList());
- List<ManifestEntry> before =
- files.stream()
- .filter(e -> e.kind() == FileKind.DELETE)
- .map(
- e ->
- ManifestEntry.create(
- FileKind.ADD,
- e.partition(),
- e.bucket(),
- e.totalBuckets(),
- e.file()))
- .collect(Collectors.toList());
- if (before.isEmpty()) {
- result.addAll(group.getValue());
- continue;
- }
- List<Range> removed =
- materializedRowIdRanges(
- before.stream().map(ManifestEntry::file).collect(Collectors.toList()),
- after.stream()
- .filter(
- file ->
- rowIdPreservingOutputs.contains(
- file.fileName()))
- .collect(Collectors.toList()));
- if (removed.isEmpty()) {
- // Preserve explicitly requested index deletions for ordinary compaction callers.
- result.addAll(group.getValue());
- continue;
- }
- after.removeIf(file -> rowIdPreservingOutputs.contains(file.fileName()));
- // Engines without a permutation still invalidate affected indexes. Independently
- // compacted ranges in the same partition retain their original index files.
- if (after.stream()
- .anyMatch(file -> !mappedClusteringOutputs.contains(file.fileName()))) {
- group.getValue().stream()
- .filter(
- entry ->
- !Range.and(
- Collections.singletonList(
- entry.indexFile()
- .globalIndexMeta()
- .rowRange()),
- removed)
- .isEmpty())
- .forEach(result::add);
- continue;
- }
- after.sort(Comparator.comparingLong(DataFileMeta::nonNullFirstRowId));
- long firstAssignedRowId =
- after.isEmpty() ? Long.MAX_VALUE : after.get(0).nonNullFirstRowId();
- DataFilePathFactory dataPath =
- pathFactory.createDataFilePathFactory(
- group.getKey().getLeft(), group.getKey().getRight());
- try (DiskMapping mapping = new DiskMapping(removed, ioManager)) {
- for (DataFileMeta file : after) {
- readMapping(
- fileIO,
- dataPath,
- file,
- (ordinal, oldId) ->
- mapping.put(oldId, file.nonNullFirstRowId() + ordinal));
- }
- RowType rowType =
- schemaManager
- .latestOrThrow("Missing clustering table schema")
- .logicalRowType();
- Map<Pair<String, List<Integer>>, List<IndexManifestEntry>> validSources =
- new LinkedHashMap<>();
- for (IndexManifestEntry entry : group.getValue()) {
- GlobalIndexMeta meta = entry.indexFile().globalIndexMeta();
- if (Range.and(Collections.singletonList(meta.rowRange()), removed).isEmpty()) {
- continue;
- }
- result.add(entry);
- String type = entry.indexFile().indexType();
- if (!type.equals("btree") && !type.equals("bitmap")) {
- continue;
- }
- // Dropped indexed columns can leave old index files in the manifest.
- if (!meta.getIndexedFieldIds().stream().allMatch(rowType::containsField)) {
- continue;
- }
- List<DataField> fields = meta.getIndexedFields(rowType);
- IndexManifestEntry source =
- new IndexManifestEntry(
- FileKind.ADD,
- entry.partition(),
- entry.bucket(),
- entry.indexFile());
- // Updating row IDs cannot repair keys from an older indexed-column value.
- if (!DataEvolutionIndexSourceMeta.isDataEvolutionMeta(meta.sourceMeta())
- || !DataEvolutionGlobalIndexRefreshPlanner.findIndexesToRefresh(
- schemaManager,
- before,
- Collections.singletonList(source),
- fields)
- .isEmpty()) {
- continue;
- }
- validSources
- .computeIfAbsent(
- Pair.of(type, meta.getIndexedFieldIds()),
- ignored -> new ArrayList<>())
- .add(entry);
- }
- for (Map.Entry<Pair<String, List<Integer>>, List<IndexManifestEntry>> sourceGroup :
- validSources.entrySet()) {
- List<IndexManifestEntry> sources = sourceGroup.getValue();
- String type = sourceGroup.getKey().getLeft();
- GlobalIndexMeta firstMeta = sources.get(0).indexFile().globalIndexMeta();
- List<DataField> fields = firstMeta.getIndexedFields(rowType);
- RowRangeIndex coverage =
- RowRangeIndex.create(
- sources.stream()
- .map(
- source ->
- source.indexFile()
- .globalIndexMeta()
- .rowRange())
- .collect(Collectors.toList()),
- false);
- String sourceName = sources.get(0).indexFile().fileName();
- List<Range> freshFragments = new ArrayList<>();
- // Source shards can interleave after sorting. Their union, rather than each
- // original shard, defines the coverage shared by the rewritten payload files.
- for (DataFileMeta file : after) {
- final long[] start = {-1};
- final long[] end = {-1};
- readMapping(
- fileIO,
- dataPath,
- file,
- (ordinal, oldId) -> {
- long newId = file.nonNullFirstRowId() + ordinal;
- if (coverage.intersects(oldId, oldId)) {
- if (start[0] < 0) {
- start[0] = newId;
- }
- end[0] = newId;
- } else if (start[0] >= 0) {
- addFragment(
- freshFragments,
- new Range(start[0], end[0]),
- sourceName,
- firstAssignedRowId);
- start[0] = -1;
- }
- });
- if (start[0] >= 0) {
- addFragment(
- freshFragments,
- new Range(start[0], end[0]),
- sourceName,
- firstAssignedRowId);
- }
- }
- GlobalIndexer indexer =
- GlobalIndexer.create(type, fields, Options.fromMap(options.toMap()));
- IndexPathFactory indexPath = pathFactory.globalIndexFileFactory();
- GlobalIndexFileReadWrite indexIO =
- new GlobalIndexFileReadWrite(fileIO, indexPath);
- for (IndexManifestEntry source : sources) {
- GlobalIndexMeta meta = source.indexFile().globalIndexMeta();
- List<Range> fragments = new ArrayList<>();
- for (Range untouched : meta.rowRange().exclude(removed)) {
- addFragment(
- fragments,
- untouched,
- source.indexFile().fileName(),
- firstAssignedRowId);
- }
- for (Range fresh : freshFragments) {
- addFragment(
- fragments,
- fresh,
- source.indexFile().fileName(),
- firstAssignedRowId);
- }
- GlobalIndexIOMeta input =
- new GlobalIndexIOMeta(
- indexPath.toPath(source.indexFile()),
- source.indexFile().fileSize(),
- source.indexFile().rowCount(),
- meta.indexMeta());
- for (Range fragment : fragments) {
- List<ResultEntry> rewritten =
- indexer.rewriteRowIds(
- indexIO,
- Collections.singletonList(input),
- meta.rowRange().count(),
- fragment.count(),
- relative -> {
- try {
- long target =
- mapping.get(
- relative
- + meta.rowRangeStart());
- return target >= fragment.from
- && target <= fragment.to
- ? target - fragment.from
- : -1;
- } catch (IOException e) {
- throw new UncheckedIOException(e);
- }
- },
- indexIO);
- // Only newly assigned coverage shares a fresh baseline. Old fragments
- // keep their source's baseline, even when another shard is older.
- byte[] sourceMeta =
- fragment.from >= firstAssignedRowId
- ? new DataEvolutionIndexSourceMeta(newSnapshotId)
- .serialize()
- : meta.sourceMeta();
- for (IndexFileMeta rewrittenFile :
- GlobalIndexBuilderUtils.toIndexFileMetas(
- fileIO,
- indexPath,
- options,
- fragment,
- fields,
- type,
- rewritten,
- sourceMeta)) {
- result.add(
- new IndexManifestEntry(
- FileKind.ADD,
- source.partition(),
- source.bucket(),
- rewrittenFile));
- }
- }
- }
- }
- }
- }
- return result;
- }
-
- private static void addFragment(
- List<Range> fragments, Range fragment, String fileName, long firstAssignedRowId) {
- if (!fragments.isEmpty()) {
- Range previous = fragments.get(fragments.size() - 1);
- if (previous.to == fragment.from - 1
- && (previous.from >= firstAssignedRowId)
- == (fragment.from >= firstAssignedRowId)) {
- fragments.set(fragments.size() - 1, new Range(previous.from, fragment.to));
- return;
- }
- }
- checkArgument(
- fragments.size() < MAX_COVERAGE_FRAGMENTS,
- "Clustering exceeds %s index coverage fragments for %s. "
- + "Build smaller, fully covered index shards before clustering this range.",
- MAX_COVERAGE_FRAGMENTS,
- fileName);
- fragments.add(fragment);
- }
-
- private static void readMapping(
- FileIO fileIO,
- DataFilePathFactory pathFactory,
- DataFileMeta file,
- ClusteringRowIdMapping.RowIdConsumer consumer)
- throws IOException {
- ClusteringRowIdMapping.read(
- fileIO,
- pathFactory.toAlignedPath(ClusteringRowIdMapping.fileName(file.fileName()), file),
- file.rowCount(),
- consumer);
- }
-
- /**
- * Disk-backed inverse permutation; memory is proportional to selected ranges, not row count.
- */
- private static final class DiskMapping implements Closeable {
- private final NavigableMap<Long, Pair<Range, Long>> offsets = new TreeMap<>();
- private final File path;
- private final RandomAccessFile file;
-
- private DiskMapping(List<Range> ranges, @Nullable IOManager ioManager) throws IOException {
- long rows = 0;
- for (Range range : ranges) {
- offsets.put(range.from, Pair.of(range, rows));
- rows = Math.addExact(rows, range.count());
- }
- long length = Math.multiplyExact(rows, Long.BYTES);
- path =
- ioManager == null
- ? File.createTempFile("paimon-cluster-row-id-", ".mapping")
- : ioManager.createChannel().getPathFile();
- RandomAccessFile opened = null;
- try {
- opened = new RandomAccessFile(path, "rw");
- opened.setLength(length);
- file = opened;
- } catch (IOException | RuntimeException | Error failure) {
- if (opened != null) {
- try {
- opened.close();
- } catch (IOException closeFailure) {
- failure.addSuppressed(closeFailure);
- }
- }
- try {
- Files.deleteIfExists(path.toPath());
- } catch (IOException cleanupFailure) {
- failure.addSuppressed(cleanupFailure);
- }
- throw failure;
- }
- }
-
- @Nullable
- private Long offset(long oldId) {
- Map.Entry<Long, Pair<Range, Long>> entry = offsets.floorEntry(oldId);
- if (entry == null || oldId > entry.getValue().getLeft().to) {
- return null;
- }
- return Math.multiplyExact(
- entry.getValue().getRight() + oldId - entry.getKey(), Long.BYTES);
- }
-
- private void put(long oldId, long newId) throws IOException {
- Long offset = offset(oldId);
- checkArgument(
- offset != null, "Clustering output refers to unselected row ID %s", oldId);
- file.seek(offset);
- checkArgument(file.readLong() == 0, "Duplicate clustering row ID %s", oldId);
- file.seek(offset);
- file.writeLong(Math.addExact(newId, 1));
- }
-
- private long get(long oldId) throws IOException {
- Long offset = offset(oldId);
- if (offset == null) {
- return oldId;
- }
- file.seek(offset);
- return file.readLong() - 1;
- }
-
- @Override
- public void close() throws IOException {
- try {
- file.close();
- } finally {
- Files.deleteIfExists(path.toPath());
- }
- }
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java
index 2191e5026b..7065a37bc1 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTask.java
@@ -69,7 +69,6 @@ public class DataEvolutionNormalCompactTask extends DataEvolutionCompactTask {
private static final Logger LOG = LoggerFactory.getLogger(DataEvolutionNormalCompactTask.class);
private final List<Range> protectedRanges;
- private final int outputLevel;
public DataEvolutionNormalCompactTask(BinaryRow partition, List<DataFileMeta> files) {
this(partition, files, Collections.emptyList());
@@ -77,19 +76,8 @@ public class DataEvolutionNormalCompactTask extends DataEvolutionCompactTask {
public DataEvolutionNormalCompactTask(
BinaryRow partition, List<DataFileMeta> files, List<Range> protectedRanges) {
- this(partition, files, protectedRanges, 0);
- }
-
- /** Preserves a clustering level only when the caller has verified the complete input group. */
- public DataEvolutionNormalCompactTask(
- BinaryRow partition,
- List<DataFileMeta> files,
- List<Range> protectedRanges,
- int outputLevel) {
super(partition, files);
- checkArgument(outputLevel >= 0, "Output level must not be negative.");
checkContiguousRowRange(files);
- this.outputLevel = outputLevel;
this.protectedRanges =
Collections.unmodifiableList(Range.sortAndMergeOverlap(protectedRanges));
}
@@ -98,20 +86,15 @@ public class DataEvolutionNormalCompactTask extends DataEvolutionCompactTask {
return protectedRanges;
}
- public int outputLevel() {
- return outputLevel;
- }
-
@Override
public boolean equals(Object other) {
return super.equals(other)
- && protectedRanges.equals(((DataEvolutionNormalCompactTask) other).protectedRanges)
- && outputLevel == ((DataEvolutionNormalCompactTask) other).outputLevel;
+ && protectedRanges.equals(((DataEvolutionNormalCompactTask) other).protectedRanges);
}
@Override
public int hashCode() {
- return Objects.hash(super.hashCode(), protectedRanges, outputLevel);
+ return Objects.hash(super.hashCode(), protectedRanges);
}
@Override
@@ -195,8 +178,7 @@ public class DataEvolutionNormalCompactTask extends DataEvolutionCompactTask {
long maxSequenceNumber = maxSequenceId(compactBefore);
long nextRowId = firstRowId;
long[] writeColsSequences =
- (options.ignoreIndexColumnUpdate() || options.clusteringIncrementalEnabled())
- && !writeResult.isEmpty()
+ options.ignoreIndexColumnUpdate() && !writeResult.isEmpty()
? compactedWriteColsSequences(
table,
writeResult
@@ -210,9 +192,6 @@ public class DataEvolutionNormalCompactTask extends DataEvolutionCompactTask {
if (writeColsSequences != null) {
dataFileMeta = dataFileMeta.withWriteColsSequences(writeColsSequences);
}
- if (outputLevel > 0) {
- dataFileMeta = dataFileMeta.upgrade(outputLevel);
- }
compactAfter.add(dataFileMeta);
nextRowId += dataFileMeta.rowCount();
}
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/MaterializeDeletionVectorsCommitChangesRewriter.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/MaterializeDeletionVectorsCommitChangesRewriter.java
deleted file mode 100644
index 6f2d3502c5..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/MaterializeDeletionVectorsCommitChangesRewriter.java
+++ /dev/null
@@ -1,42 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.Snapshot.CommitKind;
-import org.apache.paimon.operation.commit.CommitChanges;
-import org.apache.paimon.table.FileStoreTable;
-
-import java.util.Collections;
-
-/** Maintains index coverage for deletion-vector materialization. */
-public final class MaterializeDeletionVectorsCommitChangesRewriter
- extends DataEvolutionCommitChangesRewriter {
-
- public MaterializeDeletionVectorsCommitChangesRewriter(FileStoreTable table) {
- super(table);
- }
-
- @Override
- public Attempt createAttempt(CommitKind kind, CommitChanges changes) {
- if (kind != CommitKind.COMPACT) {
- return Attempt.IDENTITY;
- }
- return indexRewriteAttempt(changes, Collections.emptySet());
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/compact/CompactOptions.java b/paimon-core/src/main/java/org/apache/paimon/compact/CompactOptions.java
deleted file mode 100644
index 8ece8e5759..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/compact/CompactOptions.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.compact;
-
-import org.apache.paimon.CoreOptions;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
-
-/** Runtime options for physical compaction and clustering rewrites. */
-public final class CompactOptions {
-
- public static final Map<String, String> REWRITE_OPTIONS = rewriteOptions();
-
- private CompactOptions() {}
-
- private static Map<String, String> rewriteOptions() {
- Map<String, String> options = new HashMap<>();
- // Preserve stored references, including views of upstream values that are currently null.
- options.put(CoreOptions.BLOB_AS_DESCRIPTOR.key(), "true");
- options.put(CoreOptions.BLOB_VIEW_RESOLVE_ENABLED.key(), "false");
- // Keep the layout produced by the rewrite instead of applying ingestion optimizations.
- options.put(CoreOptions.PARTITION_SINK_STRATEGY.key(), "none");
- options.put(CoreOptions.CLUSTERING_INCREMENTAL_OPTIMIZE_WRITE.key(), "false");
- return Collections.unmodifiableMap(options);
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java
index ca4f96527f..050a345a0e 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommit.java
@@ -23,7 +23,6 @@ import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.disk.IOManager;
import org.apache.paimon.fs.FileIO;
import org.apache.paimon.manifest.ManifestCommittable;
-import org.apache.paimon.operation.commit.CommitChangesRewriter;
import org.apache.paimon.operation.metrics.CommitMetrics;
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.table.sink.CommitMessage;
@@ -37,9 +36,6 @@ public interface FileStoreCommit extends AutoCloseable {
FileStoreCommit withIOManager(IOManager ioManager);
- /** Injects metadata preparation into the existing commit attempt lifecycle. */
- FileStoreCommit withCommitChangesRewriter(CommitChangesRewriter rewriter);
-
FileStoreCommit ignoreEmptyCommit(boolean ignoreEmptyCommit);
FileStoreCommit withPartitionExpire(PartitionExpire partitionExpire);
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
index 464f2ec3d6..ffe79cf4bb 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/FileStoreCommitImpl.java
@@ -45,7 +45,6 @@ import org.apache.paimon.manifest.PartitionEntry;
import org.apache.paimon.manifest.SimpleFileEntry;
import org.apache.paimon.operation.commit.CommitChanges;
import org.apache.paimon.operation.commit.CommitChangesProvider;
-import org.apache.paimon.operation.commit.CommitChangesRewriter;
import org.apache.paimon.operation.commit.CommitCleaner;
import org.apache.paimon.operation.commit.CommitResult;
import org.apache.paimon.operation.commit.CommitRollback;
@@ -110,6 +109,7 @@ import java.util.stream.Collectors;
import static java.util.Collections.emptyList;
import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
+import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
import static org.apache.paimon.manifest.ManifestEntry.nullableRecordCount;
import static org.apache.paimon.manifest.ManifestEntry.recordCountAdd;
import static org.apache.paimon.manifest.ManifestEntry.recordCountDelete;
@@ -117,6 +117,7 @@ import static org.apache.paimon.operation.commit.ManifestEntryChanges.changedPar
import static org.apache.paimon.operation.commit.RowTrackingCommitUtils.assignRowTracking;
import static org.apache.paimon.partition.PartitionPredicate.createBinaryPartitions;
import static org.apache.paimon.partition.PartitionPredicate.createPartitionPredicate;
+import static org.apache.paimon.types.VectorType.isVectorStoreFile;
import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkNotNull;
@@ -168,8 +169,6 @@ public class FileStoreCommitImpl implements FileStoreCommit {
private final ConflictDetection conflictDetection;
private final CommitCleaner commitCleaner;
- @Nullable private CommitChangesRewriter commitChangesRewriter;
-
private boolean ignoreEmptyCommit;
private CommitMetrics commitMetrics;
private boolean appendCommitCheckConflict = false;
@@ -253,12 +252,6 @@ public class FileStoreCommitImpl implements FileStoreCommit {
}
}
- @Override
- public FileStoreCommitImpl withCommitChangesRewriter(CommitChangesRewriter rewriter) {
- this.commitChangesRewriter = checkNotNull(rewriter);
- return this;
- }
-
@Override
public FileStoreCommit withIOManager(IOManager ioManager) {
this.ioManager = ioManager;
@@ -354,6 +347,7 @@ public class FileStoreCommitImpl implements FileStoreCommit {
List<CommitMessage> commitMessages = committable.fileCommittables();
configureRowIdCheckFromMessages(commitMessages);
ManifestEntryChanges changes = collectChanges(commitMessages);
+ Set<Pair<BinaryRow, Integer>> materializedBuckets = materializedBuckets(commitMessages);
try {
List<SimpleFileEntry> appendSimpleEntries =
SimpleFileEntry.from(changes.appendTableFiles);
@@ -411,7 +405,7 @@ public class FileStoreCommitImpl implements FileStoreCommit {
changes.compactTableFiles,
changes.compactChangelog,
changes.compactIndexFiles),
- commitMessages),
+ materializedBuckets),
committable.identifier(),
committable.watermark(),
committable.properties(),
@@ -600,7 +594,7 @@ public class FileStoreCommitImpl implements FileStoreCommit {
changes.compactTableFiles,
emptyList(),
changes.compactIndexFiles),
- Collections.emptyList()),
+ Collections.emptySet()),
committable.identifier(),
committable.watermark(),
committable.properties(),
@@ -841,18 +835,74 @@ public class FileStoreCommitImpl implements FileStoreCommit {
}
}
- private CommitChangesProvider compactChangesProvider(
- CommitChanges changes, List<CommitMessage> messages) {
- if (commitChangesRewriter != null) {
- return commitChangesRewriter.compactChangesProvider(changes, messages);
- }
- return ReassignCompactChangesProvider.create(
- fileIO,
- pathFactory,
- snapshotManager,
- options,
- changes,
- conflictDetection.shouldCheckRowIdFromSnapshot(CommitKind.COMPACT));
+ private Set<Pair<BinaryRow, Integer>> materializedBuckets(List<CommitMessage> commitMessages) {
+ if (!options.dataEvolutionEnabled() || !options.deletionVectorsEnabled()) {
+ return Collections.emptySet();
+ }
+
+ Set<Pair<BinaryRow, Integer>> result = new HashSet<>();
+ for (CommitMessage message : commitMessages) {
+ CommitMessageImpl commitMessage = (CommitMessageImpl) message;
+ if (commitMessage.compactIncrement().compactBefore().stream()
+ .noneMatch(
+ file ->
+ !isBlobFile(file.fileName())
+ && !isVectorStoreFile(file.fileName()))
+ || commitMessage.compactIncrement().compactAfter().stream()
+ .anyMatch(file -> file.firstRowId() != null)) {
+ continue;
+ }
+ result.add(Pair.of(commitMessage.partition(), commitMessage.bucket()));
+ }
+ return result;
+ }
+
+ @VisibleForTesting
+ CommitChangesProvider compactChangesProvider(
+ CommitChanges changes, Set<Pair<BinaryRow, Integer>> materializedBuckets) {
+ if (materializedBuckets.isEmpty()) {
+ Long lastSafeSnapshot = options.commitLastSafeSnapshot().orElse(null);
+ if (options.dataEvolutionEnabled()
+ && !conflictDetection.shouldCheckRowIdFromSnapshot(CommitKind.COMPACT)
+ && lastSafeSnapshot != null
+ && lastSafeSnapshot >= 0) {
+ return new ReassignCompactChangesProvider(
+ fileIO, pathFactory, snapshotManager, lastSafeSnapshot, changes);
+ }
+ return CommitChangesProvider.provider(
+ changes.tableFiles, changes.changelogFiles, changes.indexFiles);
+ }
+
+ return latestSnapshot -> {
+ List<IndexManifestEntry> indexFiles =
+ changes.indexFiles.stream()
+ // Replace global-index deletions prepared against an older snapshot.
+ .filter(
+ entry ->
+ entry.kind() != FileKind.DELETE
+ || entry.indexFile().globalIndexMeta() == null
+ || !materializedBuckets.contains(
+ Pair.of(
+ entry.partition(),
+ entry.bucket())))
+ .collect(Collectors.toList());
+
+ // This provider is invoked again after every optimistic-commit conflict. Scanning the
+ // latest snapshot here guarantees that an index committed concurrently is either
+ // deleted by this attempt or makes this attempt retry and is deleted by the next one.
+ if (latestSnapshot != null && latestSnapshot.indexManifest() != null) {
+ for (IndexManifestEntry entry :
+ indexManifestFile.read(latestSnapshot.indexManifest())) {
+ if (entry.indexFile().globalIndexMeta() != null
+ && materializedBuckets.contains(
+ Pair.of(entry.partition(), entry.bucket()))) {
+ indexFiles.add(entry.toDeleteEntry());
+ }
+ }
+ }
+
+ return new CommitChanges(changes.tableFiles, changes.changelogFiles, indexFiles);
+ };
}
private int tryCommit(
@@ -1103,15 +1153,6 @@ public class FileStoreCommitImpl implements FileStoreCommit {
List<ManifestEntry> deltaFiles = changes.tableFiles;
List<ManifestEntry> changelogFiles = changes.changelogFiles;
List<IndexManifestEntry> indexFiles = changes.indexFiles;
- CommitChangesRewriter.Attempt rewriteAttempt =
- commitChangesRewriter == null
- ? CommitChangesRewriter.Attempt.IDENTITY
- : commitChangesRewriter.createAttempt(
- commitKind,
- new CommitChanges(
- Collections.unmodifiableList(changes.tableFiles),
- Collections.unmodifiableList(changes.changelogFiles),
- Collections.unmodifiableList(changes.indexFiles)));
Set<Long> rebasedReassignments = changesProvider.rebasedReassignments();
long newSnapshotId = Snapshot.FIRST_SNAPSHOT_ID;
@@ -1285,14 +1326,6 @@ public class FileStoreCommitImpl implements FileStoreCommit {
deltaFiles = assigned.assignedEntries;
}
- indexFiles =
- checkNotNull(
- rewriteAttempt.rewriteIndexFiles(
- newSnapshotId,
- Collections.unmodifiableList(deltaFiles),
- Collections.unmodifiableList(indexFiles),
- ioManager));
-
if (options.snapshotSequenceOrdering()) {
deltaFiles = stampSequenceWithSnapshotId(newSnapshotId, commitKind, deltaFiles);
}
@@ -1388,16 +1421,13 @@ public class FileStoreCommitImpl implements FileStoreCommit {
boolean success;
final List<SimpleFileEntry> finalBaseFiles = baseDataFiles;
final List<ManifestEntry> finalDeltaFiles = deltaFiles;
- final List<IndexManifestEntry> finalIndexFiles = indexFiles;
commitPreCallbacks.forEach(
callback ->
- callback.call(
- finalBaseFiles, finalDeltaFiles, finalIndexFiles, newSnapshot));
+ callback.call(finalBaseFiles, finalDeltaFiles, indexFiles, newSnapshot));
try {
success = commitSnapshotImpl(latestSnapshot, newSnapshot, deltaPartitionEntries);
} catch (Exception e) {
// commit exception, not sure about the situation and should not clean up the files
- // This also retains index payloads generated for this possibly published candidate.
LOG.warn(
"Retry commit for exception when committing snapshot #{} for table {} by user {}.",
newSnapshotId,
@@ -1408,8 +1438,6 @@ public class FileStoreCommitImpl implements FileStoreCommit {
}
if (!success) {
- // Some committers can publish the snapshot and still report false. Retain the index
- // payloads together with the manifests until the next attempt reconciles publication.
long commitTime = (System.currentTimeMillis() - startMillis) / 1000;
LOG.warn(
"Atomic commit failed for snapshot #{} for table {} by user {} "
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitChangesRewriter.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitChangesRewriter.java
deleted file mode 100644
index 00f31f2d74..0000000000
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/CommitChangesRewriter.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.operation.commit;
-
-import org.apache.paimon.Snapshot.CommitKind;
-import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.manifest.ManifestEntry;
-import org.apache.paimon.table.sink.CommitMessage;
-
-import javax.annotation.Nullable;
-
-import java.io.IOException;
-import java.util.List;
-
-/** Prepares commit metadata while reusing the committer's conflict and publication lifecycle. */
-public interface CommitChangesRewriter {
-
- CommitChangesRewriter IDENTITY = new CommitChangesRewriter() {};
-
- /** Creates one provider whose state is retained across optimistic retries of this compact. */
- default CommitChangesProvider compactChangesProvider(
- CommitChanges changes, List<CommitMessage> messages) {
- return CommitChangesProvider.provider(
- changes.tableFiles, changes.changelogFiles, changes.indexFiles);
- }
-
- /**
- * Creates isolated state for one attempt, after resolving its latest changes and before row-ID
- * assignment. The changes are read-only; this method must not create files.
- */
- default Attempt createAttempt(CommitKind kind, CommitChanges changes) {
- return Attempt.IDENTITY;
- }
-
- /** Index-only preparation after conflict checking and final row-ID assignment. */
- @FunctionalInterface
- interface Attempt {
-
- Attempt IDENTITY = (snapshotId, files, indexes, ioManager) -> indexes;
-
- /**
- * Returns index changes for this candidate snapshot. Inputs are read-only. Generated index
- * files are retained on failure; unreferenced files can be removed by orphan-file cleanup.
- */
- List<IndexManifestEntry> rewriteIndexFiles(
- long newSnapshotId,
- List<ManifestEntry> assignedFiles,
- List<IndexManifestEntry> indexFiles,
- @Nullable IOManager ioManager)
- throws IOException;
- }
-}
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java
index 344c9e3892..b3a7c82fae 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/DataEvolutionConflictDetection.java
@@ -63,7 +63,6 @@ import java.util.stream.Collectors;
import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
import static org.apache.paimon.operation.commit.ManifestEntryChanges.changedPartitions;
import static org.apache.paimon.types.VectorType.isVectorStoreFile;
-import static org.apache.paimon.utils.DataEvolutionUtils.materializedRowIdRanges;
import static org.apache.paimon.utils.Preconditions.checkState;
/** Conflict detection for Data Evolution tables. */
@@ -567,19 +566,19 @@ public class DataEvolutionConflictDetection extends ConflictDetection {
SchemaManager schemaManager,
List<ManifestEntry> deltaFiles,
boolean nestedFieldEnabled) {
- // Only materialized ranges require exclusive access. Row-ID-preserving compaction
- // keeps column sequences and can coexist with later partial-column writes.
- List<DataFileMeta> before =
+ // Materializing deletion vectors rewrites complete row ranges. A concurrent ADD in a
+ // deleted normal-file range can otherwise restore logically deleted rows.
+ List<DataFileMeta> deletedNormalFiles =
deltaFiles.stream()
.filter(entry -> entry.kind() == FileKind.DELETE)
.map(ManifestEntry::file)
+ .filter(file -> file.firstRowId() != null)
+ .filter(
+ file ->
+ !isBlobFile(file.fileName())
+ && !isVectorStoreFile(file.fileName()))
.collect(Collectors.toList());
- List<DataFileMeta> after =
- deltaFiles.stream()
- .filter(entry -> entry.kind() == FileKind.ADD)
- .map(ManifestEntry::file)
- .collect(Collectors.toList());
- return RowIdRangeConflictChecker.fromRanges(materializedRowIdRanges(before, after));
+ return RowIdRangeConflictChecker.fromDataFiles(deletedNormalFiles);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ReassignCompactChangesProvider.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ReassignCompactChangesProvider.java
index 0a66a6b488..88d3dcf906 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/ReassignCompactChangesProvider.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/ReassignCompactChangesProvider.java
@@ -18,7 +18,6 @@
package org.apache.paimon.operation.commit;
-import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
import org.apache.paimon.append.dataevolution.SerializationAssignment;
import org.apache.paimon.fs.FileIO;
@@ -68,26 +67,6 @@ public final class ReassignCompactChangesProvider implements CommitChangesProvid
this.changes = changes;
}
- /** Selects the ordinary compaction provider, allowing reuse across concurrent reassignment. */
- public static CommitChangesProvider create(
- FileIO fileIO,
- FileStorePathFactory paths,
- SnapshotManager snapshots,
- CoreOptions options,
- CommitChanges changes,
- boolean checkRowIdsFromSnapshot) {
- Long lastSafeSnapshot = options.commitLastSafeSnapshot().orElse(null);
- if (options.dataEvolutionEnabled()
- && !checkRowIdsFromSnapshot
- && lastSafeSnapshot != null
- && lastSafeSnapshot >= 0) {
- return new ReassignCompactChangesProvider(
- fileIO, paths, snapshots, lastSafeSnapshot, changes);
- }
- return CommitChangesProvider.provider(
- changes.tableFiles, changes.changelogFiles, changes.indexFiles);
- }
-
@Override
public CommitChanges provide(@Nullable Snapshot latest) {
checkState(
diff --git a/paimon-core/src/main/java/org/apache/paimon/operation/commit/RowIdRangeConflictChecker.java b/paimon-core/src/main/java/org/apache/paimon/operation/commit/RowIdRangeConflictChecker.java
index d0c65aaa5c..cf6bea0a51 100644
--- a/paimon-core/src/main/java/org/apache/paimon/operation/commit/RowIdRangeConflictChecker.java
+++ b/paimon-core/src/main/java/org/apache/paimon/operation/commit/RowIdRangeConflictChecker.java
@@ -19,7 +19,6 @@
package org.apache.paimon.operation.commit;
import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RowRangeIndex;
import java.util.List;
@@ -30,20 +29,17 @@ public class RowIdRangeConflictChecker implements RowIdConflictChecker {
private final RowRangeIndex rowRangeIndex;
- private RowIdRangeConflictChecker(List<Range> ranges) {
- this.rowRangeIndex = RowRangeIndex.create(ranges);
+ private RowIdRangeConflictChecker(List<DataFileMeta> deltaFiles) {
+ this.rowRangeIndex =
+ RowRangeIndex.create(
+ deltaFiles.stream()
+ .filter(file -> file.firstRowId() != null)
+ .map(DataFileMeta::nonNullRowIdRange)
+ .collect(Collectors.toList()));
}
public static RowIdRangeConflictChecker fromDataFiles(List<DataFileMeta> deltaFiles) {
- return fromRanges(
- deltaFiles.stream()
- .filter(file -> file.firstRowId() != null)
- .map(DataFileMeta::nonNullRowIdRange)
- .collect(Collectors.toList()));
- }
-
- public static RowIdRangeConflictChecker fromRanges(List<Range> ranges) {
- return new RowIdRangeConflictChecker(ranges);
+ return new RowIdRangeConflictChecker(deltaFiles);
}
@Override
diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
index 9aaca5f6d9..c98600f7ee 100644
--- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
+++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaValidation.java
@@ -1628,6 +1628,9 @@ public class SchemaValidation {
checkArgument(
rowTrackingEnabled,
"Data evolution config must enabled with row-tracking.enabled");
+ checkArgument(
+ !options.clusteringIncrementalEnabled(),
+ "Data evolution config must disabled with clustering.incremental");
}
if (options.dataEvolutionNestedFieldEnabled()) {
@@ -1922,16 +1925,6 @@ public class SchemaValidation {
schema.primaryKeys().isEmpty(),
"Cannot define %s for incremental clustering table.",
PRIMARY_KEY.key());
- if (options.dataEvolutionEnabled()) {
- checkArgument(
- fieldNamesInBlobFile(schema.logicalRowType(), options.blobInlineField())
- .isEmpty(),
- "Data evolution incremental clustering does not support dedicated BLOB files.");
- checkArgument(
- vectorFileFormat(options) == null
- || fieldsInVectorFile(schema.logicalRowType(), true).isEmpty(),
- "Data evolution incremental clustering does not support dedicated VECTOR files.");
- }
if (options.bucket() != -1) {
checkArgument(
!options.bucketAppendOrdered(),
diff --git a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
index ee5fcd2eb9..23f1132fd7 100644
--- a/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
+++ b/paimon-core/src/main/java/org/apache/paimon/table/sink/TableCommitImpl.java
@@ -31,7 +31,6 @@ import org.apache.paimon.manifest.ManifestCommittable;
import org.apache.paimon.metrics.MetricRegistry;
import org.apache.paimon.operation.FileStoreCommit;
import org.apache.paimon.operation.PartitionExpire;
-import org.apache.paimon.operation.commit.CommitChangesRewriter;
import org.apache.paimon.operation.metrics.CommitMetrics;
import org.apache.paimon.stats.Statistics;
import org.apache.paimon.tag.Tag;
@@ -182,11 +181,6 @@ public class TableCommitImpl implements InnerTableCommit {
return this;
}
- public TableCommitImpl withCommitChangesRewriter(CommitChangesRewriter rewriter) {
- commit.withCommitChangesRewriter(rewriter);
- return this;
- }
-
@Override
public TableCommitImpl withOperation(Snapshot.Operation operation) {
commit.withOperation(operation);
diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/DataEvolutionUtils.java b/paimon-core/src/main/java/org/apache/paimon/utils/DataEvolutionUtils.java
index d38bccc754..f8a2112eb6 100644
--- a/paimon-core/src/main/java/org/apache/paimon/utils/DataEvolutionUtils.java
+++ b/paimon-core/src/main/java/org/apache/paimon/utils/DataEvolutionUtils.java
@@ -331,72 +331,6 @@ public class DataEvolutionUtils {
return anchor;
}
- /** Returns normal-file row ranges removed without row-ID-preserving replacement files. */
- public static List<Range> materializedRowIdRanges(
- List<DataFileMeta> before, List<DataFileMeta> after) {
- List<Range> retained =
- Range.sortAndMergeOverlap(
- after.stream()
- .filter(file -> file.firstRowId() != null)
- .filter(
- file ->
- !isBlobFile(file.fileName())
- && !isVectorStoreFile(file.fileName()))
- .map(DataFileMeta::nonNullRowIdRange)
- .collect(Collectors.toList()));
- List<Range> removed =
- Range.sortAndMergeOverlap(
- before.stream()
- .filter(file -> file.firstRowId() != null)
- .filter(
- file ->
- !isBlobFile(file.fileName())
- && !isVectorStoreFile(file.fileName()))
- .map(DataFileMeta::nonNullRowIdRange)
- .collect(Collectors.toList()));
- List<Range> result = new ArrayList<>();
- int retainedIndex = 0;
- for (Range range : removed) {
- long cursor = range.from;
- boolean fullyRetained = false;
- while (retainedIndex < retained.size()) {
- Range output = retained.get(retainedIndex);
- if (output.to < cursor) {
- retainedIndex++;
- continue;
- }
- if (output.from > range.to) {
- break;
- }
- if (output.from > cursor) {
- addMaterializedRange(result, cursor, output.from - 1);
- }
- if (output.to >= range.to) {
- fullyRetained = true;
- break;
- }
- // The preceding comparison guarantees this cannot overflow at Long.MAX_VALUE.
- cursor = output.to + 1;
- retainedIndex++;
- }
- if (!fullyRetained) {
- addMaterializedRange(result, cursor, range.to);
- }
- }
- return result;
- }
-
- private static void addMaterializedRange(List<Range> ranges, long from, long to) {
- if (!ranges.isEmpty()) {
- Range previous = ranges.get(ranges.size() - 1);
- if (previous.to != Long.MAX_VALUE && previous.to + 1 == from) {
- ranges.set(ranges.size() - 1, new Range(previous.from, to));
- return;
- }
- }
- ranges.add(new Range(from, to));
- }
-
/** Check files row ranges. */
public static Range checkContiguousRowRange(List<DataFileMeta> files) {
checkArgument(!files.isEmpty(), "%s should not be empty.", "Data evolution compact files");
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/DataEvolutionClusterCompactionTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/DataEvolutionClusterCompactionTest.java
deleted file mode 100644
index 5129266344..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/DataEvolutionClusterCompactionTest.java
+++ /dev/null
@@ -1,277 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.cluster;
-
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
-import org.apache.paimon.append.dataevolution.DataEvolutionNormalCompactTask;
-import org.apache.paimon.compact.CompactUnit;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.manifest.FileSource;
-import org.apache.paimon.partition.PartitionPredicate;
-import org.apache.paimon.schema.FileSystemSchemaManager;
-import org.apache.paimon.schema.Schema;
-import org.apache.paimon.schema.SchemaChange;
-import org.apache.paimon.schema.SchemaManager;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.FileStoreTableFactory;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.table.source.snapshot.SnapshotReader;
-import org.apache.paimon.types.DataTypes;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-/** Compaction fallback for accumulated data-evolution column versions. */
-class DataEvolutionClusterCompactionTest {
-
- @TempDir java.nio.file.Path tempDir;
-
- @Test
- void testThresholdIncludesEveryVersionAndPreservesEachGroup() throws Exception {
- FileStoreTable table = createTable();
- List<DataFileMeta> files = versions(table, 0, 5);
- files.addAll(versions(table, 3, 4));
- List<DataEvolutionCompactTask> tasks = plan(table, files, Collections.emptyMap());
- assertThat(tasks).hasSize(1);
- assertThat(tasks.get(0).compactBefore())
- .containsExactlyInAnyOrderElementsOf(files.subList(0, 5));
- assertThat(((DataEvolutionNormalCompactTask) tasks.get(0)).outputLevel()).isEqualTo(2);
-
- files.add(file(table, "fifth-second-group", 3, 30, 0, "payload"));
- tasks = plan(table, files, Collections.emptyMap());
- assertThat(tasks).hasSize(2);
- assertThat(tasks)
- .allSatisfy(
- task -> {
- assertThat(task.compactBefore()).hasSize(5);
- assertThat(task.compactBefore())
- .extracting(DataFileMeta::nonNullRowIdRange)
- .containsOnly(task.compactBefore().get(0).nonNullRowIdRange());
- });
- }
-
- @Test
- void testSingleFileIsNotRewrittenEvenWithLowThreshold() throws Exception {
- FileStoreTable table =
- createTable()
- .copy(
- Collections.singletonMap(
- CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "1"));
- assertThat(plan(table, versions(table, 0, 1), Collections.emptyMap())).isEmpty();
- }
-
- @Test
- void testClusteringSelectedGroupIsNotCompactedAgain() throws Exception {
- FileStoreTable table = createTable();
- List<DataFileMeta> files = versions(table, 0, 5);
- List<DataFileMeta> secondGroup = versions(table, 3, 5);
- files.addAll(secondGroup);
- Map<BinaryRow, CompactUnit> clustering =
- Collections.singletonMap(
- BinaryRow.EMPTY_ROW, CompactUnit.fromFiles(2, files.subList(0, 5), true));
- List<DataEvolutionCompactTask> tasks = plan(table, files, clustering);
- assertThat(tasks).hasSize(1);
- assertThat(tasks.get(0).compactBefore()).containsExactlyInAnyOrderElementsOf(secondGroup);
- }
-
- @Test
- void testKeyUpdatesAndChangedDefinitionsDoNotInheritLevel() throws Exception {
- FileStoreTable table = createTable();
- List<DataFileMeta> keyUpdated = versions(table, 0, 5);
- keyUpdated.add(file(table, "key-update", 0, 30, 0, "id"));
- assertUnsortedCompaction(plan(table, keyUpdated, Collections.emptyMap()), keyUpdated);
-
- List<DataFileMeta> versions = versions(table, 0, 5);
- for (String[] definition : new String[][] {{"payload", "order"}, {"id", "zorder"}}) {
- FileStoreTable changed =
- table.copy(
- table.schemaManager()
- .commitChanges(
- Arrays.asList(
- SchemaChange.setOption(
- CoreOptions.CLUSTERING_COLUMNS.key(),
- definition[0]),
- SchemaChange.setOption(
- CoreOptions.CLUSTERING_STRATEGY.key(),
- definition[1]))));
- assertUnsortedCompaction(plan(changed, versions, Collections.emptyMap()), versions);
- }
- FileStoreTable newKey =
- table.copy(
- table.schemaManager()
- .commitChanges(
- Arrays.asList(
- SchemaChange.addColumn("new_key", DataTypes.INT()),
- SchemaChange.setOption(
- CoreOptions.CLUSTERING_COLUMNS.key(),
- "new_key"))));
- assertUnsortedCompaction(plan(newKey, versions, Collections.emptyMap()), versions);
- }
-
- @Test
- void testUnsortedOrIncompleteAnchorDoesNotInheritLevel() throws Exception {
- FileStoreTable table = createTable();
- List<DataFileMeta> unsorted = versions(table, 0, 5);
- unsorted.set(0, file(table, "unsorted-anchor", 0, 10, 0, null));
- assertUnsortedCompaction(plan(table, unsorted, Collections.emptyMap()), unsorted);
-
- List<DataFileMeta> incomplete = versions(table, 0, 5);
- incomplete.set(0, file(table, "partial-anchor", 0, 10, 2, "payload"));
- assertUnsortedCompaction(plan(table, incomplete, Collections.emptyMap()), incomplete);
- }
-
- @Test
- void testSingleDirtyRunSkippedByMinorStillCompactsColumnVersions() throws Exception {
- FileStoreTable table = createTable();
- List<DataFileMeta> files = versions(table, 0, 5);
- files.set(4, file(table, "key-update", 0, 30, 0, "id"));
- IncrementalClusterStrategy strategy =
- new IncrementalClusterStrategy(
- table.schemaManager(), table.coreOptions().clusteringColumns(), 25, 1, 2);
- assertThat(
- strategy.pick(
- 3,
- new DataEvolutionIncrementalClusterRewriter(table, null)
- .constructPartitionLevels(files),
- false))
- .isEmpty();
-
- assertUnsortedCompaction(plan(table, files, Collections.emptyMap()), files);
- }
-
- private static void assertUnsortedCompaction(
- List<DataEvolutionCompactTask> tasks, List<DataFileMeta> input) {
- assertThat(tasks).hasSize(1);
- assertThat(tasks.get(0).compactBefore()).containsExactlyInAnyOrderElementsOf(input);
- assertThat(((DataEvolutionNormalCompactTask) tasks.get(0)).outputLevel()).isZero();
- }
-
- private List<DataEvolutionCompactTask> plan(
- FileStoreTable table,
- List<DataFileMeta> files,
- Map<BinaryRow, CompactUnit> clustering) {
- FileStoreTable input = spy(table);
- Snapshot snapshot = mock(Snapshot.class);
- PartitionPredicate predicate = mock(PartitionPredicate.class);
- SnapshotReader reader = mock(SnapshotReader.class);
- SnapshotReader.Plan plan = mock(SnapshotReader.Plan.class);
- doReturn(reader).when(input).newSnapshotReader();
- when(reader.dropStats()).thenReturn(reader);
- when(reader.withSnapshot(snapshot)).thenReturn(reader);
- when(reader.withPartitionFilter(predicate)).thenReturn(reader);
- when(reader.read()).thenReturn(plan);
- // Scan splits may cut between physical column versions. Planning must regroup them.
- List<DataSplit> splits = new ArrayList<>();
- for (DataFileMeta file : files) {
- splits.add(
- DataSplit.builder()
- .withPartition(BinaryRow.EMPTY_ROW)
- .withBucket(0)
- .withBucketPath("bucket-0")
- .withDataFiles(Collections.singletonList(file))
- .build());
- }
- when(plan.dataSplits()).thenReturn(splits);
- List<DataEvolutionCompactTask> result =
- DataEvolutionClusterCompaction.plan(input, snapshot, predicate, clustering);
- verify(reader).withSnapshot(snapshot);
- verify(reader).withPartitionFilter(predicate);
- return result;
- }
-
- private FileStoreTable createTable() throws Exception {
- Schema schema =
- Schema.newBuilder()
- .column("id", DataTypes.INT())
- .column("payload", DataTypes.STRING())
- .option(CoreOptions.BUCKET.key(), "-1")
- .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
- .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
- .option(CoreOptions.CLUSTERING_INCREMENTAL.key(), "true")
- .option(CoreOptions.CLUSTERING_COLUMNS.key(), "id")
- .option(CoreOptions.NUM_LEVELS.key(), "3")
- .option(CoreOptions.COMPACTION_MIN_FILE_NUM.key(), "5")
- .build();
- LocalFileIO fileIO = LocalFileIO.create();
- Path path = new Path(tempDir.toString());
- SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, path);
- return FileStoreTableFactory.create(fileIO, path, schemaManager.createTable(schema));
- }
-
- private static List<DataFileMeta> versions(FileStoreTable table, long firstRowId, int count) {
- List<DataFileMeta> files = new ArrayList<>();
- files.add(file(table, "base-" + firstRowId, firstRowId, 10, 2, null));
- for (int i = 1; i < count; i++) {
- files.add(
- file(
- table,
- "update-" + firstRowId + "-" + i,
- firstRowId,
- 10 + i,
- 0,
- "payload"));
- }
- return files;
- }
-
- private static DataFileMeta file(
- FileStoreTable table,
- String name,
- long firstRowId,
- long sequence,
- int level,
- String writeColumn) {
- DataFileMeta file =
- DataFileMeta.forAppend(
- name + ".parquet",
- 100,
- 3,
- EMPTY_STATS,
- sequence,
- sequence,
- table.schema().id(),
- Collections.emptyList(),
- null,
- FileSource.APPEND,
- null,
- null,
- firstRowId,
- writeColumn == null ? null : Collections.singletonList(writeColumn));
- return level == 0 ? file : file.upgrade(level);
- }
-}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/DataEvolutionIncrementalClusterRewriterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/DataEvolutionIncrementalClusterRewriterTest.java
deleted file mode 100644
index 41e06cb993..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/DataEvolutionIncrementalClusterRewriterTest.java
+++ /dev/null
@@ -1,282 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.cluster;
-
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.compact.CompactUnit;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.manifest.FileSource;
-import org.apache.paimon.schema.FileSystemSchemaManager;
-import org.apache.paimon.schema.Schema;
-import org.apache.paimon.schema.SchemaChange;
-import org.apache.paimon.schema.SchemaManager;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.FileStoreTableFactory;
-import org.apache.paimon.types.DataTypes;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.ValueSource;
-
-import javax.annotation.Nullable;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Optional;
-
-import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS;
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Planner behavior for data-evolution column versions. */
-class DataEvolutionIncrementalClusterRewriterTest {
-
- private static final int NUM_LEVELS = 3;
- private static final int MAX_LEVEL = NUM_LEVELS - 1;
-
- @TempDir java.nio.file.Path tempDir;
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testNonKeyVersionsStayClusteredUntilKeyIsWritten(boolean fullSchemaUpdate)
- throws Exception {
- FileStoreTable original =
- createTable(
- Schema.newBuilder()
- .column("id", DataTypes.INT())
- .column("payload", DataTypes.STRING()),
- "id",
- "auto");
- FileStoreTable table =
- evolve(
- original,
- SchemaChange.setOption(CoreOptions.CLUSTERING_STRATEGY.key(), "order"));
- List<DataFileMeta> files = new ArrayList<>();
- for (long firstRowId : new long[] {0, 3}) {
- files.add(file(original, "base-" + firstRowId, firstRowId, 10, MAX_LEVEL, null));
- files.add(
- file(
- table,
- "payload-" + firstRowId,
- firstRowId,
- 20,
- 0,
- Collections.singletonList("payload"))
- .withWriteColsSequences(new long[] {20}));
- }
-
- // "auto" and "order" resolve to the same strategy for a single key. Non-key versions
- // neither create extra runs nor make a full cluster rewrite already sorted rows.
- assertThat(pick(table, files, true)).isEmpty();
- assertThat(pick(table, files, false)).isEmpty();
-
- files.add(
- file(
- table,
- "key-update",
- 0,
- 30,
- 0,
- fullSchemaUpdate ? null : Collections.singletonList("id")));
- assertSelectedFiles(pick(table, files, true), files);
- assertSelectedFiles(pick(table, files, false), files);
- }
-
- @Test
- void testAnchorMissingKeyRequiresClustering() throws Exception {
- FileStoreTable table =
- createTable(
- Schema.newBuilder()
- .column("id", DataTypes.INT())
- .column("payload", DataTypes.STRING()),
- "id",
- "order");
- List<DataFileMeta> files =
- Arrays.asList(
- file(
- table,
- "partial-anchor",
- 0,
- 10,
- MAX_LEVEL,
- Collections.singletonList("payload")),
- file(
- table,
- "later-key",
- 0,
- 20,
- MAX_LEVEL,
- Collections.singletonList("id")));
-
- // Neither file has a lower level, but the initial partial rows were not sorted by id.
- assertSelectedFiles(pick(table, files, true), files);
- }
-
- @Test
- void testOlderSchemaCompactFileDoesNotWriteNewKey() throws Exception {
- FileStoreTable original =
- createTable(
- Schema.newBuilder().column("payload", DataTypes.STRING()),
- "payload",
- "order");
- FileStoreTable table =
- evolve(
- original,
- SchemaChange.addColumn("id", DataTypes.INT()),
- SchemaChange.setOption(CoreOptions.CLUSTERING_COLUMNS.key(), "id"));
- DataFileMeta anchor = file(table, "new-key-anchor", 0, 10, MAX_LEVEL, null);
- DataFileMeta compactedPayload =
- file(original, "old-schema-compact", 0, 20, 0, null)
- .assignFileSource(FileSource.COMPACT)
- .withWriteColsSequences(new long[] {20});
-
- // Null write-cols means this file's physical schema, not the current schema. A compact
- // output from an older schema cannot write a key which was introduced later.
- assertThat(pick(table, Arrays.asList(compactedPayload, anchor), true)).isEmpty();
- }
-
- @ParameterizedTest
- @CsvSource({"payload, order", "id, zorder"})
- void testFirstCurrentGroupCannotHideDifferentClusteringDefinition(
- String oldKey, String oldStrategy) throws Exception {
- FileStoreTable original =
- createTable(
- Schema.newBuilder()
- .column("id", DataTypes.INT())
- .column("payload", DataTypes.STRING()),
- oldKey,
- oldStrategy);
- FileStoreTable table =
- evolve(
- original,
- SchemaChange.setOption(CoreOptions.CLUSTERING_COLUMNS.key(), "id"),
- SchemaChange.setOption(CoreOptions.CLUSTERING_STRATEGY.key(), "order"));
- List<DataFileMeta> files =
- Arrays.asList(
- file(table, "current-definition", 0, 10, MAX_LEVEL, null),
- file(original, "old-definition", 3, 10, MAX_LEVEL, null));
-
- // The shared full planner checks the first file's schema for a single highest-level run.
- // Each group's provenance must survive grouping even when a current-schema group is first.
- assertSelectedFiles(pick(table, files, true), files);
- }
-
- @Test
- void testPartitionOnlyKeyDoesNotRequirePhysicalColumn() throws Exception {
- FileStoreTable table =
- createTable(
- Schema.newBuilder()
- .column("pt", DataTypes.STRING())
- .column("payload", DataTypes.STRING())
- .partitionKeys("pt"),
- "pt",
- "order");
- List<DataFileMeta> files =
- Arrays.asList(
- file(
- table,
- "partition-anchor",
- 0,
- 10,
- MAX_LEVEL,
- Collections.singletonList("payload")),
- file(
- table,
- "payload-update",
- 0,
- 20,
- 0,
- Collections.singletonList("payload")));
-
- // A partition key is constant and need not be stored in either column version.
- assertThat(pick(table, files, true)).isEmpty();
- }
-
- private FileStoreTable createTable(
- Schema.Builder builder, String clusterColumns, String strategy) throws Exception {
- Schema schema =
- builder.option(CoreOptions.BUCKET.key(), "-1")
- .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
- .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
- .option(CoreOptions.CLUSTERING_INCREMENTAL.key(), "true")
- .option(CoreOptions.CLUSTERING_COLUMNS.key(), clusterColumns)
- .option(CoreOptions.CLUSTERING_STRATEGY.key(), strategy)
- .option(CoreOptions.NUM_LEVELS.key(), String.valueOf(NUM_LEVELS))
- .build();
- LocalFileIO fileIO = LocalFileIO.create();
- Path path = new Path(tempDir.toString());
- SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, path);
- return FileStoreTableFactory.create(fileIO, path, schemaManager.createTable(schema));
- }
-
- private static FileStoreTable evolve(FileStoreTable table, SchemaChange... changes)
- throws Exception {
- return table.copy(table.schemaManager().commitChanges(Arrays.asList(changes)));
- }
-
- private static DataFileMeta file(
- FileStoreTable table,
- String name,
- long firstRowId,
- long sequenceNumber,
- int level,
- @Nullable List<String> writeCols) {
- DataFileMeta file =
- DataFileMeta.forAppend(
- name + ".parquet",
- 100,
- 3,
- EMPTY_STATS,
- sequenceNumber,
- sequenceNumber,
- table.schema().id(),
- Collections.emptyList(),
- null,
- FileSource.APPEND,
- null,
- null,
- firstRowId,
- writeCols);
- return level == 0 ? file : file.upgrade(level);
- }
-
- private static Optional<CompactUnit> pick(
- FileStoreTable table, List<DataFileMeta> files, boolean full) {
- IncrementalClusterStrategy strategy =
- new IncrementalClusterStrategy(
- table.schemaManager(), table.coreOptions().clusteringColumns(), 25, 1, 2);
- return strategy.pick(
- NUM_LEVELS,
- new DataEvolutionIncrementalClusterRewriter(table, null)
- .constructPartitionLevels(files),
- full);
- }
-
- private static void assertSelectedFiles(
- Optional<CompactUnit> selected, List<DataFileMeta> expected) {
- assertThat(selected).isPresent();
- assertThat(selected.get().outputLevel()).isEqualTo(MAX_LEVEL);
- assertThat(selected.get().files()).containsExactlyInAnyOrderElementsOf(expected);
- }
-}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java
index 566d70e20d..2d64663f9a 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java
@@ -79,8 +79,7 @@ public class HistoryPartitionClusterTest {
table,
PartitionPredicate.fromMultiple(
RowType.of(DataTypes.INT()),
- Lists.newArrayList(BinaryRow.singleColumn("pt1"))),
- table.snapshotManager().latestSnapshot())
+ Lists.newArrayList(BinaryRow.singleColumn("pt1"))))
.historyPartitionCluster();
Map<BinaryRow, List<LevelSortedRun>> partitionLevels =
historyPartitionCluster.constructLevelsForHistoryPartitions();
@@ -93,17 +92,14 @@ public class HistoryPartitionClusterTest {
table,
PartitionPredicate.fromMultiple(
RowType.of(DataTypes.INT()),
- Lists.newArrayList(BinaryRow.singleColumn("pt3"))),
- table.snapshotManager().latestSnapshot())
+ Lists.newArrayList(BinaryRow.singleColumn("pt3"))))
.historyPartitionCluster();
partitionLevels = historyPartitionCluster.constructLevelsForHistoryPartitions();
assertThat(partitionLevels.size()).isEqualTo(1);
assertThat(partitionLevels.get(BinaryRow.singleColumn("pt1"))).isNotEmpty();
// test not specify partition and disable history partition auto clustering
- historyPartitionCluster =
- new IncrementalClusterManager(table, null, table.snapshotManager().latestSnapshot())
- .historyPartitionCluster();
+ historyPartitionCluster = new IncrementalClusterManager(table).historyPartitionCluster();
assertThat(historyPartitionCluster).isNull();
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
index 484a206afd..464c3c7703 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java
@@ -43,8 +43,6 @@ import org.apache.paimon.types.RowType;
import org.assertj.core.util.Lists;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
import java.util.ArrayList;
import java.util.Collections;
@@ -66,42 +64,19 @@ public class IncrementalClusterManagerTest {
options.put(CoreOptions.BUCKET.key(), "-1");
options.put(CoreOptions.CLUSTERING_INCREMENTAL.key(), "false");
FileStoreTable table = createTable(options, Collections.emptyList());
- assertThatThrownBy(
- () ->
- new IncrementalClusterManager(
- table, null, table.snapshotManager().latestSnapshot()))
+ assertThatThrownBy(() -> new IncrementalClusterManager(table))
.isInstanceOf(IllegalArgumentException.class)
.hasMessageContaining(
"Only support incremental clustering when 'clustering.incremental' is true.");
}
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- public void testEmptyPlanningSnapshotDoesNotReadConcurrentFirstAppend(boolean dataEvolution)
- throws Exception {
+ @Test
+ public void testConstructPartitionLevels() throws Exception {
+ // Create a valid table for IncrementalClusterManager
Map<String, String> options = new HashMap<>();
- options.put(CoreOptions.WRITE_ONLY.key(), "true");
- options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), String.valueOf(dataEvolution));
- options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), String.valueOf(dataEvolution));
FileStoreTable table = createTable(options, Collections.emptyList());
- assertThat(table.snapshotManager().latestSnapshot()).isNull();
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, null);
-
- writeOnce(
- table,
- GenericRow.of(1, 2, BinaryString.fromString("a"), BinaryString.fromString("b")));
-
- assertThat(manager.createCompactUnits(true)).isEmpty();
- assertThat(manager.createCompactUnits(false)).isEmpty();
- assertThat(
- new IncrementalClusterManager(
- table, null, table.snapshotManager().latestSnapshot())
- .createCompactUnits(true))
- .isNotEmpty();
- }
+ IncrementalClusterManager incrementalClusterManager = new IncrementalClusterManager(table);
- @Test
- public void testConstructPartitionLevels() {
// Create test files with different levels
List<DataFileMeta> partitionFiles = new ArrayList<>();
@@ -123,7 +98,7 @@ public class IncrementalClusterManagerTest {
// Call the method under test
List<LevelSortedRun> result =
- new IncrementalClusterRewriter().constructPartitionLevels(partitionFiles);
+ incrementalClusterManager.constructPartitionLevels(partitionFiles);
// Verify the results
assertThat(result).hasSize(4); // 2 level-0 runs + 1 level-1 run + 1 level-2 run
@@ -177,9 +152,7 @@ public class IncrementalClusterManagerTest {
// Test default mode is GLOBAL_SORT
Map<String, String> options = new HashMap<>();
FileStoreTable table = createTable(options, Collections.emptyList());
- IncrementalClusterManager manager =
- new IncrementalClusterManager(
- table, null, table.snapshotManager().latestSnapshot());
+ IncrementalClusterManager manager = new IncrementalClusterManager(table);
assertThat(manager.clusteringIncrementalMode())
.isEqualTo(CoreOptions.ClusteringIncrementalMode.GLOBAL_SORT);
}
@@ -190,9 +163,7 @@ public class IncrementalClusterManagerTest {
Map<String, String> options = new HashMap<>();
options.put(CoreOptions.CLUSTERING_INCREMENTAL_MODE.key(), "local-sort");
FileStoreTable table = createTable(options, Collections.emptyList());
- IncrementalClusterManager manager =
- new IncrementalClusterManager(
- table, null, table.snapshotManager().latestSnapshot());
+ IncrementalClusterManager manager = new IncrementalClusterManager(table);
assertThat(manager.clusteringIncrementalMode())
.isEqualTo(CoreOptions.ClusteringIncrementalMode.LOCAL_SORT);
}
@@ -203,9 +174,7 @@ public class IncrementalClusterManagerTest {
Map<String, String> options = new HashMap<>();
options.put(CoreOptions.CLUSTERING_INCREMENTAL_MODE.key(), "global-sort");
FileStoreTable table = createTable(options, Collections.emptyList());
- IncrementalClusterManager manager =
- new IncrementalClusterManager(
- table, null, table.snapshotManager().latestSnapshot());
+ IncrementalClusterManager manager = new IncrementalClusterManager(table);
assertThat(manager.clusteringIncrementalMode())
.isEqualTo(CoreOptions.ClusteringIncrementalMode.GLOBAL_SORT);
}
@@ -242,8 +211,7 @@ public class IncrementalClusterManagerTest {
table,
PartitionPredicate.fromMultiple(
RowType.of(DataTypes.INT()),
- Lists.newArrayList(BinaryRow.singleColumn("pt3"))),
- table.snapshotManager().latestSnapshot());
+ Lists.newArrayList(BinaryRow.singleColumn("pt3"))));
Map<BinaryRow, CompactUnit> partitionLevels =
incrementalClusterManager.createCompactUnits(true);
assertThat(partitionLevels.size()).isEqualTo(2);
@@ -251,26 +219,21 @@ public class IncrementalClusterManagerTest {
assertThat(partitionLevels.get(BinaryRow.singleColumn("pt3"))).isNotNull();
// test don't specify partition and enable history partition auto clustering
- incrementalClusterManager =
- new IncrementalClusterManager(
- table, null, table.snapshotManager().latestSnapshot());
+ incrementalClusterManager = new IncrementalClusterManager(table);
partitionLevels = incrementalClusterManager.createCompactUnits(true);
assertThat(partitionLevels.size()).isEqualTo(4);
// test specify partition and disable history partition auto clustering
SchemaChange schemaChange =
SchemaChange.removeOption(CoreOptions.CLUSTERING_HISTORY_PARTITION_IDLE_TIME.key());
- FileStoreTable tableWithoutHistoryClustering =
- table.copy(
- table.schemaManager()
- .commitChanges(Collections.singletonList(schemaChange)));
incrementalClusterManager =
new IncrementalClusterManager(
- tableWithoutHistoryClustering,
+ table.copy(
+ table.schemaManager()
+ .commitChanges(Collections.singletonList(schemaChange))),
PartitionPredicate.fromMultiple(
RowType.of(DataTypes.INT()),
- Lists.newArrayList(BinaryRow.singleColumn("pt3"))),
- tableWithoutHistoryClustering.snapshotManager().latestSnapshot());
+ Lists.newArrayList(BinaryRow.singleColumn("pt3"))));
partitionLevels = incrementalClusterManager.createCompactUnits(true);
assertThat(partitionLevels.size()).isEqualTo(1);
assertThat(partitionLevels.get(BinaryRow.singleColumn("pt3"))).isNotNull();
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/ClusteringRowIdMappingTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/ClusteringRowIdMappingTest.java
deleted file mode 100644
index 78ba1a7a36..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/ClusteringRowIdMappingTest.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.fs.FileIO;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.local.LocalFileIO;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-
-import java.io.DataOutputStream;
-import java.io.EOFException;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.List;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests the output-file ordinal mapping and rejects incomplete or corrupt side files. */
-class ClusteringRowIdMappingTest {
-
- private static final int MAGIC = 0x43524944;
- private final FileIO fileIO = LocalFileIO.create();
-
- @TempDir java.nio.file.Path tempPath;
-
- @Test
- void testRoundTripKeepsPhysicalOrder() throws Exception {
- Path path = path("round-trip");
- try (ClusteringRowIdMapping.Writer writer =
- ClusteringRowIdMapping.createWriter(fileIO, path)) {
- writer.write(10);
- writer.write(0);
- writer.write(Long.MAX_VALUE);
- }
- List<Long> ordinals = new ArrayList<>();
- List<Long> oldRowIds = new ArrayList<>();
- ClusteringRowIdMapping.read(
- fileIO,
- path,
- 3,
- (ordinal, oldRowId) -> {
- ordinals.add(ordinal);
- oldRowIds.add(oldRowId);
- });
- assertThat(ordinals).containsExactly(0L, 1L, 2L);
- assertThat(oldRowIds).containsExactly(10L, 0L, Long.MAX_VALUE);
- }
-
- @Test
- void testRejectsTruncatedAndTrailingRows() throws Exception {
- Path path = raw("row-count", MAGIC, 1, 10L);
- assertThatThrownBy(
- () -> ClusteringRowIdMapping.read(fileIO, path, 2, (ordinal, oldId) -> {}))
- .isInstanceOf(EOFException.class);
- assertThatThrownBy(
- () -> ClusteringRowIdMapping.read(fileIO, path, 0, (ordinal, oldId) -> {}))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Unexpected trailing clustering row IDs");
- }
-
- @Test
- void testRejectsInvalidHeaderAndOldRowIds() throws Exception {
- Path magic = raw("magic", 0, 1);
- assertThatThrownBy(
- () -> ClusteringRowIdMapping.read(fileIO, magic, 0, (ordinal, oldId) -> {}))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Invalid clustering row ID mapping");
- Path version = raw("version", MAGIC, 2);
- assertThatThrownBy(
- () ->
- ClusteringRowIdMapping.read(
- fileIO, version, 0, (ordinal, oldId) -> {}))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Unsupported clustering row ID mapping");
- Path negative = raw("negative", MAGIC, 1, -1);
- assertThatThrownBy(
- () ->
- ClusteringRowIdMapping.read(
- fileIO, negative, 1, (ordinal, oldId) -> {}))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Invalid old row ID");
- assertThatThrownBy(
- () ->
- ClusteringRowIdMapping.read(
- fileIO, negative, -1, (ordinal, oldId) -> {}))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Invalid clustering row count");
- }
-
- @Test
- void testWriterRejectsNegativeRowIds() throws Exception {
- Path path = path("invalid-write");
- try (ClusteringRowIdMapping.Writer writer =
- ClusteringRowIdMapping.createWriter(fileIO, path)) {
- assertThatThrownBy(() -> writer.write(-1))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessageContaining("Invalid old row ID");
- writer.write(3);
- }
- List<Long> rows = new ArrayList<>();
- ClusteringRowIdMapping.read(fileIO, path, 1, (ordinal, oldId) -> rows.add(oldId));
- assertThat(rows).containsExactly(3L);
- }
-
- private Path raw(String name, int magic, int version, long... rows) throws IOException {
- Path path = path(name);
- try (DataOutputStream out = new DataOutputStream(fileIO.newOutputStream(path, false))) {
- out.writeInt(magic);
- out.writeInt(version);
- for (long row : rows) {
- out.writeLong(row);
- }
- }
- return path;
- }
-
- private Path path(String name) {
- return new Path(tempPath.resolve(name).toUri());
- }
-}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionClusterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionClusterTest.java
deleted file mode 100644
index 9070a9a05e..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionClusterTest.java
+++ /dev/null
@@ -1,2343 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.cluster.IncrementalClusterManager;
-import org.apache.paimon.append.cluster.IncrementalClusterRewriter;
-import org.apache.paimon.append.cluster.Sorter;
-import org.apache.paimon.catalog.RenamingSnapshotCommit;
-import org.apache.paimon.catalog.SnapshotCommit;
-import org.apache.paimon.compact.CompactUnit;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.data.serializer.InternalRowSerializer;
-import org.apache.paimon.deletionvectors.BitmapDeletionVector;
-import org.apache.paimon.deletionvectors.DeletionVector;
-import org.apache.paimon.deletionvectors.append.BaseAppendDeleteFileMaintainer;
-import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner;
-import org.apache.paimon.globalindex.GlobalIndexBuilderUtils;
-import org.apache.paimon.globalindex.GlobalIndexResult;
-import org.apache.paimon.globalindex.sorted.SortedGlobalIndexScanner;
-import org.apache.paimon.globalindex.sorted.SortedGlobalIndexTestUtils;
-import org.apache.paimon.index.DataEvolutionIndexSourceMeta;
-import org.apache.paimon.index.GlobalIndexMeta;
-import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.io.CompactIncrement;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataFilePathFactory;
-import org.apache.paimon.io.DataIncrement;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.operation.Lock;
-import org.apache.paimon.operation.commit.RowIdExistenceConflictException;
-import org.apache.paimon.partition.PartitionPredicate;
-import org.apache.paimon.partition.PartitionStatistics;
-import org.apache.paimon.predicate.Predicate;
-import org.apache.paimon.predicate.PredicateBuilder;
-import org.apache.paimon.reader.RecordReader;
-import org.apache.paimon.reader.RecordReaderIterator;
-import org.apache.paimon.schema.Schema;
-import org.apache.paimon.table.CatalogEnvironment;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.FileStoreTableFactory;
-import org.apache.paimon.table.SpecialFields;
-import org.apache.paimon.table.TableTestBase;
-import org.apache.paimon.table.sink.BatchTableCommit;
-import org.apache.paimon.table.sink.BatchTableWrite;
-import org.apache.paimon.table.sink.BatchWriteBuilder;
-import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.table.sink.TableCommitImpl;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.table.source.ReadBuilder;
-import org.apache.paimon.types.DataTypes;
-import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.MutableObjectIterator;
-import org.apache.paimon.utils.Pair;
-import org.apache.paimon.utils.ProjectedRow;
-import org.apache.paimon.utils.Range;
-import org.apache.paimon.utils.SnapshotManager;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.ValueSource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Collectors;
-
-import static org.apache.paimon.errors.ErrorMessages.DATA_EVOLUTION_ROW_ID_CONFLICT_MESSAGE;
-import static org.apache.paimon.table.BucketMode.UNAWARE_BUCKET;
-import static org.apache.paimon.utils.DataEvolutionUtils.retrieveAnchorFile;
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Behavioral tests for sorting data-evolution rows into fresh row-ID ranges. */
-class DataEvolutionClusterTest extends TableTestBase {
-
- @Test
- void testClusterSortsLatestValuesAndAppliesDeletionVectors() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- append(table, "a", 60, 40, 50);
- updatePayload(table, "a", 0L, 30, 10, 20);
- delete(table, "a", new Range(0, 2), 2L);
-
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, true);
- assertThat(messages).isNotEmpty();
- commitCluster(table, snapshot, messages);
-
- assertThat(readRows(table))
- .containsExactly(
- "a|10|updated-10|6",
- "a|30|updated-30|7",
- "a|40|base-40|8",
- "a|50|base-50|9",
- "a|60|base-60|10");
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(11L);
- assertThat(
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() == null))
- .isEmpty();
- }
-
- @Test
- void testFullClusterSkipsAlreadyClusteredPartition() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareCluster(table, snapshot, null, true));
- List<String> clusteredRows = readRows(table);
- List<String> clusteredFiles = fileNames(table, "a");
-
- Snapshot clusteredSnapshot = table.latestSnapshot().get();
- assertThat(prepareCluster(table, clusteredSnapshot, null, true)).isEmpty();
- assertThat(readRows(table)).isEqualTo(clusteredRows);
- assertThat(fileNames(table, "a")).isEqualTo(clusteredFiles);
- }
-
- @Test
- void testClusterRemovesFullyDeletedGroup() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- delete(table, "a", new Range(0, 2), 0L, 1L, 2L);
-
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, true);
- assertThat(messages).isNotEmpty();
- commitCluster(table, snapshot, messages);
-
- assertThat(readRows(table)).isEmpty();
- assertThat(fileNames(table, "a")).isEmpty();
- assertThat(table.store().newIndexFileHandler().scanEntries()).isEmpty();
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(3L);
- }
-
- @Test
- void testClusterHonorsOutputFileRowLimit() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 70, 10, 50, 30, 60, 20, 40);
- table = table.copy(Collections.singletonMap(CoreOptions.TARGET_FILE_ROW_NUM.key(), "2"));
-
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareCluster(table, snapshot, null, true));
-
- List<DataFileMeta> files =
- table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .map(entry -> entry.file())
- .collect(Collectors.toList());
- assertThat(files).hasSize(4);
- assertThat(files).allSatisfy(file -> assertThat(file.rowCount()).isBetween(1L, 2L));
- assertThat(
- files.stream()
- .map(DataFileMeta::nonNullRowIdRange)
- .sorted(Comparator.comparingLong(range -> range.from)))
- .containsExactly(
- new Range(7, 8), new Range(9, 10), new Range(11, 12), new Range(13, 13));
- assertThat(readRows(table))
- .containsExactly(
- "a|10|base-10|7",
- "a|20|base-20|8",
- "a|30|base-30|9",
- "a|40|base-40|10",
- "a|50|base-50|11",
- "a|60|base-60|12",
- "a|70|base-70|13");
- }
-
- @Test
- void testPartialSortKeyUpdateDirtiesClusteredGroup() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareCluster(table, snapshot, null, true));
-
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- RowType writeType = table.rowType().project(Arrays.asList("pt", "id"));
- try (BatchTableWrite write = builder.newWrite().withWriteType(writeType);
- BatchTableCommit commit = builder.newCommit()) {
- for (int id : new int[] {50, 10, 30}) {
- write.write(GenericRow.of(BinaryString.fromString("a"), id));
- }
- List<CommitMessage> updates = write.prepareCommit();
- assignFirstRowId(updates, 3L);
- commit.commit(updates);
- }
-
- Snapshot updated = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, updated, null, true);
- assertThat(messages).isNotEmpty();
- commitCluster(table, updated, messages);
-
- assertThat(readRows(table))
- .containsExactly("a|10|base-20|6", "a|30|base-30|7", "a|50|base-10|8");
- }
-
- @Test
- void testNonClusteringUpdatesDoNotDirtyClusteredGroups() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 60, 10, 50, 30, 20, 40);
- table = table.copy(Collections.singletonMap(CoreOptions.TARGET_FILE_ROW_NUM.key(), "3"));
- Snapshot initial = table.latestSnapshot().get();
- commitCluster(table, initial, prepareMappedCluster(table, initial, true));
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- List<String> indexes = globalIndexNames(table);
- assertThat(indexes).isNotEmpty();
-
- updatePayload(table, "a", 6L, 10, 20, 30);
- updatePayload(table, "a", 9L, 40, 50, 60);
- Snapshot updated = table.latestSnapshot().get();
- List<String> files = fileNames(table, "a");
- assertThat(files).hasSize(4);
- List<String> latestRows =
- Arrays.asList(
- "a|10|updated-10|6",
- "a|20|updated-20|7",
- "a|30|updated-30|8",
- "a|40|updated-40|9",
- "a|50|updated-50|10",
- "a|60|updated-60|11");
- for (boolean fullCompaction : new boolean[] {false, true}) {
- List<CommitMessage> messages = prepareMappedCluster(table, updated, fullCompaction);
- assertThat(messages).isEmpty();
- commitCluster(table, updated, messages);
- assertThat(table.latestSnapshot().get().id()).isEqualTo(updated.id());
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(12L);
- assertThat(fileNames(table, "a")).isEqualTo(files);
- assertThat(globalIndexNames(table)).isEqualTo(indexes);
- assertThat(readRows(table)).containsExactlyElementsOf(latestRows);
- }
- assertIndexRows(
- table,
- new PredicateBuilder(table.rowType())
- .equal(table.rowType().getFieldIndex("id"), 20),
- new Range(7, 7));
-
- append(table, "a", 90, 70, 80);
- Snapshot appended = table.latestSnapshot().get();
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, appended);
- Map<BinaryRow, CompactUnit> units = manager.createCompactUnits(true);
- assertThat(units).containsOnlyKeys(partition(table, "a"));
- assertThat(units.get(partition(table, "a")).files())
- .hasSize(5)
- .extracting(DataFileMeta::fileName)
- .containsAll(files);
- commitCluster(table, appended, rewriteClusterUnits(table, manager, units, true));
-
- assertThat(readRows(table))
- .containsExactly(
- "a|10|updated-10|15",
- "a|20|updated-20|16",
- "a|30|updated-30|17",
- "a|40|updated-40|18",
- "a|50|updated-50|19",
- "a|60|updated-60|20",
- "a|70|base-70|21",
- "a|80|base-80|22",
- "a|90|base-90|23");
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(24L);
- assertIndexRows(
- table,
- new PredicateBuilder(table.rowType())
- .equal(table.rowType().getFieldIndex("id"), 20),
- new Range(16, 16));
- }
-
- @Test
- void testIncrementalClusterWaitsForCompactionThreshold() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- assertThat(prepareCluster(table, table.latestSnapshot().get(), null, false)).isEmpty();
-
- append(table, "a", 60, 40, 50);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, false);
- assertThat(messages).isNotEmpty();
- commitCluster(table, snapshot, messages);
-
- assertThat(readRows(table))
- .containsExactly(
- "a|10|base-10|6",
- "a|20|base-20|7",
- "a|30|base-30|8",
- "a|40|base-40|9",
- "a|50|base-50|10",
- "a|60|base-60|11");
- }
-
- @Test
- void testMinorClusterSelectsCompleteDirtyGroupAndRetainsUntouchedGroup() throws Exception {
- Map<String, String> options = new HashMap<>();
- options.put(CoreOptions.TARGET_FILE_ROW_NUM.key(), "3");
- options.put(CoreOptions.FILE_COMPRESSION.key(), "none");
- options.put(CoreOptions.COMPACTION_MAX_SIZE_AMPLIFICATION_PERCENT.key(), "10000");
- options.put(CoreOptions.COMPACTION_SIZE_RATIO.key(), "0");
- FileStoreTable table = createTable(false).copy(options);
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- try (BatchTableWrite write = builder.newWrite();
- BatchTableCommit commit = builder.newCommit()) {
- for (int id : new int[] {60, 10, 50, 30, 20, 40}) {
- Random random = new Random(id);
- int length = id <= 30 ? 8192 : 131072;
- StringBuilder payload = new StringBuilder(length);
- for (int i = 0; i < length; i++) {
- payload.append((char) ('a' + random.nextInt(26)));
- }
- write.write(
- GenericRow.of(
- BinaryString.fromString("a"),
- id,
- BinaryString.fromString(payload.toString())));
- }
- commit.commit(write.prepareCommit());
- }
- Snapshot initial = table.latestSnapshot().get();
- commitCluster(table, initial, prepareCluster(table, initial, null, true));
-
- List<DataFileMeta> clusteredFiles =
- table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .map(entry -> entry.file())
- .sorted(Comparator.comparingLong(DataFileMeta::nonNullFirstRowId))
- .collect(Collectors.toList());
- assertThat(clusteredFiles).hasSize(2);
- DataFileMeta dirtyBase = clusteredFiles.get(0);
- DataFileMeta untouchedBase = clusteredFiles.get(1);
- assertThat(dirtyBase.nonNullRowIdRange()).isEqualTo(new Range(6, 8));
- assertThat(untouchedBase.nonNullRowIdRange()).isEqualTo(new Range(9, 11));
- assertThat(untouchedBase.fileSize()).isGreaterThan(dirtyBase.fileSize() * 8);
- List<String> expected = new ArrayList<>(readRows(table).subList(3, 6));
-
- updatePayload(table, "a", 6L, 10, 20, 30);
- updateSortKeys(table, "a", 6L, 10, 20, 30);
- append(table, "a", 90, 70, 80);
- Snapshot snapshot = table.latestSnapshot().get();
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, snapshot);
- Map<BinaryRow, CompactUnit> units = manager.createCompactUnits(false);
- assertThat(units).hasSize(1);
- CompactUnit unit = units.get(partition(table, "a"));
- assertThat(unit.files())
- .extracting(DataFileMeta::fileName)
- .contains(dirtyBase.fileName())
- .doesNotContain(untouchedBase.fileName());
- assertThat(unit.files())
- .filteredOn(file -> file.nonNullRowIdRange().equals(new Range(6, 8)))
- .hasSize(3);
- commitCluster(table, snapshot, rewriteClusterUnits(table, manager, units));
-
- expected.addAll(
- Arrays.asList(
- "a|10|updated-10|15",
- "a|20|updated-20|16",
- "a|30|updated-30|17",
- "a|70|base-70|18",
- "a|80|base-80|19",
- "a|90|base-90|20"));
- assertThat(readRows(table)).containsExactlyElementsOf(expected);
- assertThat(fileNames(table, "a"))
- .contains(untouchedBase.fileName())
- .doesNotContain(dirtyBase.fileName());
- }
-
- @Test
- void testConcurrentPartialUpdateRejectsStaleCluster() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, true);
-
- updatePayload(table, "a", 0L, 30, 10, 20);
- long updateSnapshot = table.latestSnapshot().get().id();
- assertThatThrownBy(() -> commitCluster(table, snapshot, messages))
- .isInstanceOf(RuntimeException.class)
- .hasMessageContaining(DATA_EVOLUTION_ROW_ID_CONFLICT_MESSAGE);
- assertThat(table.latestSnapshot().get().id()).isEqualTo(updateSnapshot);
- assertThat(readRows(table))
- .containsExactly("a|30|updated-30|0", "a|10|updated-10|1", "a|20|updated-20|2");
- }
-
- @Test
- void testConcurrentDeleteRejectsStaleCluster() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, true);
-
- delete(table, "a", new Range(0, 2), 1L);
- long deleteSnapshot = table.latestSnapshot().get().id();
- // The delete changes the anchor's DV metadata, so normal file conflict detection rejects
- // the stale removal before the additional row-ID conflict check is reached.
- assertThatThrownBy(() -> commitCluster(table, snapshot, messages))
- .isInstanceOf(RuntimeException.class)
- .hasMessageContaining("File deletion conflicts detected!");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(deleteSnapshot);
- assertThat(readRows(table)).containsExactly("a|30|base-30|0", "a|20|base-20|2");
- }
-
- @Test
- void testConcurrentIndependentAppendIsPreserved() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, true);
-
- append(table, "a", 60, 40, 50);
- commitCluster(table, snapshot, messages);
-
- assertThat(readRows(table))
- .containsExactly(
- "a|60|base-60|3",
- "a|40|base-40|4",
- "a|50|base-50|5",
- "a|10|base-10|6",
- "a|20|base-20|7",
- "a|30|base-30|8");
- }
-
- @Test
- void testPartitionClusterDropsOnlyAffectedGlobalIndexes() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- append(table, "b", 60, 40, 50);
- addGlobalIndex(table, "a", "index-a", new Range(0, 2), "id");
- addGlobalIndex(table, "b", "index-b", new Range(3, 5), "id");
- List<String> untouchedFiles = fileNames(table, "b");
-
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages =
- prepareCluster(table, snapshot, partitionPredicate(table, "a"), true);
- addGlobalIndex(table, "a", "concurrent-index-a", new Range(0, 2), "payload");
- assertThat(globalIndexNames(table))
- .containsExactly("concurrent-index-a", "index-a", "index-b");
- commitCluster(table, snapshot, messages);
-
- assertThat(globalIndexNames(table)).containsExactly("index-b");
- assertThat(fileNames(table, "b")).isEqualTo(untouchedFiles);
- assertThat(readRows(table))
- .containsExactly(
- "b|60|base-60|3",
- "b|40|base-40|4",
- "b|50|base-50|5",
- "a|10|base-10|6",
- "a|20|base-20|7",
- "a|30|base-30|8");
- }
-
- @Test
- void testWithoutDeletionVectorsDropsIndexCommittedAfterPreparation() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- addGlobalIndex(table, "a", "index-before-plan", new Range(0, 2), "id");
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(table, snapshot, null, true);
- messages =
- IncrementalClusterRewriter.create(table, snapshot).prepareCommitMessages(messages);
-
- addGlobalIndex(table, "a", "index-after-preparation", new Range(0, 2), "payload");
- assertThat(globalIndexNames(table))
- .containsExactly("index-after-preparation", "index-before-plan");
- commitChecked(table, messages);
-
- assertThat(globalIndexNames(table)).isEmpty();
- assertThat(readRows(table))
- .containsExactly("a|10|base-10|3", "a|20|base-20|4", "a|30|base-30|5");
- }
-
- @ParameterizedTest
- @CsvSource({"btree,false", "btree,true", "bitmap,false", "bitmap,true"})
- void testClusterRemapsScalarIndexPostings(String indexType, boolean deletionVectors)
- throws Exception {
- FileStoreTable table = createTable(deletionVectors);
- appendIndexRows(
- table,
- new int[] {30, 10, 20, 40, 50},
- new String[] {"x", "x", null, "deleted", "y"});
- commit(
- table,
- prepareGlobalIndex(
- table, table.latestSnapshot().get(), "payload", indexType, null));
- if (deletionVectors) {
- delete(table, "a", new Range(0, 4), 3L);
- }
- table = table.copy(Collections.singletonMap(CoreOptions.TARGET_FILE_ROW_NUM.key(), "2"));
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMappedCluster(table, snapshot, true));
-
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int payload = table.rowType().getFieldIndex("payload");
- assertIndexRows(
- table,
- predicates.equal(payload, BinaryString.fromString("x")),
- new Range(5, 5),
- new Range(7, 7));
- assertIndexRows(table, predicates.isNull(payload), new Range(6, 6));
- assertIndexRows(
- table,
- predicates.equal(payload, BinaryString.fromString("y")),
- new Range(deletionVectors ? 8 : 9, deletionVectors ? 8 : 9));
- assertIndexRows(
- table,
- predicates.equal(payload, BinaryString.fromString("deleted")),
- deletionVectors ? new Range[0] : new Range[] {new Range(8, 8)});
- assertThat(readRows(table))
- .containsExactlyElementsOf(
- deletionVectors
- ? Arrays.asList("a|10|x|5", "a|20|null|6", "a|30|x|7", "a|50|y|8")
- : Arrays.asList(
- "a|10|x|5",
- "a|20|null|6",
- "a|30|x|7",
- "a|40|deleted|8",
- "a|50|y|9"));
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testIndexRemappingRequiresClusteringRewriter(boolean materializeDeletionVectors)
- throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMappedCluster(table, snapshot, true);
- // An ordinary caller explicitly invalidates its old indexes. Only a clustering committer
- // should replace those deletions with remapped postings from the permutation sidecar.
- messages.addAll(
- new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages));
- messages =
- IncrementalClusterRewriter.create(table, snapshot).prepareCommitMessages(messages);
- try (TableCommitImpl commit = table.newCommit("test-scoped-index-rewriter")) {
- commit.materializeDvRowIdCheck();
- if (materializeDeletionVectors) {
- commit.withCommitChangesRewriter(
- new MaterializeDeletionVectorsCommitChangesRewriter(table));
- }
- commit.commit(messages);
- }
-
- assertThat(globalIndexNames(table)).isEmpty();
- assertThat(readRows(table))
- .containsExactly("a|10|base-10|3", "a|20|base-20|4", "a|30|base-30|5");
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree", "bitmap"})
- void testClusterPreservesExactPartialIndexCoverage(String indexType) throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 90, 10, 50);
- append(table, "a", 80, 20, 40);
- commit(
- table,
- prepareGlobalIndex(
- table, table.latestSnapshot().get(), "id", indexType, new Range(0, 2)));
- table = table.copy(Collections.singletonMap(CoreOptions.TARGET_FILE_ROW_NUM.key(), "2"));
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMappedCluster(table, snapshot, true));
-
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- assertIndexRows(table, predicates.equal(id, 10), new Range(6, 6));
- assertIndexRows(table, predicates.equal(id, 50), new Range(9, 9));
- assertIndexRows(table, predicates.equal(id, 90), new Range(11, 11));
- assertIndexRows(table, predicates.equal(id, 20));
- List<Range> coverage =
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .map(entry -> entry.indexFile().globalIndexMeta().rowRange())
- .collect(Collectors.toList());
- assertThat(Range.sortAndMergeOverlap(coverage, true))
- .containsExactly(new Range(6, 6), new Range(9, 9), new Range(11, 11));
- FileStoreTable detail =
- table.copy(
- Collections.singletonMap(
- CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "detail"));
- assertThat(readRows(detail, predicates.equal(id, 20))).containsExactly("a|20|base-20|7");
- assertThat(readRows(detail, predicates.equal(id, 80))).containsExactly("a|80|base-80|10");
- }
-
- @Test
- void testClusterRejectsTooManyIndexCoverageFragments() throws Exception {
- FileStoreTable table = createTable(false);
- int[] indexed = new int[65];
- int[] unindexed = new int[65];
- for (int i = 0; i < indexed.length; i++) {
- indexed[i] = i * 2 + 1;
- unindexed[i] = i * 2 + 2;
- }
- append(table, "a", indexed);
- append(table, "a", unindexed);
- commit(
- table,
- prepareGlobalIndex(
- table, table.latestSnapshot().get(), "id", "btree", new Range(0, 64)));
- Snapshot snapshot = table.latestSnapshot().get();
- List<String> beforeFiles = fileNames(table, "a");
- List<String> beforeIndexes = globalIndexNames(table);
- List<String> beforeRows = readRows(table);
- List<CommitMessage> messages = prepareMappedCluster(table, snapshot, true);
- assertThat(messages).hasSize(1);
- assertThat(((CommitMessageImpl) messages.get(0)).compactIncrement().compactAfter())
- .hasSize(1);
-
- // One indexed row alternates with one unindexed row: 65 exact coverage fragments.
- assertThatThrownBy(() -> commitCluster(table, snapshot, messages))
- .isInstanceOf(RuntimeException.class)
- .hasStackTraceContaining("coverage fragments")
- .hasStackTraceContaining("64");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(snapshot.id());
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(130L);
- assertThat(fileNames(table, "a")).isEqualTo(beforeFiles);
- assertThat(globalIndexNames(table)).isEqualTo(beforeIndexes);
- assertThat(readRows(table)).isEqualTo(beforeRows);
- assertIndexRows(
- table,
- new PredicateBuilder(table.rowType()).equal(table.rowType().getFieldIndex("id"), 1),
- new Range(0, 0));
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree", "bitmap"})
- void testClusterCombinesFullyIndexedSourceRangesBeforeCountingFragments(String indexType)
- throws Exception {
- FileStoreTable table = createTable(false);
- int[] odd = new int[65];
- int[] even = new int[65];
- for (int i = 0; i < odd.length; i++) {
- odd[i] = i * 2 + 1;
- even[i] = i * 2 + 2;
- }
- append(table, "a", odd);
- append(table, "a", even);
- Snapshot indexed = table.latestSnapshot().get();
- List<CommitMessage> indexes =
- new ArrayList<>(
- prepareGlobalIndex(table, indexed, "id", indexType, new Range(0, 64)));
- indexes.addAll(prepareGlobalIndex(table, indexed, "id", indexType, new Range(65, 129)));
- commit(table, indexes);
- assertThat(globalIndexNames(table)).hasSize(2);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMappedCluster(table, snapshot, true);
- assertThat(messages).hasSize(1);
- assertThat(((CommitMessageImpl) messages.get(0)).compactIncrement().compactAfter())
- .hasSize(1);
-
- // Each old range maps to 65 alternating positions, but their union has no coverage gap.
- commitCluster(table, snapshot, messages);
-
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(260L);
- assertThat(
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .map(entry -> entry.indexFile().globalIndexMeta().rowRange()))
- .isNotEmpty()
- .allSatisfy(range -> assertThat(range).isEqualTo(new Range(130, 259)));
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- List<String> expectedRows = new ArrayList<>();
- for (int value = 1; value <= 130; value++) {
- long rowId = 129L + value;
- assertIndexRows(table, predicates.equal(id, value), new Range(rowId, rowId));
- expectedRows.add("a|" + value + "|base-" + value + "|" + rowId);
- }
- assertIndexRows(table, predicates.equal(id, 0));
- assertIndexRows(table, predicates.isNull(id));
- assertThat(readRows(table)).containsExactlyElementsOf(expectedRows);
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testMinorClusterRemapsIndexesAcrossDisjointRanges(boolean deletionVectors)
- throws Exception {
- FileStoreTable table = createMinorClusterTable(deletionVectors);
- List<String> untouchedRows = rowsInRange(table, new Range(12, 14));
- List<String> untouchedFiles = filesInRange(table, new Range(12, 14));
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- List<String> oldIndexFiles = globalIndexNames(table);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMinorCluster(table, snapshot);
- messages =
- IncrementalClusterRewriter.create(table, snapshot).prepareCommitMessages(messages);
- List<CommitMessage> preparedGapIndex =
- prepareGlobalIndex(table, snapshot, "payload", "bitmap", new Range(12, 14));
- commit(table, prepareGlobalIndex(table, snapshot, "payload"));
- commitChecked(table, messages);
-
- assertThat(globalIndexNames(table)).isNotEmpty().doesNotContainAnyElementsOf(oldIndexFiles);
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- int payload = table.rowType().getFieldIndex("payload");
- assertIndexRows(table, predicates.equal(id, 40), new Range(12, 12));
- assertIndexRows(table, predicates.equal(id, 50), new Range(13, 13));
- assertIndexRows(table, predicates.equal(id, 60), new Range(14, 14));
- long seventy = deletionVectors ? 23 : 24;
- assertIndexRows(table, predicates.equal(id, 70), new Range(seventy, seventy));
- assertIndexRows(
- table,
- predicates.equal(payload, BinaryString.fromString("updated-70")),
- new Range(seventy, seventy));
- assertMinorClusterResult(table, deletionVectors, untouchedRows, untouchedFiles, 21L, 4);
-
- // A pre-clustering index can still commit for an unchanged row-ID group.
- commit(table, preparedGapIndex);
- assertIndexRows(table, predicates.equal(id, 40), new Range(12, 12));
- assertMinorClusterResult(table, deletionVectors, untouchedRows, untouchedFiles, 21L, 4);
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testFullyDeletedRewritePreservesIndexForUnchangedRows(boolean clustering)
- throws Exception {
- FileStoreTable table = createMinorClusterTable(true);
- List<String> untouchedRows = rowsInRange(table, new Range(12, 14));
- List<String> untouchedFiles = filesInRange(table, new Range(12, 14));
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- delete(table, "a", new Range(9, 11), 9L, 10L, 11L);
- delete(table, "a", new Range(15, 17), 15L, 16L, 17L);
- delete(table, "a", new Range(18, 20), 18L, 19L, 20L);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMinorCluster(table, snapshot);
- assertThat(messages)
- .allSatisfy(
- message ->
- assertThat(
- ((CommitMessageImpl) message)
- .compactIncrement()
- .compactAfter())
- .isEmpty());
- if (clustering) {
- commitCluster(table, snapshot, messages);
- } else {
- try (TableCommitImpl commit = table.newCommit("test-materialize-deleted-rows")) {
- commit.materializeDvRowIdCheck()
- .withCommitChangesRewriter(
- new MaterializeDeletionVectorsCommitChangesRewriter(table))
- .commit(
- IncrementalClusterRewriter.create(table, snapshot)
- .prepareCommitMessages(messages));
- }
- }
-
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- assertIndexRows(table, predicates.equal(id, 40), new Range(12, 12));
- assertIndexRows(table, predicates.equal(id, 20));
- assertIndexRows(table, predicates.equal(id, 100));
- assertThat(
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .map(entry -> entry.indexFile().globalIndexMeta().rowRange()))
- .containsExactly(new Range(12, 14));
- assertThat(readRows(table)).containsExactlyElementsOf(untouchedRows);
- assertThat(fileNames(table, "a")).containsExactlyElementsOf(untouchedFiles);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(21L);
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testMinorClusterRetryRemapsConcurrentIndexAfterAppend(boolean deletionVectors)
- throws Exception {
- Map<String, String> retryOptions = new HashMap<>();
- retryOptions.put(CoreOptions.COMMIT_MIN_RETRY_WAIT.key(), "0ms");
- retryOptions.put(CoreOptions.COMMIT_MAX_RETRY_WAIT.key(), "0ms");
- FileStoreTable original = createMinorClusterTable(deletionVectors).copy(retryOptions);
- List<String> untouchedRows = rowsInRange(original, new Range(12, 14));
- List<String> untouchedFiles = filesInRange(original, new Range(12, 14));
- commit(original, prepareGlobalIndex(original, original.latestSnapshot().get(), "id"));
- Snapshot scanSnapshot = original.latestSnapshot().get();
- List<CommitMessage> messages = prepareMinorCluster(original, scanSnapshot);
- AtomicInteger attempts = new AtomicInteger();
- CatalogEnvironment environment =
- new CatalogEnvironment(null, null, null, null, null, null, false, false) {
- @Override
- public SnapshotCommit snapshotCommit(
- SnapshotManager manager, CoreOptions options, String commitUser) {
- SnapshotCommit delegate = new RenamingSnapshotCommit(manager, Lock.empty());
- return new SnapshotCommit(Lock.empty()) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- assertThat(snapshot.commitKind())
- .isEqualTo(Snapshot.CommitKind.COMPACT);
- int attempt = attempts.incrementAndGet();
- assertThat(
- original.store()
- .indexManifestFileFactory()
- .create()
- .read(snapshot.indexManifest()))
- .isNotEmpty();
- if (attempt == 1) {
- assertThat(original.latestSnapshot().get().id())
- .isEqualTo(scanSnapshot.id());
- DataEvolutionClusterTest.this.commit(
- original,
- prepareGlobalIndex(original, scanSnapshot, "payload"));
- append(original, "a", 200, 180, 190);
- }
- boolean committed =
- delegate.commit(baseUuid, snapshot, branch, statistics);
- assertThat(committed).isEqualTo(attempt == 2);
- return committed;
- }
-
- @Override
- protected void closeImpl() throws Exception {
- delegate.close();
- }
- };
- }
- };
- FileStoreTable committingTable =
- FileStoreTableFactory.create(
- original.fileIO(), original.location(), original.schema(), environment);
- commitCluster(committingTable, scanSnapshot, messages);
-
- assertThat(attempts).hasValue(2);
- assertThat(original.latestSnapshot().get().id()).isEqualTo(scanSnapshot.id() + 3);
- long seventy = deletionVectors ? 26 : 27;
- PredicateBuilder predicates = new PredicateBuilder(original.rowType());
- assertIndexRows(
- original,
- predicates.equal(original.rowType().getFieldIndex("id"), 70),
- new Range(seventy, seventy));
- assertIndexRows(
- original,
- predicates.equal(
- original.rowType().getFieldIndex("payload"),
- BinaryString.fromString("updated-70")),
- new Range(seventy, seventy));
- untouchedRows.addAll(
- Arrays.asList("a|200|base-200|21", "a|180|base-180|22", "a|190|base-190|23"));
- assertMinorClusterResult(original, deletionVectors, untouchedRows, untouchedFiles, 24L, 5);
- FileStoreTable detail =
- original.copy(
- Collections.singletonMap(
- CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "detail"));
- assertThat(readRows(detail, predicates.equal(original.rowType().getFieldIndex("id"), 180)))
- .containsExactly("a|180|base-180|22");
- }
-
- @Test
- void testClusterDoesNotMakeStaleIndexKeysFresh() throws Exception {
- FileStoreTable table =
- createTable(false)
- .copy(
- Collections.singletonMap(
- CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION.key(),
- "ignore"));
- append(table, "a", 30, 10, 20);
- Snapshot indexed = table.latestSnapshot().get();
- List<CommitMessage> indexes =
- new ArrayList<>(prepareGlobalIndex(table, indexed, "payload"));
- indexes.addAll(prepareGlobalIndex(table, indexed, "id"));
- commit(table, indexes);
- updatePayload(table, "a", 0L, 30, 10, 20);
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMappedCluster(table, snapshot, true));
-
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- assertIndexRows(
- table, predicates.equal(table.rowType().getFieldIndex("id"), 20), new Range(4, 4));
- assertThat(
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .flatMap(
- entry ->
- entry.indexFile().globalIndexMeta()
- .getIndexedFieldIds().stream()))
- .doesNotContain(table.rowType().getField("payload").id());
- FileStoreTable detail =
- table.copy(
- Collections.singletonMap(
- CoreOptions.SCALAR_INDEX_SEARCH_MODE.key(), "detail"));
- assertThat(
- readRows(
- detail,
- predicates.equal(
- table.rowType().getFieldIndex("payload"),
- BinaryString.fromString("updated-20"))))
- .containsExactly("a|20|updated-20|4");
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testClusterKeepsIndexPayloadsAfterUncertainPublication(boolean publicationThrows)
- throws Exception {
- FileStoreTable original = createTable(false);
- append(original, "a", 30, 10, 20);
- commit(original, prepareGlobalIndex(original, original.latestSnapshot().get(), "id"));
- Snapshot scanSnapshot = original.latestSnapshot().get();
- List<CommitMessage> messages = prepareMappedCluster(original, scanSnapshot, true);
- AtomicInteger publications = new AtomicInteger();
- CatalogEnvironment environment =
- new CatalogEnvironment(null, null, null, null, null, null, false, false) {
- @Override
- public SnapshotCommit snapshotCommit(
- SnapshotManager manager, CoreOptions options, String commitUser) {
- SnapshotCommit delegate = new RenamingSnapshotCommit(manager, Lock.empty());
- return new SnapshotCommit(Lock.empty()) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- publications.incrementAndGet();
- assertThat(delegate.commit(baseUuid, snapshot, branch, statistics))
- .isTrue();
- if (publicationThrows) {
- throw new IOException("Publication response lost");
- }
- return false;
- }
-
- @Override
- protected void closeImpl() throws Exception {
- delegate.close();
- }
- };
- }
- };
- FileStoreTable committingTable =
- FileStoreTableFactory.create(
- original.fileIO(), original.location(), original.schema(), environment);
- commitCluster(committingTable, scanSnapshot, messages);
-
- assertThat(publications).hasValue(1);
- assertThat(original.latestSnapshot().get().id()).isEqualTo(scanSnapshot.id() + 1);
- assertThat(original.latestSnapshot().get().nextRowId()).isEqualTo(6L);
- List<IndexFileMeta> indexes =
- original.store().newIndexFileHandler().scanEntries().stream()
- .map(IndexManifestEntry::indexFile)
- .filter(file -> file.globalIndexMeta() != null)
- .collect(Collectors.toList());
- assertThat(indexes).isNotEmpty();
- for (IndexFileMeta index : indexes) {
- assertThat(
- original.fileIO()
- .exists(
- original.store()
- .pathFactory()
- .globalIndexFileFactory()
- .toPath(index)))
- .isTrue();
- }
- PredicateBuilder predicates = new PredicateBuilder(original.rowType());
- int id = original.rowType().getFieldIndex("id");
- assertIndexRows(original, predicates.equal(id, 10), new Range(3, 3));
- assertIndexRows(original, predicates.equal(id, 20), new Range(4, 4));
- assertIndexRows(original, predicates.equal(id, 30), new Range(5, 5));
- assertThat(readRows(original))
- .containsExactly("a|10|base-10|3", "a|20|base-20|4", "a|30|base-30|5");
- }
-
- @Test
- void testClusterRemapsCompositeBTreeIndex() throws Exception {
- FileStoreTable table = createTable(false);
- appendIndexRows(table, new int[] {30, 10, 20, 40}, new String[] {"x", "x", null, "y"});
- Snapshot indexed = table.latestSnapshot().get();
- List<String> fields = Arrays.asList("id", "payload");
- List<CommitMessage> indexes = new ArrayList<>();
- for (DataSplit split :
- GlobalIndexBuilderUtils.splitByContiguousRowRange(
- new SortedGlobalIndexScanner(table, "btree")
- .withIndexFields(fields)
- .withSnapshot(indexed)
- .scan()
- .get()
- .entries())) {
- indexes.addAll(
- SortedGlobalIndexTestUtils.buildIndex(
- table, "btree", fields, split, indexed.id()));
- }
- assertThat(indexes).isNotEmpty();
- commit(table, indexes);
- table = table.copy(Collections.singletonMap(CoreOptions.TARGET_FILE_ROW_NUM.key(), "2"));
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMappedCluster(table, snapshot, true));
-
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- int payload = table.rowType().getFieldIndex("payload");
- assertIndexRows(
- table,
- PredicateBuilder.and(
- predicates.equal(id, 10),
- predicates.equal(payload, BinaryString.fromString("x"))),
- new Range(4, 4));
- assertIndexRows(
- table,
- PredicateBuilder.and(predicates.equal(id, 20), predicates.isNull(payload)),
- new Range(5, 5));
- assertIndexRows(
- table,
- PredicateBuilder.and(
- predicates.equal(id, 30),
- predicates.equal(payload, BinaryString.fromString("x"))),
- new Range(6, 6));
- assertIndexRows(
- table,
- PredicateBuilder.and(
- predicates.equal(id, 20),
- predicates.equal(payload, BinaryString.fromString("y"))));
- }
-
- @Test
- void testClusterPreservesIndexOnlyDeleteForAnotherPartition() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- append(table, "b", 60, 40, 50);
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- List<String> untouchedFiles = fileNames(table, "b");
- List<IndexFileMeta> deletedIndexes =
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.partition().equals(partition(table, "b")))
- .map(IndexManifestEntry::indexFile)
- .filter(file -> file.globalIndexMeta() != null)
- .collect(Collectors.toList());
- assertThat(deletedIndexes).hasSize(1);
- Snapshot snapshot = table.latestSnapshot().get();
- IncrementalClusterManager manager =
- new IncrementalClusterManager(table, partitionPredicate(table, "a"), snapshot);
- Map<BinaryRow, CompactUnit> units = manager.createCompactUnits(true);
- assertThat(units).containsOnlyKeys(partition(table, "a"));
- List<CommitMessage> messages =
- manager.prepareCommitMessages(rewriteClusterUnits(table, manager, units, true));
- messages.add(
- new CommitMessageImpl(
- partition(table, "b"),
- UNAWARE_BUCKET,
- null,
- DataIncrement.emptyIncrement(),
- new CompactIncrement(
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- deletedIndexes))
- .withCheckFromSnapshot(snapshot.id()));
- commitChecked(table, messages);
-
- assertThat(table.store().newIndexFileHandler().scanEntries())
- .allSatisfy(
- entry -> assertThat(entry.partition()).isEqualTo(partition(table, "a")));
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- assertIndexRows(table, predicates.equal(id, 10), new Range(6, 6));
- assertIndexRows(table, predicates.equal(id, 20), new Range(7, 7));
- assertIndexRows(table, predicates.equal(id, 30), new Range(8, 8));
- assertIndexRows(table, predicates.equal(id, 40));
- assertThat(fileNames(table, "b")).isEqualTo(untouchedFiles);
- assertThat(readRows(table))
- .containsExactly(
- "b|60|base-60|3",
- "b|40|base-40|4",
- "b|50|base-50|5",
- "a|10|base-10|6",
- "a|20|base-20|7",
- "a|30|base-30|8");
- }
-
- @Test
- void testAdjacentIndexCoverageKeepsSeparateFreshnessBaselines() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- append(table, "a", 60, 40, 50);
- Snapshot indexed = table.latestSnapshot().get();
- commit(table, prepareGlobalIndex(table, indexed, "id"));
- Snapshot snapshot = table.latestSnapshot().get();
- List<DataFileMeta> selected =
- table.store().newScan().withSnapshot(snapshot).plan().files().stream()
- .map(entry -> entry.file())
- .filter(file -> file.nonNullRowIdRange().equals(new Range(0, 2)))
- .collect(Collectors.toList());
- assertThat(selected).hasSize(1);
- List<String> untouchedFiles = filesInRange(table, new Range(3, 5));
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, snapshot);
- Map<BinaryRow, CompactUnit> units =
- Collections.singletonMap(
- partition(table, "a"), CompactUnit.fromFiles(1, selected, true));
- commitCluster(table, snapshot, rewriteClusterUnits(table, manager, units, true));
-
- Snapshot clustered = table.latestSnapshot().get();
- Map<Range, Long> baselines =
- table.store().newIndexFileHandler().scanEntries().stream()
- .map(IndexManifestEntry::indexFile)
- .filter(file -> file.globalIndexMeta() != null)
- .collect(
- Collectors.toMap(
- file -> file.globalIndexMeta().rowRange(),
- file ->
- DataEvolutionIndexSourceMeta.fromIndexFile(file)
- .scanSnapshotId()));
- assertThat(baselines)
- .hasSize(2)
- .containsEntry(new Range(3, 5), indexed.id())
- .containsEntry(new Range(6, 8), clustered.id());
- assertThat(filesInRange(table, new Range(3, 5))).isEqualTo(untouchedFiles);
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- assertIndexRows(table, predicates.equal(id, 10), new Range(6, 6));
- assertIndexRows(table, predicates.equal(id, 60), new Range(3, 3));
-
- append(table, "a", 90, 70, 80);
- Snapshot next = table.latestSnapshot().get();
- commitCluster(table, next, prepareMappedCluster(table, next, true));
-
- // Neither the retained coverage nor the first rewrite became stale on the second pass.
- assertIndexRows(table, predicates.equal(id, 10), new Range(12, 12));
- assertIndexRows(table, predicates.equal(id, 20), new Range(13, 13));
- assertIndexRows(table, predicates.equal(id, 40), new Range(15, 15));
- assertIndexRows(table, predicates.equal(id, 60), new Range(17, 17));
- assertIndexRows(table, predicates.equal(id, 70));
- assertThat(readRows(table))
- .containsExactly(
- "a|10|base-10|12",
- "a|20|base-20|13",
- "a|30|base-30|14",
- "a|40|base-40|15",
- "a|50|base-50|16",
- "a|60|base-60|17",
- "a|70|base-70|18",
- "a|80|base-80|19",
- "a|90|base-90|20");
- }
-
- @Test
- void testMetadataOnlyCompactDoesNotReplayOldClusteringMapping() throws Exception {
- FileStoreTable table = createTable(false);
- append(table, "a", 30, 10, 20);
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- Snapshot snapshot = table.latestSnapshot().get();
- List<DataFileMeta> initialFiles =
- table.store().newScan().withSnapshot(snapshot).plan().files().stream()
- .map(entry -> entry.file())
- .collect(Collectors.toList());
- assertThat(initialFiles).hasSize(1);
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, snapshot);
- Map<BinaryRow, CompactUnit> units =
- Collections.singletonMap(
- partition(table, "a"), CompactUnit.fromFiles(1, initialFiles, true));
- commitCluster(table, snapshot, rewriteClusterUnits(table, manager, units, true));
-
- Snapshot clustered = table.latestSnapshot().get();
- DataFileMeta clusteredFile =
- table.store().newScan().withSnapshot(clustered).plan().files().get(0).file();
- assertThat(clusteredFile.nonNullRowIdRange()).isEqualTo(new Range(3, 5));
- assertThat(clusteredFile.extraFiles()).anyMatch(ClusteringRowIdMapping::isMappingFile);
- List<IndexFileMeta> deletedIndexes =
- table.store().newIndexFileHandler().scanEntries().stream()
- .map(IndexManifestEntry::indexFile)
- .filter(file -> file.globalIndexMeta() != null)
- .collect(Collectors.toList());
- assertThat(deletedIndexes).isNotEmpty();
- commit(
- table,
- new CommitMessageImpl(
- partition(table, "a"),
- UNAWARE_BUCKET,
- null,
- DataIncrement.emptyIncrement(),
- new CompactIncrement(
- Collections.singletonList(clusteredFile),
- Collections.singletonList(clusteredFile.upgrade(2)),
- Collections.emptyList(),
- Collections.emptyList(),
- deletedIndexes)));
-
- assertThat(table.latestSnapshot().get().id()).isEqualTo(clustered.id() + 1);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(6L);
- assertThat(fileNames(table, "a")).containsExactly(clusteredFile.fileName());
- DataFileMeta upgraded =
- table.store()
- .newScan()
- .withSnapshot(table.latestSnapshot().get())
- .plan()
- .files()
- .get(0)
- .file();
- assertThat(upgraded.level()).isEqualTo(2);
- assertThat(upgraded.nonNullRowIdRange()).isEqualTo(new Range(3, 5));
- assertThat(upgraded.extraFiles()).isEqualTo(clusteredFile.extraFiles());
- assertThat(globalIndexNames(table)).isEmpty();
- assertThat(readRows(table))
- .containsExactly("a|10|base-10|3", "a|20|base-20|4", "a|30|base-30|5");
- }
-
- @ParameterizedTest
- @CsvSource({"btree,false", "btree,true", "bitmap,false", "bitmap,true"})
- void testMixedClusteringAndCompactionPreservesUnchangedIndexSegments(
- String indexType, boolean deletionVectors) throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(deletionVectors);
- commit(table, prepareMixedIndexes(table, table.latestSnapshot().get(), "id", indexType));
- List<String> retainedIndexes = globalIndexNames(table, new Range(3, 5));
- List<String> remappedIndexes = globalIndexNames(table, new Range(6, 8));
- assertThat(retainedIndexes).isNotEmpty();
- assertThat(remappedIndexes).isNotEmpty();
- if (deletionVectors) {
- delete(table, "a", new Range(3, 5), 4L);
- delete(table, "a", new Range(6, 8), 7L);
- }
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMixedMaintenance(table, snapshot);
- commitCluster(table, snapshot, messages);
-
- assertThat(globalIndexNames(table))
- .containsAll(retainedIndexes)
- .doesNotContainAnyElementsOf(remappedIndexes);
- assertThat(globalIndexNames(table, new Range(3, 5))).isEqualTo(retainedIndexes);
- assertThat(filesInRange(table, new Range(3, 5))).hasSize(1);
- assertThat(readRows(table))
- .containsExactlyElementsOf(
- deletionVectors
- ? Arrays.asList(
- "a|40|updated-40|3",
- "a|60|updated-60|5",
- "a|20|base-20|9",
- "a|30|base-30|10")
- : Arrays.asList(
- "a|40|updated-40|3",
- "a|50|updated-50|4",
- "a|60|updated-60|5",
- "a|10|base-10|9",
- "a|20|base-20|10",
- "a|30|base-30|11"));
- int id = table.rowType().getFieldIndex("id");
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- assertIndexRows(table, predicates.equal(id, 40), new Range(3, 3));
- assertIndexRows(
- table,
- predicates.equal(id, 20),
- new Range(deletionVectors ? 9 : 10, deletionVectors ? 9 : 10));
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(deletionVectors ? 11L : 12L);
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree", "bitmap"})
- void testFullyDeletedClusteringAndCompactionRetainsOtherIndexSegments(String indexType)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(true);
- commit(table, prepareMixedIndexes(table, table.latestSnapshot().get(), "id", indexType));
- List<String> retainedIndexes = globalIndexNames(table, new Range(3, 5));
- delete(table, "a", new Range(3, 5), 4L);
- delete(table, "a", new Range(6, 8), 6L, 7L, 8L);
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMixedMaintenance(table, snapshot));
-
- assertThat(globalIndexNames(table)).isEqualTo(retainedIndexes);
- assertThat(readRows(table)).containsExactly("a|40|updated-40|3", "a|60|updated-60|5");
- assertThat(filesInRange(table, new Range(3, 5))).hasSize(1);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(9L);
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- assertIndexRows(
- table, predicates.equal(table.rowType().getFieldIndex("id"), 40), new Range(3, 3));
- assertIndexRows(table, predicates.equal(table.rowType().getFieldIndex("id"), 20));
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testNormalCompactionDoesNotInvalidateIndexes(boolean deletionVectors) throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(deletionVectors);
- if (deletionVectors) {
- delete(table, "a", new Range(3, 5), 4L);
- }
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- List<String> indexes = globalIndexNames(table);
- Snapshot snapshot = table.latestSnapshot().get();
- List<String> rows = readRows(table);
- commitCluster(
- table,
- snapshot,
- Collections.singletonList(prepareNormalCompaction(table, new Range(3, 5))));
-
- assertThat(globalIndexNames(table)).isEqualTo(indexes);
- assertThat(readRows(table)).isEqualTo(rows);
- assertThat(filesInRange(table, new Range(3, 5))).hasSize(1);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(9L);
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree", "bitmap"})
- void testMixedMaintenanceSplitsIndexSpanningRetainedAndReassignedRows(String indexType)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(false);
- commit(
- table,
- prepareGlobalIndex(table, table.latestSnapshot().get(), "id", indexType, null));
- assertThat(globalIndexNames(table, new Range(3, 8))).isNotEmpty();
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMixedMaintenance(table, snapshot));
-
- assertThat(
- table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .map(entry -> entry.indexFile().globalIndexMeta().rowRange()))
- .containsExactlyInAnyOrder(new Range(3, 5), new Range(9, 11));
- PredicateBuilder predicates = new PredicateBuilder(table.rowType());
- int id = table.rowType().getFieldIndex("id");
- assertIndexRows(table, predicates.equal(id, 40), new Range(3, 3));
- assertIndexRows(table, predicates.equal(id, 20), new Range(10, 10));
- assertIndexRows(table, predicates.notEqual(id, 40), new Range(4, 5), new Range(9, 11));
- }
-
- @ParameterizedTest
- @CsvSource({"btree,false", "btree,true", "bitmap,false", "bitmap,true"})
- void testMixedMaintenanceRetryRemapsConcurrentIndexAfterAppend(
- String indexType, boolean deletionVectors) throws Exception {
- Map<String, String> retryOptions = new HashMap<>();
- retryOptions.put(CoreOptions.COMMIT_MIN_RETRY_WAIT.key(), "0ms");
- retryOptions.put(CoreOptions.COMMIT_MAX_RETRY_WAIT.key(), "0ms");
- FileStoreTable original = createMixedMaintenanceTable(deletionVectors).copy(retryOptions);
- if (deletionVectors) {
- delete(original, "a", new Range(3, 5), 4L);
- delete(original, "a", new Range(6, 8), 7L);
- }
- commit(
- original,
- prepareMixedIndexes(original, original.latestSnapshot().get(), "id", indexType));
- Snapshot scanSnapshot = original.latestSnapshot().get();
- List<CommitMessage> messages = prepareMixedMaintenance(original, scanSnapshot);
- List<String> retainedIndexes = new ArrayList<>(globalIndexNames(original, new Range(3, 5)));
- AtomicInteger attempts = new AtomicInteger();
- CatalogEnvironment environment =
- new CatalogEnvironment(null, null, null, null, null, null, false, false) {
- @Override
- public SnapshotCommit snapshotCommit(
- SnapshotManager manager, CoreOptions options, String commitUser) {
- SnapshotCommit delegate = new RenamingSnapshotCommit(manager, Lock.empty());
- return new SnapshotCommit(Lock.empty()) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- int attempt = attempts.incrementAndGet();
- if (attempt == 1) {
- DataEvolutionClusterTest.this.commit(
- original,
- prepareMixedIndexes(
- original, scanSnapshot, "payload", indexType));
- retainedIndexes.clear();
- retainedIndexes.addAll(
- globalIndexNames(original, new Range(3, 5)));
- append(original, "a", 90, 70, 80);
- }
- boolean committed =
- delegate.commit(baseUuid, snapshot, branch, statistics);
- assertThat(committed).isEqualTo(attempt == 2);
- return committed;
- }
-
- @Override
- protected void closeImpl() throws Exception {
- delegate.close();
- }
- };
- }
- };
- FileStoreTable committingTable =
- FileStoreTableFactory.create(
- original.fileIO(), original.location(), original.schema(), environment);
- commitCluster(committingTable, scanSnapshot, messages);
-
- assertThat(attempts).hasValue(2);
- assertThat(original.latestSnapshot().get().id()).isEqualTo(scanSnapshot.id() + 3);
- assertThat(globalIndexNames(original, new Range(3, 5))).isEqualTo(retainedIndexes);
- assertThat(original.latestSnapshot().get().nextRowId())
- .isEqualTo(deletionVectors ? 14L : 15L);
- PredicateBuilder predicates = new PredicateBuilder(original.rowType());
- int id = original.rowType().getFieldIndex("id");
- assertIndexRows(original, predicates.equal(id, 40), new Range(3, 3));
- assertIndexRows(
- original,
- predicates.equal(id, 20),
- new Range(deletionVectors ? 12 : 13, deletionVectors ? 12 : 13));
- assertIndexRows(
- original,
- predicates.equal(
- original.rowType().getFieldIndex("payload"),
- BinaryString.fromString("updated-40")),
- new Range(3, 3));
- assertIndexRows(
- original,
- predicates.equal(
- original.rowType().getFieldIndex("payload"),
- BinaryString.fromString("base-20")),
- new Range(deletionVectors ? 12 : 13, deletionVectors ? 12 : 13));
- assertThat(readRows(original))
- .contains("a|90|base-90|9", "a|70|base-70|10", "a|80|base-80|11");
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testMixedMaintenanceAllowsConcurrentPartialUpdateOfRetainedRows(boolean deletionVectors)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(deletionVectors);
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMixedMaintenance(table, snapshot);
- updatePayload(table, "a", 3L, 400, 500, 600);
- commitCluster(table, snapshot, messages);
-
- assertThat(readRows(table))
- .containsExactly(
- "a|40|updated-400|3",
- "a|50|updated-500|4",
- "a|60|updated-600|5",
- "a|10|base-10|9",
- "a|20|base-20|10",
- "a|30|base-30|11");
- assertThat(filesInRange(table, new Range(3, 5))).hasSize(2);
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testMixedMaintenanceRejectsConcurrentPartialUpdateOfReassignedRows(boolean deletionVectors)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(deletionVectors);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMixedMaintenance(table, snapshot);
- updatePayload(table, "a", 6L, 300, 100, 200);
- long updatedSnapshot = table.latestSnapshot().get().id();
- assertThatThrownBy(() -> commitCluster(table, snapshot, messages))
- .isInstanceOf(RuntimeException.class)
- .hasMessageContaining(DATA_EVOLUTION_ROW_ID_CONFLICT_MESSAGE);
- assertThat(table.latestSnapshot().get().id()).isEqualTo(updatedSnapshot);
- assertThat(readRows(table))
- .contains("a|30|updated-300|6", "a|10|updated-100|7", "a|20|updated-200|8");
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testMixedMaintenanceAllowsOldIndexOnlyForRetainedRows(boolean deletionVectors)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(deletionVectors);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> retainedIndex =
- prepareGlobalIndex(table, snapshot, "id", "bitmap", new Range(3, 5));
- List<CommitMessage> reassignedIndex =
- prepareGlobalIndex(table, snapshot, "id", "bitmap", new Range(6, 8));
- commitCluster(table, snapshot, prepareMixedMaintenance(table, snapshot));
- commit(table, retainedIndex);
- List<String> indexes = globalIndexNames(table);
- long committedSnapshot = table.latestSnapshot().get().id();
- assertThatThrownBy(() -> commit(table, reassignedIndex))
- .isInstanceOf(RuntimeException.class)
- .hasMessageContaining("Global index row ID existence conflict");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(committedSnapshot);
- assertThat(globalIndexNames(table)).isEqualTo(indexes);
- assertIndexRows(
- table,
- new PredicateBuilder(table.rowType())
- .equal(table.rowType().getFieldIndex("id"), 40),
- new Range(3, 3));
- }
-
- @ParameterizedTest
- @ValueSource(longs = {3L, 6L})
- void testMixedMaintenanceRejectsConcurrentDeletionVectorChange(long firstRowId)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(true);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> messages = prepareMixedMaintenance(table, snapshot);
- delete(table, "a", new Range(firstRowId, firstRowId + 2), firstRowId + 1);
- List<String> deletedRows = readRows(table);
- long deletedSnapshot = table.latestSnapshot().get().id();
- assertThatThrownBy(() -> commitCluster(table, snapshot, messages))
- .isInstanceOf(RuntimeException.class)
- .hasMessageContaining("File deletion conflicts detected!");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(deletedSnapshot);
- assertThat(readRows(table)).isEqualTo(deletedRows);
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testMixedMaintenanceAllowsPreviouslyPreparedUpdateOnlyForRetainedRows(
- boolean deletionVectors) throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(deletionVectors);
- Snapshot snapshot = table.latestSnapshot().get();
- List<CommitMessage> retainedUpdate =
- preparePayloadUpdate(table, "a", 3L, 400, 500, 600).stream()
- .map(
- message ->
- ((CommitMessageImpl) message)
- .withCheckFromSnapshot(snapshot.id()))
- .collect(Collectors.toList());
- List<CommitMessage> reassignedUpdate =
- preparePayloadUpdate(table, "a", 6L, 300, 100, 200).stream()
- .map(
- message ->
- ((CommitMessageImpl) message)
- .withCheckFromSnapshot(snapshot.id()))
- .collect(Collectors.toList());
- commitCluster(table, snapshot, prepareMixedMaintenance(table, snapshot));
- commit(table, retainedUpdate);
- long committedSnapshot = table.latestSnapshot().get().id();
- assertThatThrownBy(() -> commit(table, reassignedUpdate))
- .isInstanceOf(RowIdExistenceConflictException.class)
- .hasMessageContaining("Row ID existence conflict");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(committedSnapshot);
- assertThat(readRows(table))
- .contains("a|40|updated-400|3", "a|50|updated-500|4", "a|60|updated-600|5");
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"btree", "bitmap"})
- void testMixedMaintenanceWithoutMappingOnlyInvalidatesReassignedIndexRange(String indexType)
- throws Exception {
- FileStoreTable table = createMixedMaintenanceTable(false);
- commit(table, prepareMixedIndexes(table, table.latestSnapshot().get(), "id", indexType));
- List<String> retainedIndexes = globalIndexNames(table, new Range(3, 5));
- Snapshot snapshot = table.latestSnapshot().get();
- commitCluster(table, snapshot, prepareMixedMaintenance(table, snapshot, false));
-
- assertThat(globalIndexNames(table)).isEqualTo(retainedIndexes);
- assertThat(readRows(table))
- .containsExactly(
- "a|40|updated-40|3",
- "a|50|updated-50|4",
- "a|60|updated-60|5",
- "a|10|base-10|9",
- "a|20|base-20|10",
- "a|30|base-30|11");
- assertIndexRows(
- table,
- new PredicateBuilder(table.rowType())
- .equal(table.rowType().getFieldIndex("id"), 40),
- new Range(3, 3));
- }
-
- private FileStoreTable createMixedMaintenanceTable(boolean deletionVectors) throws Exception {
- FileStoreTable table = createTable(deletionVectors);
- append(table, "a", 60, 40, 50);
- Snapshot initial = table.latestSnapshot().get();
- commitCluster(table, initial, prepareMappedCluster(table, initial, true));
- for (int i = 0; i < 4; i++) {
- updatePayload(table, "a", 3L, 40, 50, 60);
- }
- append(table, "a", 30, 10, 20);
- assertThat(filesInRange(table, new Range(3, 5))).hasSize(5);
- return table;
- }
-
- private List<CommitMessage> prepareMixedMaintenance(FileStoreTable table, Snapshot snapshot)
- throws Exception {
- return prepareMixedMaintenance(table, snapshot, true);
- }
-
- private List<CommitMessage> prepareMixedMaintenance(
- FileStoreTable table, Snapshot snapshot, boolean preserveIndexes) throws Exception {
- List<DataFileMeta> selected =
- table.store().newScan().withSnapshot(snapshot).plan().files().stream()
- .map(entry -> entry.file())
- .filter(file -> file.nonNullRowIdRange().equals(new Range(6, 8)))
- .collect(Collectors.toList());
- assertThat(selected).isNotEmpty();
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, snapshot);
- List<CommitMessage> messages =
- rewriteClusterUnits(
- table,
- manager,
- Collections.singletonMap(
- partition(table, "a"), CompactUnit.fromFiles(3, selected, true)),
- preserveIndexes);
- messages.add(prepareNormalCompaction(table, new Range(3, 5)));
- return messages;
- }
-
- private CommitMessage prepareNormalCompaction(FileStoreTable table, Range range)
- throws Exception {
- List<DataFileMeta> files =
- table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .map(entry -> entry.file())
- .filter(file -> file.nonNullRowIdRange().equals(range))
- .collect(Collectors.toList());
- return new DataEvolutionNormalCompactTask(partition(table, "a"), files)
- .doCompact(table, "test-normal-compact");
- }
-
- private List<String> globalIndexNames(FileStoreTable table, Range range) {
- return table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .filter(entry -> entry.indexFile().globalIndexMeta().rowRange().equals(range))
- .map(entry -> entry.indexFile().fileName())
- .sorted()
- .collect(Collectors.toList());
- }
-
- private List<CommitMessage> prepareMixedIndexes(
- FileStoreTable table, Snapshot snapshot, String column, String indexType)
- throws Exception {
- List<CommitMessage> indexes =
- new ArrayList<>(
- prepareGlobalIndex(table, snapshot, column, indexType, new Range(3, 5)));
- indexes.addAll(prepareGlobalIndex(table, snapshot, column, indexType, new Range(6, 8)));
- return indexes;
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testClusterRetryDropsIndexCommittedBeforeSnapshotPublication(boolean deletionVectors)
- throws Exception {
- Map<String, String> retryOptions = new HashMap<>();
- retryOptions.put(CoreOptions.COMMIT_MIN_RETRY_WAIT.key(), "0ms");
- retryOptions.put(CoreOptions.COMMIT_MAX_RETRY_WAIT.key(), "0ms");
- FileStoreTable original = createTable(deletionVectors).copy(retryOptions);
- append(original, "a", 30, 10, 20);
- if (deletionVectors) {
- delete(original, "a", new Range(0, 2), 1L);
- }
- commit(original, prepareGlobalIndex(original, original.latestSnapshot().get(), "id"));
- Snapshot scanSnapshot = original.latestSnapshot().get();
- List<CommitMessage> messages = prepareCluster(original, scanSnapshot, null, true);
- AtomicInteger attempts = new AtomicInteger();
- CatalogEnvironment environment =
- new CatalogEnvironment(null, null, null, null, null, null, false, false) {
- @Override
- public SnapshotCommit snapshotCommit(
- SnapshotManager manager, CoreOptions options, String commitUser) {
- SnapshotCommit delegate = new RenamingSnapshotCommit(manager, Lock.empty());
- return new SnapshotCommit(Lock.empty()) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- assertThat(snapshot.commitKind())
- .isEqualTo(Snapshot.CommitKind.COMPACT);
- int attempt = attempts.incrementAndGet();
- // Index cleanup has already been prepared for this candidate.
- assertThat(
- original.store()
- .indexManifestFileFactory()
- .create()
- .read(snapshot.indexManifest()))
- .isEmpty();
- if (attempt == 1) {
- assertThat(original.latestSnapshot().get().id())
- .isEqualTo(scanSnapshot.id());
- DataEvolutionClusterTest.this.commit(
- original,
- prepareGlobalIndex(original, scanSnapshot, "payload"));
- assertThat(globalIndexNames(original)).hasSize(2);
- }
- boolean committed =
- delegate.commit(baseUuid, snapshot, branch, statistics);
- assertThat(committed).isEqualTo(attempt == 2);
- return committed;
- }
-
- @Override
- protected void closeImpl() throws Exception {
- delegate.close();
- }
- };
- }
- };
- FileStoreTable committingTable =
- FileStoreTableFactory.create(
- original.fileIO(), original.location(), original.schema(), environment);
- commitCluster(committingTable, scanSnapshot, messages);
-
- assertThat(attempts).hasValue(2);
- assertThat(original.latestSnapshot().get().id()).isEqualTo(scanSnapshot.id() + 2);
- assertThat(original.latestSnapshot().get().nextRowId())
- .isEqualTo(deletionVectors ? 5L : 6L);
- assertThat(globalIndexNames(original)).isEmpty();
- assertThat(readRows(original))
- .containsExactlyElementsOf(
- deletionVectors
- ? Arrays.asList("a|20|base-20|3", "a|30|base-30|4")
- : Arrays.asList(
- "a|10|base-10|3", "a|20|base-20|4", "a|30|base-30|5"));
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testClusterRejectsPreviouslyPreparedPartialUpdate(boolean deletionVectors)
- throws Exception {
- FileStoreTable table = createTable(deletionVectors);
- append(table, "a", 30, 10, 20);
- Snapshot scanSnapshot = table.latestSnapshot().get();
- List<CommitMessage> updates =
- preparePayloadUpdate(table, "a", 0L, 30, 10, 20).stream()
- .map(
- message ->
- ((CommitMessageImpl) message)
- .withCheckFromSnapshot(scanSnapshot.id()))
- .collect(Collectors.toList());
- commitCluster(table, scanSnapshot, prepareCluster(table, scanSnapshot, null, true));
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
-
- assertStaleCommitPreservesCluster(
- table, updates, RowIdExistenceConflictException.class, "Row ID existence conflict");
- }
-
- @Test
- void testClusterRejectsPreviouslyPreparedDelete() throws Exception {
- FileStoreTable table = createTable(true);
- append(table, "a", 30, 10, 20);
- Snapshot scanSnapshot = table.latestSnapshot().get();
- CommitMessage deletion =
- ((CommitMessageImpl) prepareDelete(table, "a", new Range(0, 2), 1L))
- .withCheckFromSnapshot(scanSnapshot.id());
- commitCluster(table, scanSnapshot, prepareCluster(table, scanSnapshot, null, true));
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
-
- assertStaleCommitPreservesCluster(
- table,
- Collections.singletonList(deletion),
- RuntimeException.class,
- "File deletion conflicts detected!");
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testClusterRejectsPreviouslyBuiltGlobalIndex(boolean deletionVectors) throws Exception {
- FileStoreTable table = createTable(deletionVectors);
- append(table, "a", 30, 10, 20);
- Snapshot scanSnapshot = table.latestSnapshot().get();
- List<CommitMessage> indexes = prepareGlobalIndex(table, scanSnapshot, "id");
- commitCluster(table, scanSnapshot, prepareCluster(table, scanSnapshot, null, true));
- commit(table, prepareGlobalIndex(table, table.latestSnapshot().get(), "id"));
-
- assertStaleCommitPreservesCluster(
- table, indexes, RuntimeException.class, "Global index row ID existence conflict");
- }
-
- private void assertStaleCommitPreservesCluster(
- FileStoreTable table,
- List<CommitMessage> staleMessages,
- Class<? extends RuntimeException> exceptionClass,
- String conflictMessage)
- throws Exception {
- Snapshot clustered = table.latestSnapshot().get();
- List<String> files = fileNames(table, "a");
- List<String> indexes = globalIndexNames(table);
- assertThat(indexes).isNotEmpty();
- assertThatThrownBy(() -> commit(table, staleMessages))
- .isInstanceOf(exceptionClass)
- .hasMessageContaining(conflictMessage);
- assertThat(table.latestSnapshot().get().id()).isEqualTo(clustered.id());
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(6L);
- assertThat(fileNames(table, "a")).isEqualTo(files);
- assertThat(globalIndexNames(table)).isEqualTo(indexes);
- assertThat(readRows(table))
- .containsExactly("a|10|base-10|3", "a|20|base-20|4", "a|30|base-30|5");
- }
-
- private FileStoreTable createTable(boolean deletionVectors) throws Exception {
- Schema schema =
- Schema.newBuilder()
- .column("pt", DataTypes.STRING())
- .column("id", DataTypes.INT())
- .column("payload", DataTypes.STRING())
- .partitionKeys("pt")
- .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
- .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
- .option(
- CoreOptions.DELETION_VECTORS_ENABLED.key(),
- String.valueOf(deletionVectors))
- .option(CoreOptions.CLUSTERING_COLUMNS.key(), "id")
- .option(CoreOptions.CLUSTERING_STRATEGY.key(), "order")
- .option(CoreOptions.CLUSTERING_INCREMENTAL.key(), "true")
- .option(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER.key(), "2")
- .build();
- catalog.createTable(identifier(), schema, false);
- return getTableDefault();
- }
-
- private FileStoreTable createMinorClusterTable(boolean deletionVectors) throws Exception {
- Map<String, String> options = new HashMap<>();
- options.put(CoreOptions.TARGET_FILE_ROW_NUM.key(), "3");
- options.put(CoreOptions.FILE_COMPRESSION.key(), "none");
- options.put(CoreOptions.COMPACTION_MAX_SIZE_AMPLIFICATION_PERCENT.key(), "10000");
- options.put(CoreOptions.COMPACTION_SIZE_RATIO.key(), "0");
- FileStoreTable table = createTable(deletionVectors).copy(options);
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- try (BatchTableWrite write = builder.newWrite();
- BatchTableCommit commit = builder.newCommit()) {
- for (int id : new int[] {90, 10, 60, 30, 80, 40, 20, 70, 50}) {
- Random random = new Random(id);
- int length = id >= 40 && id <= 60 ? 262144 : 8192;
- StringBuilder payload = new StringBuilder(length);
- for (int i = 0; i < length; i++) {
- payload.append((char) ('a' + random.nextInt(26)));
- }
- write.write(
- GenericRow.of(
- BinaryString.fromString("a"),
- id,
- BinaryString.fromString(payload.toString())));
- }
- commit.commit(write.prepareCommit());
- }
- Snapshot initial = table.latestSnapshot().get();
- commitCluster(table, initial, prepareCluster(table, initial, null, true));
- List<DataFileMeta> clusteredFiles =
- table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .map(entry -> entry.file())
- .sorted(Comparator.comparingLong(DataFileMeta::nonNullFirstRowId))
- .collect(Collectors.toList());
- assertThat(clusteredFiles)
- .extracting(DataFileMeta::nonNullRowIdRange)
- .containsExactly(new Range(9, 11), new Range(12, 14), new Range(15, 17));
- assertThat(clusteredFiles.get(1).fileSize())
- .isGreaterThan(
- (clusteredFiles.get(0).fileSize() + clusteredFiles.get(2).fileSize()) * 8);
-
- updatePayload(table, "a", 9L, 10, 20, 30);
- updatePayload(table, "a", 15L, 70, 80, 90);
- updateSortKeys(table, "a", 9L, 10, 20, 30);
- updateSortKeys(table, "a", 15L, 70, 80, 90);
- append(table, "a", 120, 100, 110);
- if (deletionVectors) {
- delete(table, "a", new Range(9, 11), 9L);
- }
- return table;
- }
-
- private List<CommitMessage> prepareMinorCluster(FileStoreTable table, Snapshot snapshot)
- throws Exception {
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, snapshot);
- Map<BinaryRow, CompactUnit> units = manager.createCompactUnits(false);
- assertThat(units).hasSize(1);
- List<DataFileMeta> selected = units.get(partition(table, "a")).files();
- assertThat(selected)
- .extracting(DataFileMeta::nonNullRowIdRange)
- .containsExactlyInAnyOrder(
- new Range(9, 11),
- new Range(9, 11),
- new Range(9, 11),
- new Range(15, 17),
- new Range(15, 17),
- new Range(15, 17),
- new Range(18, 20));
- assertThat(selected)
- .extracting(DataFileMeta::fileName)
- .doesNotContainAnyElementsOf(filesInRange(table, new Range(12, 14)));
- return rewriteClusterUnits(table, manager, units, true);
- }
-
- private void assertMinorClusterResult(
- FileStoreTable table,
- boolean deletionVectors,
- List<String> untouchedRows,
- List<String> untouchedFiles,
- long firstAssignedRowId,
- int expectedFileCount)
- throws Exception {
- List<String> expected = new ArrayList<>(untouchedRows);
- long rowId = firstAssignedRowId;
- for (int id : new int[] {10, 20, 30, 70, 80, 90, 100, 110, 120}) {
- if (deletionVectors && id == 10) {
- continue;
- }
- expected.add("a|" + id + "|" + (id < 100 ? "updated-" : "base-") + id + "|" + rowId++);
- }
- assertThat(readRows(table)).containsExactlyElementsOf(expected);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(rowId);
- assertThat(filesInRange(table, new Range(12, 14)))
- .containsExactlyElementsOf(untouchedFiles);
- assertThat(fileNames(table, "a")).hasSize(expectedFileCount).containsAll(untouchedFiles);
- }
-
- private List<String> rowsInRange(FileStoreTable table, Range range) throws Exception {
- return readRows(table).stream()
- .filter(
- row -> {
- long rowId = Long.parseLong(row.substring(row.lastIndexOf('|') + 1));
- return rowId >= range.from && rowId <= range.to;
- })
- .collect(Collectors.toList());
- }
-
- private List<String> filesInRange(FileStoreTable table, Range range) {
- return table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .filter(entry -> entry.partition().equals(partition(table, "a")))
- .map(entry -> entry.file())
- .filter(file -> file.nonNullRowIdRange().equals(range))
- .map(DataFileMeta::fileName)
- .sorted()
- .collect(Collectors.toList());
- }
-
- private void append(FileStoreTable table, String partition, int... values) throws Exception {
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- try (BatchTableWrite write = builder.newWrite();
- BatchTableCommit commit = builder.newCommit()) {
- for (int value : values) {
- write.write(
- GenericRow.of(
- BinaryString.fromString(partition),
- value,
- BinaryString.fromString("base-" + value)));
- }
- commit.commit(write.prepareCommit());
- }
- }
-
- private void updatePayload(
- FileStoreTable table, String partition, long firstRowId, int... values)
- throws Exception {
- commit(table, preparePayloadUpdate(table, partition, firstRowId, values));
- }
-
- private void updateSortKeys(
- FileStoreTable table, String partition, long firstRowId, int... values)
- throws Exception {
- RowType writeType = table.rowType().project(Arrays.asList("pt", "id"));
- try (BatchTableWrite write =
- table.newBatchWriteBuilder().newWrite().withWriteType(writeType)) {
- for (int value : values) {
- write.write(GenericRow.of(BinaryString.fromString(partition), value));
- }
- List<CommitMessage> messages = write.prepareCommit();
- assignFirstRowId(messages, firstRowId);
- commit(table, messages);
- }
- }
-
- private List<CommitMessage> preparePayloadUpdate(
- FileStoreTable table, String partition, long firstRowId, int... values)
- throws Exception {
- RowType writeType = table.rowType().project(Arrays.asList("pt", "payload"));
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- try (BatchTableWrite write = builder.newWrite().withWriteType(writeType)) {
- for (int value : values) {
- write.write(
- GenericRow.of(
- BinaryString.fromString(partition),
- BinaryString.fromString("updated-" + value)));
- }
- List<CommitMessage> messages = write.prepareCommit();
- assignFirstRowId(messages, firstRowId);
- return messages;
- }
- }
-
- private void assignFirstRowId(List<CommitMessage> messages, long firstRowId) {
- for (CommitMessage message : messages) {
- List<DataFileMeta> files = ((CommitMessageImpl) message).newFilesIncrement().newFiles();
- List<DataFileMeta> assigned =
- files.stream()
- .map(file -> file.assignFirstRowId(firstRowId))
- .collect(Collectors.toList());
- files.clear();
- files.addAll(assigned);
- }
- }
-
- private List<CommitMessage> prepareCluster(
- FileStoreTable table,
- Snapshot snapshot,
- PartitionPredicate partitionPredicate,
- boolean fullCompaction)
- throws Exception {
- IncrementalClusterManager manager =
- new IncrementalClusterManager(table, partitionPredicate, snapshot);
- return rewriteClusterUnits(table, manager, manager.createCompactUnits(fullCompaction));
- }
-
- private List<CommitMessage> prepareMappedCluster(
- FileStoreTable table, Snapshot snapshot, boolean fullCompaction) throws Exception {
- IncrementalClusterManager manager = new IncrementalClusterManager(table, null, snapshot);
- return rewriteClusterUnits(
- table, manager, manager.createCompactUnits(fullCompaction), true);
- }
-
- // Run the engine-independent read/sort/write stages in-process. The source selection and
- // split planning are the same shared manager used by the distributed engine implementations.
- private List<CommitMessage> rewriteClusterUnits(
- FileStoreTable table,
- IncrementalClusterManager manager,
- Map<BinaryRow, CompactUnit> units)
- throws Exception {
- return rewriteClusterUnits(table, manager, units, false);
- }
-
- private List<CommitMessage> rewriteClusterUnits(
- FileStoreTable table,
- IncrementalClusterManager manager,
- Map<BinaryRow, CompactUnit> units,
- boolean preserveIndexes)
- throws Exception {
- Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> planned =
- manager.toSplitsAndRewriteDvFiles(units);
- List<CommitMessage> messages = new ArrayList<>();
- FileStoreTable writeTable =
- table.copy(Collections.singletonMap(CoreOptions.WRITE_ONLY.key(), "true"));
- for (Map.Entry<BinaryRow, CompactUnit> entry : units.entrySet()) {
- Pair<List<DataSplit>, CommitMessage> partitionPlan = planned.get(entry.getKey());
- assertThat(partitionPlan.getRight()).isNull();
- List<DataFileMeta> after = new ArrayList<>();
- List<Long> oldRowIds = new ArrayList<>();
- RowType readType =
- preserveIndexes
- ? SpecialFields.rowTypeWithRowId(table.rowType())
- : table.rowType();
- ProjectedRow dataRow = ProjectedRow.from(table.rowType(), readType);
- try (IOManager ioManager = IOManager.create(tempPath.toString());
- RecordReaderIterator<InternalRow> reader =
- new RecordReaderIterator<>(
- table.newReadBuilder()
- .withReadType(readType)
- .newRead()
- .createReader(
- new ArrayList<>(partitionPlan.getLeft())));
- BatchTableWrite write =
- writeTable
- .newBatchWriteBuilder()
- .newWrite()
- .withWriteType(table.rowType())) {
- if (reader.hasNext()) {
- Sorter sorter =
- Sorter.getSorter(reader, ioManager, readType, table.coreOptions());
- try {
- MutableObjectIterator<BinaryRow> sorted = sorter.sort();
- BinaryRow row = new BinaryRow(sorter.arity());
- while ((row = sorted.next(row)) != null) {
- InternalRow sortedRow = sorter.removeSortKey(row);
- if (preserveIndexes) {
- oldRowIds.add(sortedRow.getLong(readType.getFieldCount() - 1));
- }
- write.write(dataRow.replaceRow(sortedRow));
- }
- } finally {
- sorter.close();
- }
- }
- for (CommitMessage output : write.prepareCommit()) {
- after.addAll(((CommitMessageImpl) output).newFilesIncrement().newFiles());
- }
- }
- DataFilePathFactory pathFactory =
- table.store().pathFactory().createDataFilePathFactory(entry.getKey(), 0);
- int rowOffset = 0;
- for (int i = 0; preserveIndexes && i < after.size(); i++) {
- DataFileMeta file = after.get(i);
- String mappingName = ClusteringRowIdMapping.fileName(file.fileName());
- try (ClusteringRowIdMapping.Writer mapping =
- ClusteringRowIdMapping.createWriter(
- table.fileIO(), pathFactory.toAlignedPath(mappingName, file))) {
- for (long row = 0; row < file.rowCount(); row++) {
- mapping.write(oldRowIds.get(rowOffset++));
- }
- }
- List<String> extraFiles = new ArrayList<>(file.extraFiles());
- extraFiles.add(mappingName);
- after.set(i, file.copy(extraFiles));
- }
- assertThat(rowOffset).isEqualTo(oldRowIds.size());
- messages.add(
- new CommitMessageImpl(
- entry.getKey(),
- 0,
- null,
- DataIncrement.emptyIncrement(),
- new CompactIncrement(
- entry.getValue().files(),
- IncrementalClusterManager.upgrade(
- after, entry.getValue().outputLevel()),
- Collections.emptyList())));
- }
- return messages;
- }
-
- private void commitCluster(
- FileStoreTable table, Snapshot snapshot, List<CommitMessage> messages)
- throws Exception {
- commitChecked(
- table,
- IncrementalClusterRewriter.create(table, snapshot).prepareCommitMessages(messages));
- }
-
- private void commitChecked(FileStoreTable table, List<CommitMessage> messages)
- throws Exception {
- try (TableCommitImpl commit = table.newCommit("test-cluster")) {
- commit.materializeDvRowIdCheck()
- .withCommitChangesRewriter(new ClusteringCommitChangesRewriter(table))
- .commit(messages);
- }
- }
-
- private void delete(FileStoreTable table, String value, Range range, long... rowIds)
- throws Exception {
- commit(table, prepareDelete(table, value, range, rowIds));
- }
-
- private CommitMessage prepareDelete(
- FileStoreTable table, String value, Range range, long... rowIds) throws Exception {
- BinaryRow partition = partition(table, value);
- List<DataFileMeta> group =
- table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .filter(entry -> entry.partition().equals(partition))
- .map(entry -> entry.file())
- .filter(file -> file.nonNullRowIdRange().equals(range))
- .collect(Collectors.toList());
- DataFileMeta anchor = retrieveAnchorFile(group, file -> file);
- BaseAppendDeleteFileMaintainer maintainer =
- BaseAppendDeleteFileMaintainer.forUnawareAppend(
- table.store().newIndexFileHandler(),
- table.latestSnapshot().get(),
- partition);
- DeletionVector vector = new BitmapDeletionVector();
- for (long rowId : rowIds) {
- vector.delete(rowId - range.from);
- }
- maintainer.notifyNewDeletionVector(anchor.fileName(), vector);
- List<IndexFileMeta> additions = new ArrayList<>();
- List<IndexFileMeta> removals = new ArrayList<>();
- for (IndexManifestEntry entry : maintainer.persist()) {
- (entry.kind() == FileKind.ADD ? additions : removals).add(entry.indexFile());
- }
- return new CommitMessageImpl(
- partition,
- UNAWARE_BUCKET,
- null,
- new DataIncrement(
- Collections.emptyList(),
- Collections.emptyList(),
- Collections.emptyList(),
- additions,
- removals),
- CompactIncrement.emptyIncrement());
- }
-
- private void addGlobalIndex(
- FileStoreTable table, String value, String fileName, Range range, String column)
- throws Exception {
- commit(table, globalIndexMessage(table, value, fileName, range, column));
- }
-
- private CommitMessage globalIndexMessage(
- FileStoreTable table, String value, String fileName, Range range, String column) {
- IndexFileMeta index =
- new IndexFileMeta(
- "test-global-index",
- fileName,
- 1,
- range.count(),
- new GlobalIndexMeta(
- range.from,
- range.to,
- table.rowType().getField(column).id(),
- null,
- null),
- null);
- return new CommitMessageImpl(
- partition(table, value),
- UNAWARE_BUCKET,
- null,
- DataIncrement.indexIncrement(Collections.singletonList(index)),
- CompactIncrement.emptyIncrement());
- }
-
- private void commit(FileStoreTable table, CommitMessage message) throws Exception {
- commit(table, Collections.singletonList(message));
- }
-
- private void commit(FileStoreTable table, List<CommitMessage> messages) throws Exception {
- try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
- commit.commit(messages);
- }
- }
-
- private List<CommitMessage> prepareGlobalIndex(
- FileStoreTable table, Snapshot snapshot, String column) throws Exception {
- return prepareGlobalIndex(table, snapshot, column, "btree", null);
- }
-
- private List<CommitMessage> prepareGlobalIndex(
- FileStoreTable table,
- Snapshot snapshot,
- String column,
- String indexType,
- Range fileRange)
- throws Exception {
- List<CommitMessage> messages = new ArrayList<>();
- for (DataSplit split :
- GlobalIndexBuilderUtils.splitByContiguousRowRange(
- new SortedGlobalIndexScanner(table, indexType)
- .withIndexFields(Collections.singletonList(column))
- .withSnapshot(snapshot)
- .scan()
- .get()
- .entries())) {
- if (fileRange != null) {
- Optional<DataSplit> filtered =
- split.filterDataFile(file -> file.nonNullRowIdRange().equals(fileRange));
- if (!filtered.isPresent()) {
- continue;
- }
- split = filtered.get();
- }
- messages.addAll(
- SortedGlobalIndexTestUtils.buildIndex(
- table, indexType, column, split, snapshot.id()));
- }
- assertThat(messages).isNotEmpty();
- return messages;
- }
-
- private List<String> readRows(FileStoreTable table) throws Exception {
- return readRows(table, null);
- }
-
- private List<String> readRows(FileStoreTable table, Predicate predicate) throws Exception {
- ReadBuilder builder =
- table.newReadBuilder()
- .withReadType(SpecialFields.rowTypeWithRowId(table.rowType()));
- if (predicate != null) {
- builder.withFilter(predicate);
- }
- List<String> rows = new ArrayList<>();
- try (RecordReader<InternalRow> reader =
- builder.newRead().createReader(builder.newScan().plan())) {
- reader.forEachRemaining(
- row -> {
- if (predicate == null || predicate.test(row)) {
- rows.add(
- row.getString(0)
- + "|"
- + row.getInt(1)
- + "|"
- + (row.isNullAt(2) ? "null" : row.getString(2))
- + "|"
- + row.getLong(3));
- }
- });
- }
- rows.sort(
- Comparator.comparingLong(
- row -> Long.parseLong(row.substring(row.lastIndexOf('|') + 1))));
- return rows;
- }
-
- private void appendIndexRows(FileStoreTable table, int[] ids, String[] payloads)
- throws Exception {
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- try (BatchTableWrite write = builder.newWrite();
- BatchTableCommit commit = builder.newCommit()) {
- for (int i = 0; i < ids.length; i++) {
- write.write(
- GenericRow.of(
- BinaryString.fromString("a"),
- ids[i],
- payloads[i] == null ? null : BinaryString.fromString(payloads[i])));
- }
- commit.commit(write.prepareCommit());
- }
- }
-
- private void assertIndexRows(FileStoreTable table, Predicate predicate, Range... expected)
- throws Exception {
- Optional<DataEvolutionGlobalIndexScanner> scanner =
- DataEvolutionGlobalIndexScanner.create(
- table, PartitionPredicate.ALWAYS_TRUE, predicate);
- if (!scanner.isPresent() && expected.length == 0) {
- return;
- }
- assertThat(scanner).isPresent();
- try (DataEvolutionGlobalIndexScanner index = scanner.get()) {
- Optional<GlobalIndexResult> result = index.scan(predicate);
- assertThat(result).isPresent();
- assertThat(result.get().results().toRangeList()).containsExactly(expected);
- }
- }
-
- private List<String> fileNames(FileStoreTable table, String value) {
- BinaryRow partition = partition(table, value);
- return table.store().newScan().withSnapshot(table.latestSnapshot().get()).plan().files()
- .stream()
- .filter(entry -> entry.partition().equals(partition))
- .map(entry -> entry.file().fileName())
- .sorted()
- .collect(Collectors.toList());
- }
-
- private List<String> globalIndexNames(FileStoreTable table) {
- return table.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .map(entry -> entry.indexFile().fileName())
- .sorted()
- .collect(Collectors.toList());
- }
-
- private BinaryRow partition(FileStoreTable table, String value) {
- return new InternalRowSerializer(table.schema().logicalPartitionType())
- .toBinaryRow(GenericRow.of(BinaryString.fromString(value)));
- }
-
- private PartitionPredicate partitionPredicate(FileStoreTable table, String value) {
- return PartitionPredicate.fromMaps(
- table.schema().logicalPartitionType(),
- Collections.singletonList(Collections.singletonMap("pt", value)),
- table.coreOptions().partitionDefaultName());
- }
-}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializerTest.java
deleted file mode 100644
index 98f273969f..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionCompactTaskSerializerTest.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.append.dataevolution;
-
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataFileMetaSerializer;
-import org.apache.paimon.io.DataInputDeserializer;
-import org.apache.paimon.io.DataOutputSerializer;
-import org.apache.paimon.manifest.FileSource;
-import org.apache.paimon.utils.Range;
-
-import org.junit.jupiter.api.Test;
-
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-
-import static org.apache.paimon.stats.SimpleStats.EMPTY_STATS;
-import static org.apache.paimon.utils.SerializationUtils.serializeBinaryRow;
-import static org.assertj.core.api.Assertions.assertThat;
-
-/** Task serialization keeps clustering provenance without changing ordinary compaction defaults. */
-class DataEvolutionCompactTaskSerializerTest {
-
- @Test
- void testOutputLevelsSurviveMixedTaskList() throws Exception {
- DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer();
- List<DataEvolutionCompactTask> tasks =
- Arrays.asList(
- new DataEvolutionNormalCompactTask(
- BinaryRow.EMPTY_ROW,
- files(),
- Collections.singletonList(new Range(0, 2)),
- 2),
- new DataEvolutionNormalCompactTask(BinaryRow.EMPTY_ROW, files()),
- new DataEvolutionBlobCompactTask(BinaryRow.EMPTY_ROW, files()));
- DataOutputSerializer out = new DataOutputSerializer(128);
- serializer.serializeList(tasks, out);
- List<DataEvolutionCompactTask> restored =
- serializer.deserializeList(
- serializer.getVersion(), new DataInputDeserializer(out.getCopyOfBuffer()));
- assertThat(restored).containsExactlyElementsOf(tasks);
- assertThat(((DataEvolutionNormalCompactTask) restored.get(0)).outputLevel()).isEqualTo(2);
- assertThat(((DataEvolutionNormalCompactTask) restored.get(1)).outputLevel()).isZero();
- }
-
- @Test
- void testVersionFourDefaultsToUnsortedOutput() throws Exception {
- DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer();
- DataOutputSerializer out = new DataOutputSerializer(128);
- List<DataFileMeta> files = files();
- serializeBinaryRow(BinaryRow.EMPTY_ROW, out);
- new DataFileMetaSerializer().serializeList(files, out);
- out.writeInt(DataEvolutionCompactTask.TaskType.NORMAL.code());
- out.writeInt(1);
- out.writeLong(0);
- out.writeLong(2);
- DataEvolutionNormalCompactTask restored =
- (DataEvolutionNormalCompactTask) serializer.deserialize(4, out.getCopyOfBuffer());
- assertThat(restored.outputLevel()).isZero();
- assertThat(restored.protectedRanges()).containsExactly(new Range(0, 2));
- assertThat(restored.compactBefore()).containsExactlyElementsOf(files);
- }
-
- private static List<DataFileMeta> files() {
- return Collections.singletonList(
- DataFileMeta.forAppend(
- "normal.parquet",
- 100,
- 3,
- EMPTY_STATS,
- 10,
- 10,
- 0,
- Collections.emptyList(),
- null,
- FileSource.APPEND,
- null,
- null,
- 0L,
- null));
- }
-}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java
index 22395b9d33..2464cbf892 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionNormalCompactTaskTest.java
@@ -81,57 +81,6 @@ public class DataEvolutionNormalCompactTaskTest extends TableTestBase {
.build();
}
- @ParameterizedTest
- @ValueSource(ints = {0, 2})
- public void testExplicitOutputLevelPreservesRowsAndRowIds(int outputLevel) throws Exception {
- write();
- FileStoreTable table = getTableDefault();
- List<DataFileMeta> input =
- table.store().newScan().plan().files().stream()
- .map(ManifestEntry::file)
- .collect(Collectors.toList());
- if (outputLevel == 0) {
- // Ordinary compaction must still reset an already clustered input to level zero.
- DataEvolutionNormalCompactTask seed =
- new DataEvolutionNormalCompactTask(
- table.store().newScan().plan().files().get(0).partition(),
- input,
- Collections.emptyList(),
- 2);
- try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
- commit.commit(Collections.singletonList(seed.doCompact(table, "seed-level")));
- }
- input = seed.compactAfter();
- }
- DataEvolutionNormalCompactTask task =
- outputLevel == 0
- ? new DataEvolutionNormalCompactTask(
- table.store().newScan().plan().files().get(0).partition(), input)
- : new DataEvolutionNormalCompactTask(
- table.store().newScan().plan().files().get(0).partition(),
- input,
- Collections.emptyList(),
- outputLevel);
- DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer();
- task =
- (DataEvolutionNormalCompactTask)
- serializer.deserialize(serializer.getVersion(), serializer.serialize(task));
- CommitMessage message = task.doCompact(table, "preserve-clustering-level");
- assertThat(task.compactAfter()).hasSize(1);
- assertThat(task.compactAfter().get(0).level()).isEqualTo(outputLevel);
- assertThat(task.compactAfter().get(0).nonNullRowIdRange())
- .isEqualTo(input.get(0).nonNullRowIdRange());
- try (BatchTableCommit commit = table.newBatchWriteBuilder().newCommit()) {
- commit.commit(Collections.singletonList(message));
- }
- List<InternalRow> rows = read(table);
- assertThat(rows).hasSize(ROW_COUNT);
- for (int i = 0; i < ROW_COUNT; i++) {
- assertThat(rows.get(i).getInt(1)).isEqualTo(i);
- assertThat(rows.get(i).getString(2).toString()).isEqualTo("f1_" + i);
- }
- }
-
@Test
public void testPropagateWriteColsSequencesAcrossCompactions() throws Exception {
write();
@@ -181,22 +130,6 @@ public class DataEvolutionNormalCompactTaskTest extends TableTestBase {
.isLessThan(partialUpdate.maxSequenceNumber());
}
- @Test
- public void testLevelZeroClusteringCompactionPreservesFieldSequences() throws Exception {
- write();
- DataFileMeta compacted =
- updateColumnsAndCompact(
- Collections.singletonList("f1"),
- 1,
- CoreOptions.GlobalIndexColumnUpdateAction.THROW_ERROR,
- true);
- assertThat(compacted.level()).isZero();
- assertThat(writeColSequence(compacted, getTableDefault().rowType().getField("f0").id()))
- .isLessThan(compacted.maxSequenceNumber());
- assertThat(writeColSequence(compacted, getTableDefault().rowType().getField("f1").id()))
- .isEqualTo(compacted.maxSequenceNumber());
- }
-
@Test
public void testOmitWriteColsSequencesUnlessUpdatesAreIgnored() throws Exception {
write();
@@ -807,22 +740,9 @@ public class DataEvolutionNormalCompactTaskTest extends TableTestBase {
int updateRound,
CoreOptions.GlobalIndexColumnUpdateAction updateAction)
throws Exception {
- return updateColumnsAndCompact(columns, updateRound, updateAction, false);
- }
-
- private DataFileMeta updateColumnsAndCompact(
- List<String> columns,
- int updateRound,
- CoreOptions.GlobalIndexColumnUpdateAction updateAction,
- boolean incrementalClustering)
- throws Exception {
Map<String, String> writeOptions = new HashMap<>();
writeOptions.put(
CoreOptions.GLOBAL_INDEX_COLUMN_UPDATE_ACTION.key(), updateAction.toString());
- if (incrementalClustering) {
- writeOptions.put(CoreOptions.CLUSTERING_INCREMENTAL.key(), "true");
- writeOptions.put(CoreOptions.CLUSTERING_COLUMNS.key(), "f0");
- }
FileStoreTable table = getTableDefault().copy(writeOptions);
List<String> writeColumns = new ArrayList<>();
writeColumns.add("dt");
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitRewriterTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitRewriterTest.java
deleted file mode 100644
index 8880bb9887..0000000000
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitRewriterTest.java
+++ /dev/null
@@ -1,453 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.operation;
-
-import org.apache.paimon.AppendOnlyFileStore;
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.catalog.RenamingSnapshotCommit;
-import org.apache.paimon.catalog.SnapshotCommit;
-import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.index.GlobalIndexMeta;
-import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.manifest.FileKind;
-import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.manifest.ManifestCommittable;
-import org.apache.paimon.manifest.ManifestEntry;
-import org.apache.paimon.manifest.SimpleFileEntry;
-import org.apache.paimon.operation.commit.CommitChanges;
-import org.apache.paimon.operation.commit.CommitChangesRewriter;
-import org.apache.paimon.operation.commit.ConflictDetection;
-import org.apache.paimon.partition.PartitionStatistics;
-import org.apache.paimon.schema.FileSystemSchemaManager;
-import org.apache.paimon.schema.Schema;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.TableTestBase;
-import org.apache.paimon.table.sink.BatchTableWrite;
-import org.apache.paimon.table.sink.CommitPreCallback;
-import org.apache.paimon.types.DataTypes;
-import org.apache.paimon.utils.Range;
-
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Commit-attempt isolation and index payload retention for injected index rewriters. */
-class FileStoreCommitRewriterTest extends TableTestBase {
-
- @Test
- void testRetryCreatesAttemptFromLatestSnapshotAndFinalRowIds() throws Exception {
- FileStoreTable table = createTable();
- write(table, GenericRow.of(1));
- ManifestCommittable pending = prepareAppend(table, 10, 20);
- List<Long> snapshots = new ArrayList<>();
- List<Long> firstRowIds = new ArrayList<>();
- List<Path> outputs = new ArrayList<>();
- List<CommitChangesRewriter.Attempt> attempts = new ArrayList<>();
- AtomicInteger publications = new AtomicInteger();
- SnapshotCommit publisher =
- new DelegatingSnapshotCommit(table) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- if (publications.incrementAndGet() == 1) {
- write(table, GenericRow.of(2), GenericRow.of(3));
- assertThat(delegate.commit(baseUuid, snapshot, branch, statistics))
- .isFalse();
- return false;
- }
- return delegate.commit(baseUuid, snapshot, branch, statistics);
- }
- };
- CommitChangesRewriter rewriter =
- new CommitChangesRewriter() {
- @Override
- public Attempt createAttempt(Snapshot.CommitKind kind, CommitChanges changes) {
- assertThat(kind).isEqualTo(Snapshot.CommitKind.APPEND);
- assertThat(changes.tableFiles).hasSize(1);
- assertThat(changes.tableFiles.get(0).file().firstRowId()).isNull();
- assertReadOnly(changes.tableFiles);
- assertReadOnly(changes.changelogFiles);
- assertReadOnly(changes.indexFiles);
- AtomicInteger calls = new AtomicInteger();
- Attempt attempt =
- (snapshotId, assigned, indexes, ioManager) -> {
- assertThat(calls.incrementAndGet()).isEqualTo(1);
- snapshots.add(snapshotId);
- assertReadOnly(assigned);
- assertReadOnly(indexes);
- firstRowIds.add(assigned.get(0).file().firstRowId());
- return writeIndex(table, assigned.get(0), outputs);
- };
- attempts.add(attempt);
- return attempt;
- }
- };
- try (FileStoreCommit commit =
- newCommit(table, publisher, Collections.emptyList())
- .withCommitChangesRewriter(rewriter)) {
- commit.commit(pending, false);
- }
-
- assertThat(publications).hasValue(2);
- assertThat(snapshots).containsExactly(2L, 3L);
- assertThat(firstRowIds).containsExactly(1L, 3L);
- assertThat(attempts).hasSize(2);
- assertThat(attempts.get(0)).isNotSameAs(attempts.get(1));
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(5L);
- assertThat(read(table))
- .extracting(row -> row.getInt(0))
- .containsExactlyInAnyOrder(1, 2, 3, 10, 20);
- List<IndexManifestEntry> indexes = table.store().newIndexFileHandler().scanEntries();
- assertThat(indexes).hasSize(1);
- assertThat(indexes.get(0).indexFile().fileName()).isEqualTo(outputs.get(1).getName());
- assertThat(indexes.get(0).indexFile().globalIndexMeta().rowRange())
- .isEqualTo(new Range(3, 4));
- // A false publication response is uncertain, even when this test's publisher knows it lost.
- for (Path output : outputs) {
- assertThat(table.fileIO().exists(output)).isTrue();
- }
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testPreparationFailureRetainsOrphanIndex(boolean fatalError) throws Exception {
- FileStoreTable table = createTable();
- ManifestCommittable pending = prepareAppend(table, 10);
- List<Path> outputs = new ArrayList<>();
- Throwable failure =
- fatalError
- ? new AssertionError("rewrite failed")
- : new IOException("rewrite failed");
- AtomicInteger publications = new AtomicInteger();
- CommitChangesRewriter rewriter =
- new CommitChangesRewriter() {
- @Override
- public Attempt createAttempt(Snapshot.CommitKind kind, CommitChanges changes) {
- return (snapshotId, assigned, indexes, ioManager) -> {
- writeIndex(table, assigned.get(0), outputs);
- if (fatalError) {
- throw (AssertionError) failure;
- }
- throw (IOException) failure;
- };
- }
- };
- try (FileStoreCommit commit =
- newCommit(table, countingPublisher(table, publications), Collections.emptyList())
- .withCommitChangesRewriter(rewriter)) {
- assertThatThrownBy(() -> commit.commit(pending, false))
- .hasMessageContaining("preparing snapshot #1")
- .hasCauseReference(failure);
- }
- assertThat(outputs).hasSize(1);
- assertThat(table.fileIO().readFileUtf8(outputs.get(0))).isEqualTo("index payload");
- assertThat(publications).hasValue(0);
- assertThat(table.latestSnapshot()).isEmpty();
- }
-
- @Test
- void testManifestPreparationFailureRetainsOrphanIndex() throws Exception {
- FileStoreTable table = createTable();
- ManifestCommittable pending = prepareAppend(table, 10);
- List<Path> outputs = new ArrayList<>();
- AtomicInteger publications = new AtomicInteger();
- CommitChangesRewriter rewriter =
- new CommitChangesRewriter() {
- @Override
- public Attempt createAttempt(Snapshot.CommitKind kind, CommitChanges changes) {
- return (snapshotId, assigned, indexes, ioManager) -> {
- IndexManifestEntry index =
- writeIndex(table, assigned.get(0), outputs).get(0);
- // Fail after the extension returns, while merging the index manifest.
- return Collections.singletonList(index.toDeleteEntry());
- };
- }
- };
- try (FileStoreCommit commit =
- newCommit(table, countingPublisher(table, publications), Collections.emptyList())
- .withCommitChangesRewriter(rewriter)) {
- assertThatThrownBy(() -> commit.commit(pending, false))
- .hasMessageContaining("preparing snapshot #1")
- .hasRootCauseInstanceOf(IllegalStateException.class)
- .hasStackTraceContaining("which does not exist.");
- }
- assertThat(outputs).hasSize(1);
- assertThat(table.fileIO().readFileUtf8(outputs.get(0))).isEqualTo("index payload");
- assertThat(publications).hasValue(0);
- assertThat(table.latestSnapshot()).isEmpty();
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testPreCallbackFailureRetainsOrphanIndex(boolean fatalError) throws Exception {
- FileStoreTable table = createTable();
- ManifestCommittable pending = prepareAppend(table, 10);
- List<Path> outputs = new ArrayList<>();
- Throwable failure =
- fatalError
- ? new AssertionError("validation failed")
- : new IllegalStateException("validation failed");
- AtomicInteger publications = new AtomicInteger();
- CommitPreCallback callback =
- new CommitPreCallback() {
- @Override
- public void call(
- List<SimpleFileEntry> base,
- List<ManifestEntry> delta,
- List<IndexManifestEntry> indexes,
- Snapshot snapshot) {
- assertThat(indexes).hasSize(1);
- if (fatalError) {
- throw (AssertionError) failure;
- }
- throw (IllegalStateException) failure;
- }
-
- @Override
- public void close() {}
- };
- try (FileStoreCommit commit =
- newCommit(
- table,
- countingPublisher(table, publications),
- Collections.singletonList(callback))
- .withCommitChangesRewriter(indexRewriter(table, outputs))) {
- assertThatThrownBy(() -> commit.commit(pending, false)).isSameAs(failure);
- }
- assertThat(outputs).hasSize(1);
- assertThat(table.fileIO().readFileUtf8(outputs.get(0))).isEqualTo("index payload");
- assertThat(publications).hasValue(0);
- assertThat(table.latestSnapshot()).isEmpty();
- }
-
- @ParameterizedTest
- @ValueSource(booleans = {false, true})
- void testUncertainPublicationRetainsVisibleIndex(boolean publicationThrows) throws Exception {
- FileStoreTable table = createTable();
- ManifestCommittable pending = prepareAppend(table, 10, 20);
- List<Path> outputs = new ArrayList<>();
- AtomicInteger attempts = new AtomicInteger();
- AtomicInteger publications = new AtomicInteger();
- SnapshotCommit publisher =
- new DelegatingSnapshotCommit(table) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- publications.incrementAndGet();
- assertThat(delegate.commit(baseUuid, snapshot, branch, statistics))
- .isTrue();
- if (publicationThrows) {
- throw new IOException("publication response lost");
- }
- return false;
- }
- };
- CommitChangesRewriter rewriter =
- new CommitChangesRewriter() {
- @Override
- public Attempt createAttempt(Snapshot.CommitKind kind, CommitChanges changes) {
- attempts.incrementAndGet();
- return (snapshotId, assigned, indexes, ioManager) ->
- writeIndex(table, assigned.get(0), outputs);
- }
- };
- try (FileStoreCommit commit =
- newCommit(table, publisher, Collections.emptyList())
- .withCommitChangesRewriter(rewriter)) {
- commit.commit(pending, false);
- }
-
- assertThat(publications).hasValue(1);
- assertThat(attempts).hasValue(1);
- // Reconciliation must happen before constructing another attempt or rewriting another
- // index.
- assertThat(outputs).hasSize(1);
- assertThat(table.fileIO().readFileUtf8(outputs.get(0))).isEqualTo("index payload");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(1L);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(2L);
- assertThat(read(table)).extracting(row -> row.getInt(0)).containsExactlyInAnyOrder(10, 20);
- List<IndexManifestEntry> indexes = table.store().newIndexFileHandler().scanEntries();
- assertThat(indexes).hasSize(1);
- assertThat(indexes.get(0).indexFile().fileName()).isEqualTo(outputs.get(0).getName());
- assertThat(indexes.get(0).indexFile().globalIndexMeta().rowRange())
- .isEqualTo(new Range(0, 1));
- }
-
- private FileStoreTable createTable() throws Exception {
- catalog.createTable(
- identifier(),
- Schema.newBuilder()
- .column("id", DataTypes.INT())
- .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
- .option(CoreOptions.WRITE_ONLY.key(), "true")
- .option(CoreOptions.COMMIT_MIN_RETRY_WAIT.key(), "0ms")
- .option(CoreOptions.COMMIT_MAX_RETRY_WAIT.key(), "0ms")
- .build(),
- false);
- return getTableDefault();
- }
-
- private ManifestCommittable prepareAppend(FileStoreTable table, int... values)
- throws Exception {
- ManifestCommittable pending = new ManifestCommittable(17L);
- try (BatchTableWrite write = table.newBatchWriteBuilder().newWrite()) {
- for (int value : values) {
- write.write(GenericRow.of(value));
- }
- write.prepareCommit().forEach(pending::addFileCommittable);
- }
- return pending;
- }
-
- private CommitChangesRewriter indexRewriter(FileStoreTable table, List<Path> outputs) {
- return new CommitChangesRewriter() {
- @Override
- public Attempt createAttempt(Snapshot.CommitKind kind, CommitChanges changes) {
- return (snapshotId, assigned, indexes, ioManager) ->
- writeIndex(table, assigned.get(0), outputs);
- }
- };
- }
-
- private List<IndexManifestEntry> writeIndex(
- FileStoreTable table, ManifestEntry assigned, List<Path> outputs) throws IOException {
- Path path = table.store().pathFactory().globalIndexFileFactory().newPath();
- outputs.add(path);
- table.fileIO().writeFile(path, "index payload", false);
- IndexFileMeta index =
- new IndexFileMeta(
- "test-index",
- path.getName(),
- 13,
- assigned.file().rowCount(),
- new GlobalIndexMeta(
- assigned.file().nonNullFirstRowId(),
- assigned.file().nonNullRowIdRange().to,
- 0,
- null,
- null),
- null);
- return Collections.singletonList(
- new IndexManifestEntry(
- FileKind.ADD, assigned.partition(), assigned.bucket(), index));
- }
-
- private SnapshotCommit countingPublisher(FileStoreTable table, AtomicInteger publications) {
- return new DelegatingSnapshotCommit(table) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- publications.incrementAndGet();
- return delegate.commit(baseUuid, snapshot, branch, statistics);
- }
- };
- }
-
- private FileStoreCommitImpl newCommit(
- FileStoreTable table, SnapshotCommit publisher, List<CommitPreCallback> callbacks) {
- AppendOnlyFileStore store = (AppendOnlyFileStore) table.store();
- CoreOptions options = store.options();
- return new FileStoreCommitImpl(
- publisher,
- table.fileIO(),
- new FileSystemSchemaManager(table.fileIO(), table.location()),
- "rewriter-test",
- "rewriter-test-user",
- store.partitionType(),
- options,
- store.pathFactory(),
- store.snapshotManager(),
- store.manifestFileFactory(),
- store.manifestListFactory(),
- store.indexManifestFileFactory(),
- store::newScan,
- store.newStatsFileHandler(),
- store.bucketMode(),
- callbacks,
- Collections.emptyList(),
- scanner ->
- ConflictDetection.create(
- "rewriter-test",
- "rewriter-test-user",
- store.partitionType(),
- store.pathFactory(),
- store.newKeyComparator(),
- store.bucketMode(),
- options.deletionVectorsEnabled(),
- false,
- false,
- options.pkClusteringOverride(),
- store.newIndexFileHandler(),
- store.snapshotManager(),
- scanner),
- null);
- }
-
- private static void assertReadOnly(List<?> entries) {
- assertThatThrownBy(entries::clear).isInstanceOf(UnsupportedOperationException.class);
- }
-
- private static class DelegatingSnapshotCommit extends SnapshotCommit {
-
- protected final SnapshotCommit delegate;
-
- private DelegatingSnapshotCommit(FileStoreTable table) {
- super(Lock.empty());
- delegate = new RenamingSnapshotCommit(table.snapshotManager(), Lock.empty());
- }
-
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- return delegate.commit(baseUuid, snapshot, branch, statistics);
- }
-
- @Override
- protected void closeImpl() throws Exception {
- delegate.close();
- }
- }
-}
diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
index 110d81faaa..99fd1f6532 100644
--- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java
@@ -24,7 +24,6 @@ import org.apache.paimon.Snapshot;
import org.apache.paimon.TestAppendFileStore;
import org.apache.paimon.TestFileStore;
import org.apache.paimon.TestKeyValueGenerator;
-import org.apache.paimon.append.dataevolution.MaterializeDeletionVectorsCommitChangesRewriter;
import org.apache.paimon.catalog.RenamingSnapshotCommit;
import org.apache.paimon.catalog.SnapshotCommit;
import org.apache.paimon.data.BinaryRow;
@@ -78,7 +77,6 @@ import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.FailingFileIO;
import org.apache.paimon.utils.JsonSerdeUtil;
import org.apache.paimon.utils.Pair;
-import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.SnapshotManager;
import org.apache.paimon.utils.TraceableFileIO;
@@ -1001,60 +999,30 @@ public class FileStoreCommitTest {
TestFileStore store = createStore(false, 2);
BinaryRow partition = gen.getPartition(gen.next());
IndexManifestEntry materializedBucketDelete =
- globalIndexDeleteEntry(partition, 0, "materialized-bucket-index", new Range(0, 0));
- IndexManifestEntry disjointRangeDelete =
- globalIndexDeleteEntry(partition, 0, "disjoint-range-index", new Range(1, 2));
+ globalIndexDeleteEntry(partition, 0, "materialized-bucket-index");
IndexManifestEntry otherBucketDelete =
- globalIndexDeleteEntry(partition, 1, "other-bucket-index", new Range(0, 0));
- IndexManifestEntry nonGlobalDelete =
- new IndexManifestEntry(
- FileKind.DELETE,
- partition,
- 0,
- new IndexFileMeta(HASH_INDEX, "non-global-index", 1, 1, null, null, null));
+ globalIndexDeleteEntry(partition, 1, "other-bucket-index");
CommitChanges changes =
new CommitChanges(
Collections.emptyList(),
Collections.emptyList(),
- Arrays.asList(
- materializedBucketDelete,
- disjointRangeDelete,
- otherBucketDelete,
- nonGlobalDelete));
+ Arrays.asList(materializedBucketDelete, otherBucketDelete));
- CommitMessage message =
- new CommitMessageImpl(
- partition,
- 0,
- 2,
- DataIncrement.emptyIncrement(),
- new CompactIncrement(
- Collections.singletonList(
- addFile(partition, 0, 2, 0).file().assignFirstRowId(0)),
- Collections.emptyList(),
- Collections.emptyList()));
- MaterializeDeletionVectorsCommitChangesRewriter rewriter =
- new MaterializeDeletionVectorsCommitChangesRewriter(
- FileStoreTableFactory.create(store.fileIO(), store.options().path()));
- CommitChanges refreshed =
- rewriter.compactChangesProvider(changes, Collections.singletonList(message))
- .provide(null);
+ try (FileStoreCommitImpl commit = store.newCommit()) {
+ CommitChanges refreshed =
+ commit.compactChangesProvider(
+ changes, Collections.singleton(Pair.of(partition, 0)))
+ .provide(null);
- // Global-index removals from earlier preparation are replaced for the entire bucket,
- // even if their ranges do not overlap the materialized input.
- assertThat(refreshed.indexFiles).containsExactly(otherBucketDelete, nonGlobalDelete);
+ assertThat(refreshed.indexFiles).containsExactly(otherBucketDelete);
+ }
}
private static IndexManifestEntry globalIndexDeleteEntry(
- BinaryRow partition, int bucket, String fileName, Range range) {
+ BinaryRow partition, int bucket, String fileName) {
IndexFileMeta file =
new IndexFileMeta(
- "btree",
- fileName,
- 1,
- range.count(),
- new GlobalIndexMeta(range.from, range.to, 0, null, null),
- null);
+ "btree", fileName, 1, 1, new GlobalIndexMeta(0, 0, 0, null, null), null);
return new IndexManifestEntry(FileKind.DELETE, partition, bucket, file);
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
index bcd43a5565..54a7088950 100644
--- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaValidationTest.java
@@ -1563,93 +1563,6 @@ class SchemaValidationTest {
assertThatCode(() -> validateTableSchema(schema)).doesNotThrowAnyException();
}
- @Test
- void testDataEvolutionIncrementalClustering() {
- List<DataField> fields =
- Arrays.asList(
- new DataField(0, "id", DataTypes.INT()),
- new DataField(1, "value", DataTypes.STRING()));
- TableSchema schema =
- new TableSchema(
- 1,
- fields,
- 1,
- emptyList(),
- emptyList(),
- dataEvolutionClusteringOptions(),
- "");
-
- assertThatCode(() -> validateTableSchema(schema)).doesNotThrowAnyException();
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"blob-descriptor-field", "blob-view-field"})
- void testDataEvolutionIncrementalClusteringAllowsInlineBlob(String inlineOption) {
- Map<String, String> options = dataEvolutionClusteringOptions();
- options.put(inlineOption, "payload");
- List<DataField> fields =
- Arrays.asList(
- new DataField(0, "id", DataTypes.INT()),
- new DataField(1, "payload", DataTypes.BLOB()));
- TableSchema schema = new TableSchema(1, fields, 1, emptyList(), emptyList(), options, "");
-
- assertThatCode(() -> validateTableSchema(schema)).doesNotThrowAnyException();
- }
-
- @Test
- void testDataEvolutionIncrementalClusteringRejectsDedicatedBlob() {
- for (DataType blobType :
- Arrays.asList(
- DataTypes.BLOB(),
- DataTypes.ARRAY(DataTypes.BLOB()),
- DataTypes.MAP(DataTypes.STRING(), DataTypes.BLOB()))) {
- List<DataField> fields =
- Arrays.asList(
- new DataField(0, "id", DataTypes.INT()),
- new DataField(1, "payload", blobType));
- TableSchema schema =
- new TableSchema(
- 1,
- fields,
- 1,
- emptyList(),
- emptyList(),
- dataEvolutionClusteringOptions(),
- "");
-
- assertThatThrownBy(() -> validateTableSchema(schema))
- .hasMessage(
- "Data evolution incremental clustering does not support dedicated BLOB files.");
- }
- }
-
- @Test
- void testDataEvolutionIncrementalClusteringVectorStorage() {
- Map<String, String> options = dataEvolutionClusteringOptions();
- List<DataField> fields =
- Arrays.asList(
- new DataField(0, "id", DataTypes.INT()),
- new DataField(1, "embedding", DataTypes.VECTOR(3, DataTypes.FLOAT())));
- TableSchema schema = new TableSchema(1, fields, 1, emptyList(), emptyList(), options, "");
-
- assertThatCode(() -> validateTableSchema(schema)).doesNotThrowAnyException();
-
- options.put(VECTOR_FILE_FORMAT.key(), "json");
- assertThatThrownBy(() -> validateTableSchema(schema))
- .hasMessage(
- "Data evolution incremental clustering does not support dedicated VECTOR files.");
- }
-
- private Map<String, String> dataEvolutionClusteringOptions() {
- Map<String, String> options = new HashMap<>();
- options.put(BUCKET.key(), "-1");
- options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
- options.put(DATA_EVOLUTION_ENABLED.key(), "true");
- options.put(CoreOptions.CLUSTERING_INCREMENTAL.key(), "true");
- options.put(CoreOptions.CLUSTERING_COLUMNS.key(), "id");
- return options;
- }
-
@Test
public void testFileIndexColumns() {
List<String> keys =
diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java
index 2620f51746..1ec1343ab1 100644
--- a/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/table/DataEvolutionDeletionVectorTest.java
@@ -26,9 +26,6 @@ import org.apache.paimon.append.dataevolution.DataEvolutionCompactTaskSerializer
import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation;
import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorMaterializeCoordinator;
import org.apache.paimon.append.dataevolution.DataEvolutionRowIdReassigner;
-import org.apache.paimon.append.dataevolution.MaterializeDeletionVectorsCommitChangesRewriter;
-import org.apache.paimon.catalog.RenamingSnapshotCommit;
-import org.apache.paimon.catalog.SnapshotCommit;
import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryString;
import org.apache.paimon.data.BlobData;
@@ -42,9 +39,7 @@ import org.apache.paimon.format.blob.BlobFileFormat;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.SeekableInputStream;
import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.globalindex.DataEvolutionGlobalIndexScanner;
import org.apache.paimon.globalindex.IndexedSplit;
-import org.apache.paimon.globalindex.sorted.SortedGlobalIndexTestUtils;
import org.apache.paimon.index.DeletionVectorMeta;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
@@ -55,11 +50,7 @@ import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFileMeta;
-import org.apache.paimon.operation.Lock;
import org.apache.paimon.partition.PartitionPredicate;
-import org.apache.paimon.partition.PartitionStatistics;
-import org.apache.paimon.predicate.Predicate;
-import org.apache.paimon.predicate.PredicateBuilder;
import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.schema.Schema;
import org.apache.paimon.schema.SchemaChange;
@@ -80,7 +71,6 @@ import org.apache.paimon.types.DataTypes;
import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Range;
import org.apache.paimon.utils.RangeHelper;
-import org.apache.paimon.utils.SnapshotManager;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -97,7 +87,6 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.stream.Collectors;
import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
@@ -906,125 +895,6 @@ public class DataEvolutionDeletionVectorTest extends DataEvolutionTestBase {
.doesNotContainAnyElementsOf(oldAnchorFiles);
}
- @ParameterizedTest
- @ValueSource(strings = {"btree", "bitmap"})
- public void testMaterializeRetryRefreshesConcurrentScalarIndexes(String indexType)
- throws Exception {
- Map<String, String> retryOptions = new HashMap<>();
- retryOptions.put(CoreOptions.COMMIT_MIN_RETRY_WAIT.key(), "0ms");
- retryOptions.put(CoreOptions.COMMIT_MAX_RETRY_WAIT.key(), "0ms");
- FileStoreTable original =
- createPartitionedReassignTable("materialize_retry_index_table", false)
- .copy(retryOptions);
- writePartitionRows(original, "a", 0, 1, 2);
- writePartitionRows(original, "a", 3, 4, 5);
- commitDeletionVectors(
- original,
- partition(original, "a"),
- Collections.singletonList(new DvSpec(new Range(0, 2), 1, 2)));
- Snapshot planningSnapshot = original.latestSnapshot().get();
- List<CommitMessage> materialized =
- prepareMaterializeDeletionVectors(original, planningSnapshot, null);
- AtomicInteger attempts = new AtomicInteger();
- List<String> untouchedIndexes = new ArrayList<>();
- CatalogEnvironment environment =
- new CatalogEnvironment(null, null, null, null, null, null, false, false) {
- @Override
- public SnapshotCommit snapshotCommit(
- SnapshotManager manager, CoreOptions options, String commitUser) {
- SnapshotCommit delegate = new RenamingSnapshotCommit(manager, Lock.empty());
- return new SnapshotCommit(Lock.empty()) {
- @Override
- protected boolean commitImpl(
- String baseUuid,
- Snapshot snapshot,
- String branch,
- List<PartitionStatistics> statistics)
- throws Exception {
- int attempt = attempts.incrementAndGet();
- if (attempt == 1) {
- List<CommitMessage> indexes = new ArrayList<>();
- for (DataSplit split :
- original.newSnapshotReader()
- .withSnapshot(planningSnapshot)
- .read()
- .dataSplits()) {
- for (DataFileMeta file : split.dataFiles()) {
- indexes.addAll(
- SortedGlobalIndexTestUtils.buildIndex(
- original,
- indexType,
- "f0",
- split.filterDataFile(file::equals)
- .get(),
- planningSnapshot.id()));
- }
- }
- DataEvolutionDeletionVectorTest.commit(original, indexes);
- assertThat(liveGlobalIndexFileNames(original)).hasSize(2);
- for (IndexManifestEntry entry :
- original.store().newIndexFileHandler().scanEntries()) {
- GlobalIndexMeta meta = entry.indexFile().globalIndexMeta();
- if (meta != null
- && meta.rowRange().equals(new Range(3, 5))) {
- untouchedIndexes.add(entry.indexFile().fileName());
- }
- }
- assertThat(untouchedIndexes).hasSize(1);
- // Move the next available row ID after the first candidate has
- // already assigned its output. The retry must assign it again.
- writePartitionRows(original, "a", 6);
- }
- boolean committed =
- delegate.commit(baseUuid, snapshot, branch, statistics);
- assertThat(committed).isEqualTo(attempt == 2);
- return committed;
- }
-
- @Override
- protected void closeImpl() throws Exception {
- delegate.close();
- }
- };
- }
- };
- FileStoreTable committingTable =
- FileStoreTableFactory.create(
- original.fileIO(), original.location(), original.schema(), environment);
- commitMaterializeDeletionVectors(
- committingTable, planningSnapshot, materialized, "test-materialize-index-retry");
-
- assertThat(attempts).hasValue(2);
- assertThat(original.latestSnapshot().get().id()).isEqualTo(planningSnapshot.id() + 3);
- assertThat(original.latestSnapshot().get().nextRowId()).isEqualTo(8L);
- assertThat(readPartitionedRowsWithRowIds(original))
- .containsExactly("a|3|3", "a|4|4", "a|5|5", "a|6|6", "a|0|7");
- assertThat(readPartitionedRows(original))
- .containsExactlyInAnyOrder(
- "a|0|base-0", "a|3|base-3", "a|4|base-4", "a|5|base-5", "a|6|base-6");
- assertThat(liveDeletionVectorDataFileNames(original)).isEmpty();
- assertThat(liveGlobalIndexFileNames(original)).containsExactlyElementsOf(untouchedIndexes);
- assertThat(
- original.store().newIndexFileHandler().scanEntries().stream()
- .filter(entry -> entry.indexFile().globalIndexMeta() != null)
- .map(entry -> entry.indexFile().globalIndexMeta().rowRange()))
- .containsExactly(new Range(3, 5));
- Predicate predicate = new PredicateBuilder(original.rowType()).equal(1, 4);
- try (DataEvolutionGlobalIndexScanner scanner =
- DataEvolutionGlobalIndexScanner.create(
- original, PartitionPredicate.ALWAYS_TRUE, predicate)
- .get()) {
- assertThat(scanner.scan(predicate).get().results().toRangeList())
- .containsExactly(new Range(4, 4));
- assertThat(
- scanner.scan(new PredicateBuilder(original.rowType()).equal(1, 1))
- .get()
- .results()
- .toRangeList())
- .isEmpty();
- }
- }
-
@Test
public void testStaleMaterializeRejectedAfterConcurrentUpdate() throws Exception {
createTableDefault();
@@ -1622,10 +1492,7 @@ public class DataEvolutionDeletionVectorTest extends DataEvolutionTestBase {
((CommitMessageImpl) message)
.withCheckFromSnapshot(snapshot.id()))
.collect(Collectors.toList());
- commit.materializeDvRowIdCheck()
- .withCommitChangesRewriter(
- new MaterializeDeletionVectorsCommitChangesRewriter(table))
- .commit(checkedMessages);
+ commit.materializeDvRowIdCheck().commit(checkedMessages);
}
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/DataEvolutionUtilsTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/DataEvolutionUtilsTest.java
index 066bace5a6..028c12ee44 100644
--- a/paimon-core/src/test/java/org/apache/paimon/utils/DataEvolutionUtilsTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/utils/DataEvolutionUtilsTest.java
@@ -426,106 +426,6 @@ public class DataEvolutionUtilsTest {
.isSameAs(smallerFileName);
}
- @Test
- public void testMaterializedRowIdRangesExcludePreservedCompactionOutputs() {
- List<DataFileMeta> before =
- Arrays.asList(
- rangedDataFile("first.parquet", 0L, 5L),
- rangedDataFile("first-update.parquet", 0L, 5L),
- rangedDataFile("second.parquet", 8L, 3L));
- List<DataFileMeta> after =
- Arrays.asList(
- rangedDataFile("retained-first.parquet", 1L, 2L),
- rangedDataFile("retained-second.parquet", 3L, 1L),
- rangedDataFile("fresh.parquet", null, 4L),
- rangedDataFile("blob-output.blob", 0L, 11L),
- rangedDataFile("vector-output.vector.lance", 0L, 11L));
-
- assertThat(DataEvolutionUtils.materializedRowIdRanges(before, after))
- .containsExactly(new Range(0, 0), new Range(4, 4), new Range(8, 10));
- assertThat(DataEvolutionUtils.materializedRowIdRanges(before, before)).isEmpty();
- assertThat(DataEvolutionUtils.materializedRowIdRanges(before, Collections.emptyList()))
- .containsExactly(new Range(0, 4), new Range(8, 10));
- assertThat(
- DataEvolutionUtils.materializedRowIdRanges(
- Arrays.asList(
- rangedDataFile("input.blob", 0L, 5L),
- rangedDataFile("input.vector.lance", 0L, 5L)),
- Collections.emptyList()))
- .isEmpty();
- }
-
- @Test
- public void testMaterializedRowIdRangesWalkInterleavedGroupsAndOutputs() {
- List<DataFileMeta> before =
- Arrays.asList(
- rangedDataFile("last.parquet", 20L, 5L),
- rangedDataFile("first.parquet", 0L, 5L),
- rangedDataFile("middle.parquet", 10L, 5L),
- rangedDataFile("adjacent.parquet", 5L, 2L));
- List<DataFileMeta> after =
- Arrays.asList(
- rangedDataFile("middle-cut.parquet", 12L, 1L),
- rangedDataFile("spanning-gaps.parquet", 3L, 9L),
- rangedDataFile("outside.parquet", 30L, 3L),
- rangedDataFile("prefix.parquet", 0L, 1L),
- rangedDataFile("suffix.parquet", 24L, 3L));
-
- assertThat(DataEvolutionUtils.materializedRowIdRanges(before, after))
- .containsExactly(new Range(1, 2), new Range(13, 14), new Range(20, 23));
- assertThat(DataEvolutionUtils.materializedRowIdRanges(before, Collections.emptyList()))
- .containsExactly(new Range(0, 6), new Range(10, 14), new Range(20, 24));
- assertThat(
- DataEvolutionUtils.materializedRowIdRanges(
- before,
- Collections.singletonList(rangedDataFile("all.parquet", 0L, 100L))))
- .isEmpty();
- assertThat(DataEvolutionUtils.materializedRowIdRanges(Collections.emptyList(), after))
- .isEmpty();
- }
-
- @Test
- public void testMaterializedRowIdRangesHandleMaximumRowIdWithoutOverflow() {
- List<DataFileMeta> before =
- Arrays.asList(
- rangedDataFile("last.parquet", Long.MAX_VALUE - 2, 3L),
- rangedDataFile("last-update.parquet", Long.MAX_VALUE - 2, 3L),
- rangedDataFile("adjacent.parquet", Long.MAX_VALUE - 3, 1L),
- rangedDataFile("first.parquet", 0L, 1L));
- assertThat(
- DataEvolutionUtils.materializedRowIdRanges(
- before,
- Collections.singletonList(
- rangedDataFile("suffix.parquet", Long.MAX_VALUE - 1, 2L))))
- .containsExactly(
- new Range(0, 0), new Range(Long.MAX_VALUE - 3, Long.MAX_VALUE - 2));
- assertThat(
- DataEvolutionUtils.materializedRowIdRanges(
- before,
- Collections.singletonList(
- rangedDataFile("prefix.parquet", Long.MAX_VALUE - 3, 3L))))
- .containsExactly(new Range(0, 0), new Range(Long.MAX_VALUE, Long.MAX_VALUE));
- assertThat(DataEvolutionUtils.materializedRowIdRanges(before, before)).isEmpty();
- }
-
- private static DataFileMeta rangedDataFile(String fileName, Long firstRowId, long rowCount) {
- return DataFileMeta.forAppend(
- fileName,
- 1L,
- rowCount,
- SimpleStats.EMPTY_STATS,
- 1L,
- 1L,
- 1L,
- Collections.emptyList(),
- null,
- null,
- null,
- null,
- firstRowId,
- null);
- }
-
private static DataFileMeta dataFile(String fileName, long maxSequenceNumber) {
return dataFile(fileName, maxSequenceNumber, Collections.emptyList());
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowData.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowData.java
index 8db5c7d9e0..2016ab4bf4 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowData.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkRowData.java
@@ -24,12 +24,6 @@ import org.apache.paimon.data.InternalArray;
import org.apache.paimon.data.InternalMap;
import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.Timestamp;
-import org.apache.paimon.types.ArrayType;
-import org.apache.paimon.types.DataType;
-import org.apache.paimon.types.MapType;
-import org.apache.paimon.types.MultisetType;
-import org.apache.paimon.types.RowType;
-import org.apache.paimon.types.VectorType;
import org.apache.flink.table.data.ArrayData;
import org.apache.flink.table.data.DecimalData;
@@ -42,23 +36,15 @@ import org.apache.flink.types.RowKind;
import org.apache.flink.types.variant.BinaryVariant;
import org.apache.flink.types.variant.Variant;
-import javax.annotation.Nullable;
-
import static org.apache.paimon.flink.FlinkRowWrapper.fromFlinkRowKind;
/** Convert to Flink row data. */
public class FlinkRowData implements RowData {
protected InternalRow row;
- @Nullable private final RowType rowType;
public FlinkRowData(InternalRow row) {
- this(row, null);
- }
-
- public FlinkRowData(InternalRow row, @Nullable RowType rowType) {
this.row = row;
- this.rowType = rowType;
}
public FlinkRowData replace(InternalRow row) {
@@ -148,39 +134,17 @@ public class FlinkRowData implements RowData {
@Override
public ArrayData getArray(int pos) {
- // VECTOR and ARRAY have different binary layouts even though Flink exposes both as ARRAY.
- DataType type = fieldType(pos);
- return new FlinkArrayData(
- type instanceof VectorType ? row.getVector(pos) : row.getArray(pos),
- arrayElementType(type));
+ return new FlinkArrayData(row.getArray(pos));
}
@Override
public MapData getMap(int pos) {
- return new FlinkMapData(row.getMap(pos), fieldType(pos));
+ return new FlinkMapData(row.getMap(pos));
}
@Override
public RowData getRow(int pos, int numFields) {
- DataType type = fieldType(pos);
- return new FlinkRowData(
- row.getRow(pos, numFields), type instanceof RowType ? (RowType) type : null);
- }
-
- @Nullable
- private DataType fieldType(int pos) {
- return rowType == null ? null : rowType.getTypeAt(pos);
- }
-
- @Nullable
- private static DataType arrayElementType(@Nullable DataType type) {
- if (type instanceof ArrayType) {
- return ((ArrayType) type).getElementType();
- }
- if (type instanceof VectorType) {
- return ((VectorType) type).getElementType();
- }
- return null;
+ return new FlinkRowData(row.getRow(pos, numFields));
}
public Variant getVariant(int pos) {
@@ -192,15 +156,9 @@ public class FlinkRowData implements RowData {
protected static class FlinkArrayData implements ArrayData {
private final InternalArray array;
- @Nullable private final DataType elementType;
protected FlinkArrayData(InternalArray array) {
- this(array, null);
- }
-
- private FlinkArrayData(InternalArray array, @Nullable DataType elementType) {
this.array = array;
- this.elementType = elementType;
}
@Override
@@ -280,21 +238,17 @@ public class FlinkRowData implements RowData {
@Override
public ArrayData getArray(int pos) {
- return new FlinkArrayData(
- elementType instanceof VectorType ? array.getVector(pos) : array.getArray(pos),
- arrayElementType(elementType));
+ return new FlinkArrayData(array.getArray(pos));
}
@Override
public MapData getMap(int pos) {
- return new FlinkMapData(array.getMap(pos), elementType);
+ return new FlinkMapData(array.getMap(pos));
}
@Override
public RowData getRow(int pos, int numFields) {
- return new FlinkRowData(
- array.getRow(pos, numFields),
- elementType instanceof RowType ? (RowType) elementType : null);
+ return new FlinkRowData(array.getRow(pos, numFields));
}
@Override
@@ -336,21 +290,9 @@ public class FlinkRowData implements RowData {
private static class FlinkMapData implements MapData {
private final InternalMap map;
- @Nullable private final DataType keyType;
- @Nullable private final DataType valueType;
- private FlinkMapData(InternalMap map, @Nullable DataType type) {
+ private FlinkMapData(InternalMap map) {
this.map = map;
- if (type instanceof MapType) {
- this.keyType = ((MapType) type).getKeyType();
- this.valueType = ((MapType) type).getValueType();
- } else {
- this.keyType =
- type instanceof MultisetType
- ? ((MultisetType) type).getElementType()
- : null;
- this.valueType = null;
- }
}
@Override
@@ -360,12 +302,12 @@ public class FlinkRowData implements RowData {
@Override
public ArrayData keyArray() {
- return new FlinkArrayData(map.keyArray(), keyType);
+ return new FlinkArrayData(map.keyArray());
}
@Override
public ArrayData valueArray() {
- return new FlinkArrayData(map.valueArray(), valueType);
+ return new FlinkArrayData(map.valueArray());
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
index bef70b2627..1d01ba1f2e 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/action/CompactAction.java
@@ -75,7 +75,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import static org.apache.paimon.compact.CompactOptions.REWRITE_OPTIONS;
import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Table compact action for Flink. */
@@ -107,7 +106,6 @@ public class CompactAction extends TableActionBase {
}
HashMap<String, String> dynamicOptions = new HashMap<>(tableConf);
dynamicOptions.put(CoreOptions.WRITE_ONLY.key(), "false");
- dynamicOptions.putAll(REWRITE_OPTIONS);
table = table.copy(dynamicOptions);
}
@@ -156,15 +154,12 @@ public class CompactAction extends TableActionBase {
if (fileStoreTable.coreOptions().bucket() == BucketMode.POSTPONE_BUCKET) {
buildForPostponeBucketCompaction(env, fileStoreTable, isStreaming);
} else if (fileStoreTable.bucketMode() == BucketMode.BUCKET_UNAWARE) {
- if (fileStoreTable.coreOptions().clusteringIncrementalEnabled()) {
- checkArgument(
- !isStreaming || !fileStoreTable.coreOptions().dataEvolutionEnabled(),
- "Data evolution incremental clustering only supports batch mode yet.");
+ if (fileStoreTable.coreOptions().dataEvolutionEnabled()) {
+ buildForDataEvolutionTableCompact(env, fileStoreTable, isStreaming);
+ } else if (fileStoreTable.coreOptions().clusteringIncrementalEnabled()) {
new IncrementalClusterCompact(
env, fileStoreTable, partitionPredicate, fullCompaction)
.build();
- } else if (fileStoreTable.coreOptions().dataEvolutionEnabled()) {
- buildForDataEvolutionTableCompact(env, fileStoreTable, isStreaming);
} else {
buildForAppendTableCompact(env, fileStoreTable, isStreaming);
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/DataEvolutionClusterWriteOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/DataEvolutionClusterWriteOperator.java
deleted file mode 100644
index 4199c5d49e..0000000000
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/DataEvolutionClusterWriteOperator.java
+++ /dev/null
@@ -1,240 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.flink.cluster;
-
-import org.apache.paimon.append.dataevolution.ClusteringRowIdMapping;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.disk.IOManager;
-import org.apache.paimon.flink.memory.FlinkMemorySegmentPool;
-import org.apache.paimon.flink.memory.MemorySegmentAllocator;
-import org.apache.paimon.flink.sink.Committable;
-import org.apache.paimon.flink.utils.BoundedOneInputOperator;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataFilePathFactory;
-import org.apache.paimon.io.DataIncrement;
-import org.apache.paimon.memory.HeapMemorySegmentPool;
-import org.apache.paimon.memory.MemoryPoolFactory;
-import org.apache.paimon.memory.MemorySegmentPool;
-import org.apache.paimon.options.Options;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.BatchTableWrite;
-import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.utils.ProjectedRow;
-
-import org.apache.flink.runtime.memory.MemoryManager;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-
-import java.io.BufferedInputStream;
-import java.io.BufferedOutputStream;
-import java.io.DataInputStream;
-import java.io.DataOutputStream;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.stream.IntStream;
-
-import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_USE_MANAGED_MEMORY;
-import static org.apache.paimon.flink.utils.ManagedMemoryUtils.computeManagedMemory;
-import static org.apache.paimon.utils.Preconditions.checkArgument;
-import static org.apache.paimon.utils.Preconditions.checkState;
-
-/** Writes sorted rows and records their original row IDs in output-file order. */
-public class DataEvolutionClusterWriteOperator
- extends BoundedOneInputOperator<InternalRow, Committable> {
-
- private static final long serialVersionUID = 1L;
-
- private final FileStoreTable table;
- private final BinaryRow partition;
- private final String commitUser;
-
- private transient IOManager ioManager;
- private transient BatchTableWrite write;
- private transient ProjectedRow valueProjection;
- private transient File rowIdSpool;
- private transient DataOutputStream rowIds;
- private transient List<Path> mappingPaths;
- private transient List<Path> preparedDataPaths;
- private transient MemorySegmentAllocator memoryAllocator;
- private transient boolean emitted;
-
- public DataEvolutionClusterWriteOperator(
- FileStoreTable table, BinaryRow partition, String commitUser) {
- this.table = table;
- this.partition = partition;
- this.commitUser = commitUser;
- }
-
- @Override
- public void open() throws Exception {
- super.open();
- ioManager =
- IOManager.create(
- getContainingTask()
- .getEnvironment()
- .getIOManager()
- .getSpillingDirectoriesPaths());
- rowIdSpool = ioManager.createChannel("cluster-row-ids").getPathFile();
- rowIds = new DataOutputStream(new BufferedOutputStream(new FileOutputStream(rowIdSpool)));
- mappingPaths = new ArrayList<>();
- preparedDataPaths = new ArrayList<>();
- valueProjection =
- ProjectedRow.from(IntStream.range(0, table.rowType().getFieldCount()).toArray());
- // Unaware append writers use NoopCompactManager. Their data files, including files
- // produced by buffer flushes or spills, retain each partition's input order.
- write = table.newWrite(commitUser).withIOManager(ioManager);
- MemorySegmentPool memoryPool;
- if (Options.fromMap(table.options()).get(SINK_USE_MANAGED_MEMORY)) {
- MemoryManager memoryManager = getContainingTask().getEnvironment().getMemoryManager();
- memoryAllocator = new MemorySegmentAllocator(this, memoryManager);
- memoryPool =
- new FlinkMemorySegmentPool(
- computeManagedMemory(this),
- memoryManager.getPageSize(),
- memoryAllocator);
- } else {
- memoryPool =
- new HeapMemorySegmentPool(
- table.coreOptions().writeBufferSize(), table.coreOptions().pageSize());
- }
- write.withMemoryPoolFactory(new MemoryPoolFactory(memoryPool));
- }
-
- @Override
- public void processElement(StreamRecord<InternalRow> element) throws Exception {
- InternalRow row = element.getValue();
- int rowIdPosition = table.rowType().getFieldCount();
- checkArgument(!row.isNullAt(rowIdPosition), "Clustering source row ID must not be null.");
- long oldRowId = row.getLong(rowIdPosition);
- InternalRow value = valueProjection.replaceRow(row);
- checkArgument(
- write.getPartition(value).equals(partition),
- "Clustering writer received a row from another partition.");
- write.write(value, 0);
- rowIds.writeLong(oldRowId);
- }
-
- @Override
- public void endInput() throws Exception {
- rowIds.close();
- rowIds = null;
- List<CommitMessage> messages = write.prepareCommit();
- // prepareCommit transfers these files out of the writer's cleanup ownership. Keep them
- // until all mapping files and committables have been produced successfully.
- for (CommitMessage message : messages) {
- DataFilePathFactory preparedPathFactory =
- table.store()
- .pathFactory()
- .createDataFilePathFactory(message.partition(), message.bucket());
- for (DataFileMeta file : ((CommitMessageImpl) message).newFilesIncrement().newFiles()) {
- preparedDataPaths.addAll(file.collectFiles(preparedPathFactory));
- }
- }
- DataFilePathFactory pathFactory =
- table.store().pathFactory().createDataFilePathFactory(partition, 0);
- List<Committable> committables = new ArrayList<>();
- try (DataInputStream input =
- new DataInputStream(new BufferedInputStream(new FileInputStream(rowIdSpool)))) {
- for (CommitMessage message : messages) {
- CommitMessageImpl impl = (CommitMessageImpl) message;
- DataIncrement increment = impl.newFilesIncrement();
- checkState(
- message.partition().equals(partition)
- && message.bucket() == 0
- && impl.compactIncrement().isEmpty()
- && increment.deletedFiles().isEmpty()
- && increment.changelogFiles().isEmpty()
- && increment.newIndexFiles().isEmpty()
- && increment.deletedIndexFiles().isEmpty(),
- "Clustering writer must only append files for its partition.");
- List<DataFileMeta> files = new ArrayList<>();
- for (DataFileMeta file : increment.newFiles()) {
- checkState(
- file.firstRowId() == null, "Clustering output must get new row IDs.");
- String mappingName = ClusteringRowIdMapping.fileName(file.fileName());
- Path mappingPath = pathFactory.toAlignedPath(mappingName, file);
- mappingPaths.add(mappingPath);
- try (ClusteringRowIdMapping.Writer mapping =
- ClusteringRowIdMapping.createWriter(table.fileIO(), mappingPath)) {
- for (long i = 0; i < file.rowCount(); i++) {
- mapping.write(input.readLong());
- }
- }
- List<String> extraFiles = new ArrayList<>(file.extraFiles());
- extraFiles.add(mappingName);
- files.add(file.copy(extraFiles));
- }
- committables.add(
- new Committable(
- Long.MAX_VALUE,
- new CommitMessageImpl(
- message.partition(),
- message.bucket(),
- message.totalBuckets(),
- new DataIncrement(
- files,
- increment.deletedFiles(),
- increment.changelogFiles()),
- impl.compactIncrement())));
- }
- checkState(input.read() == -1, "Clustering output lost rows from its input.");
- }
- for (Committable committable : committables) {
- output.collect(new StreamRecord<>(committable));
- }
- emitted = true;
- }
-
- @Override
- public void close() throws Exception {
- try {
- if (rowIds != null) {
- rowIds.close();
- }
- } finally {
- try {
- if (write != null) {
- write.close();
- }
- } finally {
- try {
- if (memoryAllocator != null) {
- memoryAllocator.release();
- }
- if (!emitted && mappingPaths != null) {
- mappingPaths.forEach(table.fileIO()::deleteQuietly);
- }
- if (!emitted && preparedDataPaths != null) {
- preparedDataPaths.forEach(table.fileIO()::deleteQuietly);
- }
- if (ioManager != null) {
- ioManager.close();
- }
- } finally {
- super.close();
- }
- }
- }
- }
-}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
index 967d862553..25fe923df5 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/cluster/IncrementalClusterSplitSource.java
@@ -27,11 +27,9 @@ import org.apache.paimon.flink.source.SimpleSourceSplit;
import org.apache.paimon.flink.source.operator.ReadOperator;
import org.apache.paimon.flink.utils.JavaTypeInfo;
import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.table.source.Split;
-import org.apache.paimon.types.RowType;
import org.apache.paimon.utils.Pair;
import org.apache.flink.api.common.eventtime.WatermarkStrategy;
@@ -110,23 +108,15 @@ public class IncrementalClusterSplitSource extends AbstractNonCoordinatedSource<
partitioned.setParallelism(parallelism);
}
- RowType readType =
- table.coreOptions().dataEvolutionEnabled()
- ? SpecialFields.rowTypeWithRowId(table.rowType())
- : table.rowType();
- ReadOperator reader =
- table.coreOptions().dataEvolutionEnabled()
- ? new ReadOperator(() -> table.newRead().withReadType(readType), null, null)
- : new ReadOperator(table::newRead, null, null);
-
return Pair.of(
new DataStream<>(source.getExecutionEnvironment(), partitioned)
.transform(
String.format(
"Incremental-cluster reader: %s - %s",
table.fullName(), partitionSpec),
- InternalTypeInfo.of(LogicalTypeConversion.toLogicalType(readType)),
- reader)
+ InternalTypeInfo.of(
+ LogicalTypeConversion.toLogicalType(table.rowType())),
+ new ReadOperator(table::newRead, null, null))
.setParallelism(partitioned.getParallelism()),
source.forward()
.transform(
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/IncrementalClusterCompact.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/IncrementalClusterCompact.java
index 45afd6c040..25d43ddbc6 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/IncrementalClusterCompact.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/compact/IncrementalClusterCompact.java
@@ -19,10 +19,7 @@
package org.apache.paimon.flink.compact;
import org.apache.paimon.CoreOptions;
-import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.cluster.DataEvolutionClusterCompaction;
import org.apache.paimon.append.cluster.IncrementalClusterManager;
-import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
import org.apache.paimon.codegen.CodeGenUtils;
import org.apache.paimon.compact.CompactUnit;
import org.apache.paimon.data.BinaryRow;
@@ -30,15 +27,10 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.data.JoinedRow;
import org.apache.paimon.flink.FlinkRowData;
import org.apache.paimon.flink.FlinkRowWrapper;
-import org.apache.paimon.flink.cluster.DataEvolutionClusterWriteOperator;
import org.apache.paimon.flink.cluster.IncrementalClusterSplitSource;
import org.apache.paimon.flink.cluster.RewriteIncrementalClusterCommittableOperator;
import org.apache.paimon.flink.sink.Committable;
import org.apache.paimon.flink.sink.CommittableTypeInfo;
-import org.apache.paimon.flink.sink.DataEvolutionClusterSink;
-import org.apache.paimon.flink.sink.DataEvolutionCommitPreparationOperator;
-import org.apache.paimon.flink.sink.DataEvolutionCompactionTaskTypeInfo;
-import org.apache.paimon.flink.sink.DataEvolutionCompactionWorkerOperator;
import org.apache.paimon.flink.sink.FlinkSinkBuilder;
import org.apache.paimon.flink.sink.RowAppendTableSink;
import org.apache.paimon.flink.sorter.SortOperator;
@@ -48,7 +40,6 @@ import org.apache.paimon.flink.utils.InternalTypeInfo;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.SpecialFields;
import org.apache.paimon.table.sink.CommitMessage;
import org.apache.paimon.table.source.DataSplit;
import org.apache.paimon.types.RowType;
@@ -68,20 +59,12 @@ import org.apache.flink.table.data.RowData;
import javax.annotation.Nullable;
import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import static org.apache.paimon.flink.FlinkConnectorOptions.PRECOMMIT_COMPACT;
import static org.apache.paimon.flink.FlinkConnectorOptions.SCAN_PARALLELISM;
-import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_COORDINATOR_COMMIT_ENABLED;
-import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_MANAGED_WRITER_BUFFER_MEMORY;
-import static org.apache.paimon.flink.FlinkConnectorOptions.SINK_USE_MANAGED_MEMORY;
-import static org.apache.paimon.flink.FlinkConnectorOptions.UNAWARE_BUCKET_COMPACTION_PARALLELISM;
-import static org.apache.paimon.flink.utils.ManagedMemoryUtils.declareManagedMemory;
import static org.apache.paimon.table.PrimaryKeyTableUtils.addKeyNamePrefix;
/** Compact for incremental clustering. */
@@ -97,7 +80,6 @@ public class IncrementalClusterCompact {
protected final int localSampleMagnification;
protected final Map<BinaryRow, CompactUnit> compactUnits;
protected final Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> compactSplits;
- private final List<DataEvolutionCompactTask> normalCompactTasks;
public IncrementalClusterCompact(
StreamExecutionEnvironment env,
@@ -105,17 +87,8 @@ public class IncrementalClusterCompact {
PartitionPredicate partitionPredicate,
@Nullable Boolean fullCompaction) {
this.env = env;
- if (table.coreOptions().dataEvolutionEnabled()) {
- Map<String, String> rewriteOptions = new HashMap<>();
- // Sorted outputs must reach the clustering committer without another rewrite or commit.
- rewriteOptions.put(PRECOMMIT_COMPACT.key(), "false");
- rewriteOptions.put(SINK_COORDINATOR_COMMIT_ENABLED.key(), "false");
- table = table.copy(rewriteOptions);
- }
this.table = table;
- this.clusterManager =
- new IncrementalClusterManager(
- table, partitionPredicate, table.snapshotManager().latestSnapshot());
+ this.clusterManager = new IncrementalClusterManager(table, partitionPredicate);
Options options = new Options(table.options());
this.partitionComputer = table.store().partitionComputer();
this.commitUser = CoreOptions.createCommitUser(options);
@@ -131,16 +104,11 @@ public class IncrementalClusterCompact {
// non-full strategy as default for incremental clustering
this.compactUnits =
clusterManager.createCompactUnits(fullCompaction != null && fullCompaction);
- this.normalCompactTasks =
- clusterManager.snapshot() == null
- ? Collections.emptyList()
- : DataEvolutionClusterCompaction.plan(
- table, clusterManager.snapshot(), partitionPredicate, compactUnits);
this.compactSplits = clusterManager.toSplitsAndRewriteDvFiles(compactUnits);
}
public void build() throws Exception {
- if (compactUnits.isEmpty() && normalCompactTasks.isEmpty()) {
+ if (compactUnits.isEmpty()) {
env.fromSequence(0, 0).name("Nothing to Cluster Source").sinkTo(new DiscardingSink<>());
return;
}
@@ -155,24 +123,6 @@ public class IncrementalClusterCompact {
entry.getValue().getRight(),
parallelism));
}
- if (!normalCompactTasks.isEmpty()) {
- Integer workerParallelism =
- Options.fromMap(table.options()).get(UNAWARE_BUCKET_COMPACTION_PARALLELISM);
- dataStreams.add(
- env.fromCollection(
- normalCompactTasks, new DataEvolutionCompactionTaskTypeInfo())
- .name("Data Evolution Cluster Compaction Tasks")
- .rebalance()
- .transform(
- "Compact data evolution column versions",
- new CommittableTypeInfo(),
- new DataEvolutionCompactionWorkerOperator.Factory(
- table, commitUser))
- .setParallelism(
- workerParallelism == null
- ? env.getParallelism()
- : workerParallelism));
- }
buildCommitOperator(dataStreams);
}
@@ -202,13 +152,9 @@ public class IncrementalClusterCompact {
}
DataStream<RowData> sorted;
- RowType clusterRowType =
- table.coreOptions().dataEvolutionEnabled()
- ? SpecialFields.rowTypeWithRowId(table.rowType())
- : table.rowType();
CoreOptions.ClusteringIncrementalMode mode = clusterManager.clusteringIncrementalMode();
if (mode == CoreOptions.ClusteringIncrementalMode.LOCAL_SORT) {
- sorted = localSort(sourcePair.getLeft(), sinkParallelism, clusterRowType);
+ sorted = localSort(sourcePair.getLeft(), sinkParallelism);
} else {
// GLOBAL_SORT: range shuffle + local sort (default)
TableSortInfo sortInfo =
@@ -225,34 +171,22 @@ public class IncrementalClusterCompact {
env,
sourcePair.getLeft(),
table.coreOptions(),
- clusterRowType,
+ table.rowType(),
sortInfo)
.sort();
}
// 2.3 write and then reorganize the committable
// set parallelism to null, and it'll forward parallelism when doWrite()
- DataStream<InternalRow> rows =
- FlinkSinkBuilder.mapToInternalRow(
- sorted, clusterRowType, table.catalogEnvironment().catalogContext());
- DataStream<Committable> written;
- if (table.coreOptions().dataEvolutionEnabled()) {
- written =
- rows.forward()
- .transform(
- "Write data evolution cluster row-id mappings",
- new CommittableTypeInfo(),
- new DataEvolutionClusterWriteOperator(
- table, partition, commitUser))
- .setParallelism(sinkParallelism);
- Options writeOptions = Options.fromMap(table.options());
- if (writeOptions.get(SINK_USE_MANAGED_MEMORY)) {
- declareManagedMemory(written, writeOptions.get(SINK_MANAGED_WRITER_BUFFER_MEMORY));
- }
- } else {
- RowAppendTableSink sink = new RowAppendTableSink(table, null, null);
- written = sink.doWrite(rows, commitUser, null);
- }
+ RowAppendTableSink sink = new RowAppendTableSink(table, null, null);
+ DataStream<Committable> written =
+ sink.doWrite(
+ FlinkSinkBuilder.mapToInternalRow(
+ sorted,
+ table.rowType(),
+ table.catalogEnvironment().catalogContext()),
+ commitUser,
+ null);
DataStream<Committable> clusterCommittable =
written.forward()
.transform(
@@ -282,9 +216,9 @@ public class IncrementalClusterCompact {
*
* <p>Pipeline: add key prefix → SortOperator → remove key prefix → back to RowData
*/
- private DataStream<RowData> localSort(
- DataStream<RowData> input, int sinkParallelism, RowType rowType) {
+ private DataStream<RowData> localSort(DataStream<RowData> input, int sinkParallelism) {
CoreOptions options = table.coreOptions();
+ RowType rowType = table.rowType();
List<String> clusterKeys = clusterManager.clusterKeys();
final int[] keyProjectionMap = rowType.projectIndexes(clusterKeys);
@@ -385,7 +319,7 @@ public class IncrementalClusterCompact {
},
InternalTypeInfo.fromRowType(rowType))
.setParallelism(sinkParallelism)
- .map(row -> new FlinkRowData(row, rowType), input.getType())
+ .map(FlinkRowData::new, input.getType())
.setParallelism(sinkParallelism);
}
@@ -395,19 +329,6 @@ public class IncrementalClusterCompact {
for (int i = 1; i < dataStreams.size(); i++) {
dataStream = dataStream.union(dataStreams.get(i));
}
- if (table.coreOptions().dataEvolutionEnabled()) {
- Snapshot snapshot = clusterManager.snapshot();
- dataStream =
- dataStream
- .transform(
- "Data Evolution Cluster Commit Preparation : " + table.name(),
- new CommittableTypeInfo(),
- DataEvolutionCommitPreparationOperator.Factory.forClustering(
- table, snapshot))
- .forceNonParallel();
- new DataEvolutionClusterSink(table).doCommit(dataStream, commitUser);
- } else {
- sink.doCommit(dataStream, commitUser);
- }
+ sink.doCommit(dataStream, commitUser);
}
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionClusterSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionClusterSink.java
deleted file mode 100644
index a53fa4abba..0000000000
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionClusterSink.java
+++ /dev/null
@@ -1,49 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.flink.sink;
-
-import org.apache.paimon.append.dataevolution.ClusteringCommitChangesRewriter;
-import org.apache.paimon.manifest.ManifestCommittable;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.TableCommitImpl;
-
-/** Sink which commits data-evolution clustering and accompanying row-preserving compaction. */
-public class DataEvolutionClusterSink extends RowAppendTableSink {
-
- private static final long serialVersionUID = 1L;
-
- public DataEvolutionClusterSink(FileStoreTable table) {
- super(table, null, null);
- }
-
- @Override
- protected Committer.Factory<Committable, ManifestCommittable> createCommitterFactory() {
- return context -> {
- TableCommitImpl commit = table.newCommit(context.commitUser());
- commit.materializeDvRowIdCheck();
- commit.withCommitChangesRewriter(new ClusteringCommitChangesRewriter(table));
- return new StoreCommitter(table, commit, context);
- };
- }
-
- @Override
- protected CommittableStateManager<ManifestCommittable> createCommittableStateManager() {
- return new NoopCommittableStateManager();
- }
-}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java
index 40c262af08..f2c0aa7bff 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionCommitPreparationOperator.java
@@ -19,7 +19,6 @@
package org.apache.paimon.flink.sink;
import org.apache.paimon.Snapshot;
-import org.apache.paimon.append.cluster.IncrementalClusterRewriter;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactionCommitPreparation;
import org.apache.paimon.options.Options;
import org.apache.paimon.table.FileStoreTable;
@@ -42,20 +41,17 @@ public class DataEvolutionCommitPreparationOperator
private final FileStoreTable table;
private final Snapshot snapshot;
private final boolean materializeDvRowIdCheck;
- private final boolean clustering;
private final List<Committable> committables;
private DataEvolutionCommitPreparationOperator(
StreamOperatorParameters<Committable> parameters,
FileStoreTable table,
Snapshot snapshot,
- boolean materializeDvRowIdCheck,
- boolean clustering) {
+ boolean materializeDvRowIdCheck) {
super(parameters, Options.fromMap(table.options()));
this.table = table;
this.snapshot = snapshot;
this.materializeDvRowIdCheck = materializeDvRowIdCheck;
- this.clustering = clustering;
this.committables = new ArrayList<>();
}
@@ -77,15 +73,6 @@ public class DataEvolutionCommitPreparationOperator
for (Committable committable : toCommit) {
messages.add(committable.commitMessage());
}
- if (clustering) {
- List<Committable> prepared = new ArrayList<>();
- for (CommitMessage message :
- IncrementalClusterRewriter.create(table, snapshot)
- .prepareCommitMessages(messages)) {
- prepared.add(new Committable(checkpointId, message));
- }
- return prepared;
- }
for (CommitMessage message :
new DataEvolutionCompactionCommitPreparation(table, snapshot).prepare(messages)) {
toCommit.add(new Committable(toCommit.get(0).checkpointId(), message));
@@ -110,30 +97,16 @@ public class DataEvolutionCommitPreparationOperator
private final FileStoreTable table;
private final Snapshot snapshot;
private final boolean materializeDvRowIdCheck;
- private final boolean clustering;
public Factory(FileStoreTable table, Snapshot snapshot) {
this(table, snapshot, false);
}
public Factory(FileStoreTable table, Snapshot snapshot, boolean materializeDvRowIdCheck) {
- this(table, snapshot, materializeDvRowIdCheck, false);
- }
-
- public static Factory forClustering(FileStoreTable table, Snapshot snapshot) {
- return new Factory(table, snapshot, true, true);
- }
-
- private Factory(
- FileStoreTable table,
- Snapshot snapshot,
- boolean materializeDvRowIdCheck,
- boolean clustering) {
super(Options.fromMap(table.options()));
this.table = table;
this.snapshot = snapshot;
this.materializeDvRowIdCheck = materializeDvRowIdCheck;
- this.clustering = clustering;
}
@Override
@@ -142,7 +115,7 @@ public class DataEvolutionCommitPreparationOperator
StreamOperatorParameters<Committable> parameters) {
return (T)
new DataEvolutionCommitPreparationOperator(
- parameters, table, snapshot, materializeDvRowIdCheck, clustering);
+ parameters, table, snapshot, materializeDvRowIdCheck);
}
@Override
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorMaterializeSink.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorMaterializeSink.java
index 553989ac38..3ca51a69d5 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorMaterializeSink.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sink/DataEvolutionDeletionVectorMaterializeSink.java
@@ -20,7 +20,6 @@ package org.apache.paimon.flink.sink;
import org.apache.paimon.Snapshot;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
-import org.apache.paimon.append.dataevolution.MaterializeDeletionVectorsCommitChangesRewriter;
import org.apache.paimon.manifest.ManifestCommittable;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.sink.TableCommitImpl;
@@ -76,8 +75,6 @@ public class DataEvolutionDeletionVectorMaterializeSink
return context -> {
TableCommitImpl commit = table.newCommit(context.commitUser());
commit.materializeDvRowIdCheck();
- commit.withCommitChangesRewriter(
- new MaterializeDeletionVectorsCommitChangesRewriter(table));
return new StoreCommitter(table, commit, context);
};
}
diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java
index adfb5c60d5..c841afa3f5 100644
--- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java
+++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/sorter/SortUtils.java
@@ -201,7 +201,7 @@ public class SortUtils {
},
InternalTypeInfo.fromRowType(valueRowType))
.setParallelism(sinkParallelism)
- .map(row -> new FlinkRowData(row, valueRowType), inputStream.getType())
+ .map(FlinkRowData::new, inputStream.getType())
.setParallelism(sinkParallelism);
} else {
return rangeShuffleResult
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java
index 2ba7e87e8a..2115105394 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/DataEvolutionDeleteSqlITCase.java
@@ -19,38 +19,15 @@
package org.apache.paimon.flink;
import org.apache.paimon.Snapshot;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.data.InternalRow;
import org.apache.paimon.index.DeletionVectorMeta;
-import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.IndexManifestEntry;
-import org.apache.paimon.operation.FileStoreWrite;
-import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.sink.BatchTableCommit;
-import org.apache.paimon.table.sink.BatchTableWrite;
-import org.apache.paimon.table.sink.BatchWriteBuilder;
-import org.apache.paimon.table.sink.CommitMessage;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.table.sink.TableWriteImpl;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.table.source.Split;
import org.apache.flink.table.api.config.TableConfigOptions;
import org.apache.flink.types.Row;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-import org.junit.jupiter.params.provider.ValueSource;
-
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.Comparator;
+
import java.util.List;
-import java.util.Random;
import java.util.stream.Collectors;
import static org.apache.paimon.deletionvectors.DeletionVectorsIndexFile.DELETION_VECTORS_INDEX;
@@ -110,604 +87,6 @@ public class DataEvolutionDeleteSqlITCase extends CatalogITCaseBase {
Row.of(6, "six", "B"));
}
- @ParameterizedTest
- @CsvSource({
- "global-sort,",
- "local-sort,",
- "global-sort,precommit-compact=true",
- "local-sort,sink.coordinator-commit.enabled=true"
- })
- public void testIncrementalClusteringProcedurePreservesUnselectedPartitionIndexes(
- String mode, String ingestionOption) throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- createTable();
- sql(
- "ALTER TABLE T SET ("
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', "
- + "'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = '"
- + mode
- + "', "
- + "'scan.parallelism' = '2', "
- + "'source.split.target-size' = '1 b', "
- + "'sink.parallelism' = '1', "
- + "'num-sorted-run.compaction-trigger' = '10')");
- // Keep more distinct rows than shuffle ranges so duplicate maximum sample boundaries do
- // not randomly send every row to one writer. Both readers still see overlapping ranges.
- List<String> oddValues = new ArrayList<>();
- List<String> evenValues = new ArrayList<>();
- for (int id = 39; id >= 1; id -= 2) {
- oddValues.add("(" + id + ", 'value-" + id + "', 'A')");
- evenValues.add("(" + (id + 1) + ", 'value-" + (id + 1) + "', 'A')");
- }
- sql("INSERT INTO T VALUES " + String.join(", ", oddValues));
- updateFirstGroupNames(paimonTable("T"));
- sql("INSERT INTO T VALUES " + String.join(", ", evenValues));
- sql("INSERT INTO T VALUES (41, 'value-41', 'B')");
- sql("DELETE FROM T WHERE id = 2");
- createGlobalIndex();
-
- FileStoreTable table = paimonTable("T");
- long nextRowId = table.latestSnapshot().get().nextRowId();
- long beforeSnapshot = table.latestSnapshot().get().id();
- assertThat(globalIndexPartitions(table)).containsExactly("A", "B");
-
- sql("CALL sys.compact(`table` => 'default.T', partitions => 'dt=A')");
- assertThat(table.latestSnapshot().get().id()).isEqualTo(beforeSnapshot);
- sql(
- "CALL sys.compact(`table` => 'default.T', partitions => 'dt=A', "
- + "compact_strategy => 'full'"
- + (ingestionOption == null ? "" : ", options => '" + ingestionOption + "'")
- + ")");
-
- List<Row> expectedRows = new ArrayList<>();
- List<Row> expectedRowIds = new ArrayList<>();
- for (int id = 1; id <= 40; id++) {
- if (id != 2) {
- expectedRows.add(Row.of(id, (id % 2 == 1 ? "updated-" : "value-") + id, "A"));
- expectedRowIds.add(Row.of(nextRowId + expectedRowIds.size()));
- }
- }
- expectedRows.add(Row.of(41, "value-41", "B"));
- assertThat(sql("SELECT id, name, dt FROM T ORDER BY id"))
- .containsExactlyElementsOf(expectedRows);
- assertThat(sql("SELECT _ROW_ID FROM T$row_tracking " + "WHERE dt = 'A' ORDER BY _ROW_ID"))
- .containsExactlyElementsOf(expectedRowIds);
- assertClusteredFileOrder(table, mode);
- assertThat(deletionVectorCardinality(table)).isZero();
- assertThat(globalIndexPartitions(table)).containsExactly("A", "B");
- sql("ALTER TABLE T SET ('scalar-index.search-mode' = 'full')");
- assertThat(sql("SELECT id FROM T WHERE name = 'updated-3'")).containsExactly(Row.of(3));
- }
-
- @ParameterizedTest
- @CsvSource({"btree,global-sort", "btree,local-sort", "bitmap,global-sort", "bitmap,local-sort"})
- public void testIncrementalClusteringRemapsScalarIndexRowIds(String indexType, String mode)
- throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- createTable();
- sql(
- "ALTER TABLE T SET ("
- + "'write-only' = 'true', "
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', "
- + "'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = '"
- + mode
- + "', 'scan.parallelism' = '2', "
- + "'source.split.target-size' = '1 b', "
- + "'sink.parallelism' = '1', 'target-file-row-num' = '7')");
- List<String> odds = new ArrayList<>();
- List<String> evens = new ArrayList<>();
- for (int id = 39; id >= 1; id -= 2) {
- odds.add(
- "("
- + id
- + ", "
- + (id == 9 ? "CAST(NULL AS STRING)" : "'key-" + id + "'")
- + ", 'A')");
- evens.add("(" + (id + 1) + ", 'key-" + (id + 1) + "', 'A')");
- }
- sql("INSERT INTO T VALUES " + String.join(", ", odds));
- sql("INSERT INTO T VALUES " + String.join(", ", evens));
- sql("INSERT INTO T VALUES (41, 'key-41', 'B')");
- sql(
- "CALL sys.create_global_index(`table` => 'default.T', "
- + "index_column => 'name', index_type => '"
- + indexType
- + "')");
- // The old postings still contain these rows; the clustering mapping must omit them.
- sql("DELETE FROM T WHERE id IN (2, 11)");
- FileStoreTable table = paimonTable("T");
- List<String> oldAIndexes = scalarIndexFiles(table, indexType, "A");
- List<String> oldBIndexes = scalarIndexFiles(table, indexType, "B");
- assertThat(oldAIndexes).isNotEmpty();
- assertThat(oldBIndexes).isNotEmpty();
- long beforeSnapshot = table.latestSnapshot().get().id();
- long firstNewRowId = table.latestSnapshot().get().nextRowId();
-
- sql(
- "CALL sys.compact(`table` => 'default.T', partitions => 'dt=A', "
- + "compact_strategy => 'full')");
-
- // Data and remapped indexes become visible atomically, without a second index-build job.
- assertThat(table.latestSnapshot().get().id()).isEqualTo(beforeSnapshot + 1);
- assertThat(table.latestSnapshot().get().commitKind())
- .isEqualTo(Snapshot.CommitKind.COMPACT);
- assertThat(scalarIndexFiles(table, indexType, "A"))
- .isNotEmpty()
- .doesNotContainAnyElementsOf(oldAIndexes);
- assertThat(scalarIndexFiles(table, indexType, "B")).isEqualTo(oldBIndexes);
- assertThat(
- table.store().newIndexFileHandler().scan(indexType).stream()
- .filter(
- entry ->
- entry.partition()
- .getString(0)
- .toString()
- .equals("A")))
- .allSatisfy(
- entry ->
- assertThat(entry.indexFile().globalIndexMeta().rowRangeStart())
- .isGreaterThanOrEqualTo(firstNewRowId));
- assertThat(sql("SELECT _ROW_ID FROM T$row_tracking WHERE dt = 'A'"))
- .hasSize(38)
- .allSatisfy(
- row ->
- assertThat((Long) row.getField(0))
- .isGreaterThanOrEqualTo(firstNewRowId));
-
- // FAST searches only indexed rows, so data-scan fallback cannot hide missing postings.
- sql("ALTER TABLE T SET ('scalar-index.search-mode' = 'fast')");
- assertThat(sql("SELECT id FROM T WHERE name IN ('key-5', 'key-2', 'key-11') ORDER BY id"))
- .containsExactly(Row.of(5));
- assertThat(sql("SELECT id FROM T WHERE name IS NULL")).containsExactly(Row.of(9));
- List<Row> negativeMatches = new ArrayList<>();
- for (int id = 1; id <= 40; id++) {
- if (id != 2 && id != 11 && id != 5 && id != 9) {
- negativeMatches.add(Row.of(id));
- }
- }
- assertThat(sql("SELECT id FROM T WHERE dt = 'A' AND name <> 'key-5' ORDER BY id"))
- .containsExactlyElementsOf(negativeMatches);
- }
-
- @ParameterizedTest
- @CsvSource({"btree,global-sort", "bitmap,local-sort"})
- public void testIncrementalClusteringSkipsNonClusteringColumnUpdates(
- String indexType, String mode) throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- createTable();
- sql(
- "ALTER TABLE T SET ("
- + "'write-only' = 'true', "
- + "'global-index.column-update-action' = 'ignore', "
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', "
- + "'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = '"
- + mode
- + "', 'scan.parallelism' = '1', 'sink.parallelism' = '1')");
- sql("INSERT INTO T VALUES (3, 'three', 'A'), (1, 'one', 'A'), (2, 'two', 'A')");
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
- sql(
- "CALL sys.create_global_index(`table` => 'default.T', "
- + "index_column => 'id', index_type => '"
- + indexType
- + "')");
- FileStoreTable table = paimonTable("T");
- List<String> indexes = scalarIndexFiles(table, indexType, "A");
- assertThat(indexes).isNotEmpty();
- List<String> clusteredFiles = plannedFiles(table);
- assertThat(clusteredFiles).hasSize(1);
- long firstRowId = table.store().newScan().plan().files().get(0).file().nonNullFirstRowId();
- List<Row> originalIds = sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id");
-
- writePartialGroup(table, "name", firstRowId, "updated-one", "updated-two", "updated-three");
- Snapshot payloadSnapshot = table.latestSnapshot().get();
- List<String> payloadFiles = plannedFiles(table);
- assertThat(payloadFiles).hasSize(2);
- assertThat(payloadSnapshot.commitKind()).isEqualTo(Snapshot.CommitKind.APPEND);
-
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
-
- assertThat(table.latestSnapshot().get().id()).isEqualTo(payloadSnapshot.id());
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(payloadSnapshot.nextRowId());
- assertThat(plannedFiles(table)).isEqualTo(payloadFiles);
- assertThat(scalarIndexFiles(table, indexType, "A")).isEqualTo(indexes);
- assertThat(sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id"))
- .containsExactlyElementsOf(originalIds);
- // The unchanged id index must still find rows whose other columns have newer versions.
- sql("ALTER TABLE T SET ('scalar-index.search-mode' = 'fast')");
- assertThat(sql("SELECT id, name FROM T WHERE id IN (1, 3) ORDER BY id"))
- .containsExactly(Row.of(1, "updated-one"), Row.of(3, "updated-three"));
-
- writePartialGroup(table, "id", firstRowId, 30, 10, 20);
- Snapshot sortKeySnapshot = table.latestSnapshot().get();
- List<String> beforeRewrite = plannedFiles(table);
- assertThat(beforeRewrite).hasSize(3);
- long nextRowId = sortKeySnapshot.nextRowId();
-
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
-
- assertThat(table.latestSnapshot().get().id()).isEqualTo(sortKeySnapshot.id() + 1);
- assertThat(table.latestSnapshot().get().commitKind())
- .isEqualTo(Snapshot.CommitKind.COMPACT);
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(nextRowId + 3);
- assertThat(plannedFiles(table)).doesNotContainAnyElementsOf(beforeRewrite);
- assertThat(sql("SELECT id, name, dt FROM T ORDER BY id"))
- .containsExactly(
- Row.of(10, "updated-two", "A"),
- Row.of(20, "updated-three", "A"),
- Row.of(30, "updated-one", "A"));
- assertThat(sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id"))
- .containsExactly(
- Row.of(10, nextRowId),
- Row.of(20, nextRowId + 1),
- Row.of(30, nextRowId + 2));
- // The sort-key update also changed indexed values, so old keys cannot be reused.
- assertThat(scalarIndexFiles(table, indexType, "A")).isEmpty();
- }
-
- @ParameterizedTest
- @CsvSource({"btree,global-sort", "bitmap,local-sort"})
- public void testIncrementalClusteringCompactsRepeatedColumnUpdates(
- String indexType, String mode) throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- createTable();
- sql(
- "ALTER TABLE T SET ('write-only' = 'true', "
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', 'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = '"
- + mode
- + "', 'compaction.min.file-num' = '5', "
- + "'scan.parallelism' = '1', 'sink.parallelism' = '1')");
- sql("INSERT INTO T VALUES (3, 'three', 'A'), (1, 'one', 'A'), (2, 'two', 'A')");
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
- createIdIndex(indexType);
- sql("DELETE FROM T WHERE id = 2");
-
- FileStoreTable table = paimonTable("T");
- DataFileMeta base = table.store().newScan().plan().files().get(0).file();
- List<String> indexes = scalarIndexFiles(table, indexType, "A");
- List<Row> originalIds = sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id");
- for (int version = 0; version < 4; version++) {
- writePartialGroup(
- table,
- "name",
- base.nonNullFirstRowId(),
- "one-" + version,
- "two-" + version,
- "three-" + version);
- }
- Snapshot before = table.latestSnapshot().get();
- List<String> beforeFiles = plannedFiles(table);
- assertThat(beforeFiles).hasSize(5);
-
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
-
- Snapshot compacted = table.latestSnapshot().get();
- assertThat(compacted.id()).isEqualTo(before.id() + 1);
- assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
- assertThat(compacted.nextRowId()).isEqualTo(before.nextRowId());
- List<DataFileMeta> afterFiles =
- table.store().newScan().plan().files().stream()
- .map(entry -> entry.file())
- .collect(Collectors.toList());
- assertThat(afterFiles)
- .singleElement()
- .satisfies(
- file -> {
- assertThat(file.nonNullRowIdRange())
- .isEqualTo(base.nonNullRowIdRange());
- assertThat(file.level()).isEqualTo(base.level());
- assertThat(file.rowCount()).isEqualTo(3);
- });
- assertThat(plannedFiles(table)).doesNotContainAnyElementsOf(beforeFiles);
- assertThat(scalarIndexFiles(table, indexType, "A")).isEqualTo(indexes);
- assertThat(deletionVectorCardinality(table)).isEqualTo(1L);
- assertThat(sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id"))
- .containsExactlyElementsOf(originalIds);
- sql("ALTER TABLE T SET ('scalar-index.search-mode' = 'fast')");
- assertThat(sql("SELECT id, name FROM T WHERE id IN (1, 2, 3) ORDER BY id"))
- .containsExactly(Row.of(1, "one-3"), Row.of(3, "three-3"));
-
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
-
- assertThat(table.latestSnapshot().get().id()).isEqualTo(compacted.id());
- assertThat(plannedFiles(table)).containsExactly(afterFiles.get(0).fileName());
- assertThat(scalarIndexFiles(table, indexType, "A")).isEqualTo(indexes);
-
- // The compacted file contains all columns, but its id values still predate the index.
- // Retaining per-column sequences lets later clustering reuse that valid index.
- sql("INSERT INTO T VALUES (4, 'four', 'A')");
- Snapshot beforeClustering = table.latestSnapshot().get();
-
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
-
- assertThat(table.latestSnapshot().get().id()).isEqualTo(beforeClustering.id() + 1);
- assertThat(table.latestSnapshot().get().nextRowId())
- .isEqualTo(beforeClustering.nextRowId() + 3);
- assertThat(scalarIndexFiles(table, indexType, "A"))
- .isNotEmpty()
- .doesNotContainAnyElementsOf(indexes);
- assertThat(sql("SELECT id, name FROM T WHERE id IN (1, 2, 3) ORDER BY id"))
- .containsExactly(Row.of(1, "one-3"), Row.of(3, "three-3"));
- assertThat(sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id"))
- .containsExactly(
- Row.of(1, beforeClustering.nextRowId()),
- Row.of(3, beforeClustering.nextRowId() + 1),
- Row.of(4, beforeClustering.nextRowId() + 2));
- assertThat(deletionVectorCardinality(table)).isZero();
- }
-
- @Test
- public void testIncrementalClusteringCombinesCompactionInSamePartition() throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- createTable();
- sql(
- "ALTER TABLE T SET ('write-only' = 'true', "
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', 'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = 'global-sort', "
- + "'compaction.min.file-num' = '5', "
- + "'num-sorted-run.compaction-trigger' = '2', "
- + "'compaction.max-size-amplification-percent' = '10000', "
- + "'compaction.size-ratio' = '0', 'file.compression' = 'none', "
- + "'target-file-row-num' = '3', "
- + "'scan.parallelism' = '1', 'sink.parallelism' = '1')");
- FileStoreTable table = paimonTable("T");
- // A large clustered run must stay outside the minor clustering candidate. Its column
- // versions still require normal compaction in the same partition and commit.
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- try (BatchTableWrite write = builder.newWrite();
- BatchTableCommit commit = builder.newCommit()) {
- for (int id : new int[] {3, 1, 2}) {
- Random random = new Random(id);
- StringBuilder name = new StringBuilder(65536);
- for (int i = 0; i < 65536; i++) {
- name.append((char) ('a' + random.nextInt(26)));
- }
- write.write(
- GenericRow.of(
- id,
- BinaryString.fromString(name.toString()),
- BinaryString.fromString("A")));
- }
- commit.commit(write.prepareCommit());
- }
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'full')");
- createIdIndex("btree");
- sql("DELETE FROM T WHERE id = 2");
- DataFileMeta base = table.store().newScan().plan().files().get(0).file();
- List<String> retainedIndexes = scalarIndexFiles(table, "btree", "A");
- List<Row> originalIds = sql("SELECT id, _ROW_ID FROM T$row_tracking ORDER BY id");
- for (int version = 0; version < 4; version++) {
- writePartialGroup(
- table,
- "name",
- base.nonNullFirstRowId(),
- "one-" + version,
- "two-" + version,
- "three-" + version);
- }
- sql("INSERT INTO T VALUES (6, 'six', 'A'), (4, 'four', 'A'), (5, 'five', 'A')");
- sql("INSERT INTO T VALUES (9, 'nine', 'A'), (7, 'seven', 'A'), (8, 'eight', 'A')");
- createIdIndex("btree");
- List<String> allIndexes = scalarIndexFiles(table, "btree", "A");
- assertThat(allIndexes)
- .containsAll(retainedIndexes)
- .hasSizeGreaterThan(retainedIndexes.size());
- List<String> rewrittenIndexes = new ArrayList<>(allIndexes);
- rewrittenIndexes.removeAll(retainedIndexes);
- Snapshot before = table.latestSnapshot().get();
- List<String> beforeFiles = plannedFiles(table);
- assertThat(beforeFiles).hasSize(7);
-
- sql("CALL sys.compact(`table` => 'default.T', compact_strategy => 'minor')");
-
- Snapshot compacted = table.latestSnapshot().get();
- assertThat(compacted.id()).isEqualTo(before.id() + 1);
- assertThat(compacted.commitKind()).isEqualTo(Snapshot.CommitKind.COMPACT);
- assertThat(compacted.nextRowId()).isEqualTo(before.nextRowId() + 6);
- List<DataFileMeta> afterFiles =
- table.store().newScan().plan().files().stream()
- .map(entry -> entry.file())
- .collect(Collectors.toList());
- assertThat(afterFiles).hasSize(3);
- assertThat(afterFiles)
- .filteredOn(file -> file.nonNullFirstRowId() == base.nonNullFirstRowId())
- .singleElement()
- .satisfies(
- file -> {
- assertThat(file.level()).isEqualTo(base.level());
- assertThat(file.nonNullRowIdRange())
- .isEqualTo(base.nonNullRowIdRange());
- });
- assertThat(plannedFiles(table)).doesNotContainAnyElementsOf(beforeFiles);
- assertThat(scalarIndexFiles(table, "btree", "A"))
- .containsAll(retainedIndexes)
- .doesNotContainAnyElementsOf(rewrittenIndexes);
- assertThat(sql("SELECT id, _ROW_ID FROM T$row_tracking WHERE id <= 3 ORDER BY id"))
- .containsExactlyElementsOf(originalIds);
- assertThat(sql("SELECT _ROW_ID FROM T$row_tracking WHERE id >= 4"))
- .hasSize(6)
- .allSatisfy(
- row ->
- assertThat((Long) row.getField(0))
- .isGreaterThanOrEqualTo(before.nextRowId()));
- assertThat(deletionVectorCardinality(table)).isEqualTo(1L);
- sql("ALTER TABLE T SET ('scalar-index.search-mode' = 'fast')");
- assertThat(sql("SELECT id, name FROM T WHERE id IN (1, 2, 3, 4, 9) ORDER BY id"))
- .containsExactly(
- Row.of(1, "one-3"),
- Row.of(3, "three-3"),
- Row.of(4, "four"),
- Row.of(9, "nine"));
- }
-
- @ParameterizedTest
- @ValueSource(strings = {"global-sort", "local-sort"})
- public void testIncrementalClusteringPreservesInlineBlobViews(String mode) throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- sql(
- "CREATE TABLE upstream_blob (id INT, picture BYTES) WITH ("
- + "'row-tracking.enabled' = 'true', "
- + "'data-evolution.enabled' = 'true', "
- + "'blob-field' = 'picture', 'sink.parallelism' = '1')");
- sql("INSERT INTO upstream_blob VALUES (3, X'03'), (1, X'01'), (5, X'05')");
- sql("INSERT INTO upstream_blob VALUES (6, X'06'), (4, X'04'), (2, CAST(NULL AS BYTES))");
- sql(
- "CREATE TABLE blob_views (id INT, image_ref BYTES) WITH ("
- + "'row-tracking.enabled' = 'true', "
- + "'data-evolution.enabled' = 'true', "
- + "'blob-view-field' = 'image_ref', "
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', 'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = '"
- + mode
- + "', 'scan.parallelism' = '2', 'source.split.target-size' = '1 b', "
- + "'sink.parallelism' = '1')");
- for (String ids : new String[] {"1, 3, 5", "2, 4, 6"}) {
- sql(
- "INSERT INTO blob_views SELECT id, "
- + "sys.blob_view('default.upstream_blob', 'picture', _ROW_ID) "
- + "FROM `upstream_blob$row_tracking` WHERE id IN ("
- + ids
- + ")");
- }
- String rawReferences =
- "SELECT id, image_ref FROM blob_views "
- + "/*+ OPTIONS('blob-view.resolve.enabled'='false') */ ORDER BY id";
- List<Row> references = sql(rawReferences);
- assertThat(references).hasSize(6);
- assertThat(references).allSatisfy(row -> assertThat(row.getField(1)).isNotNull());
- FileStoreTable table = paimonTable("blob_views");
- long nextRowId = table.latestSnapshot().get().nextRowId();
-
- sql("CALL sys.compact(`table` => 'default.blob_views', compact_strategy => 'full')");
-
- // A currently null upstream value must not erase the non-null stored reference.
- assertThat(sql(rawReferences)).containsExactlyElementsOf(references);
- assertThat(sql("SELECT id, image_ref FROM blob_views ORDER BY id"))
- .containsExactly(
- Row.of(1, new byte[] {1}),
- Row.of(2, null),
- Row.of(3, new byte[] {3}),
- Row.of(4, new byte[] {4}),
- Row.of(5, new byte[] {5}),
- Row.of(6, new byte[] {6}));
- assertThat(sql("SELECT _ROW_ID FROM blob_views$row_tracking ORDER BY _ROW_ID"))
- .containsExactly(
- Row.of(nextRowId),
- Row.of(nextRowId + 1),
- Row.of(nextRowId + 2),
- Row.of(nextRowId + 3),
- Row.of(nextRowId + 4),
- Row.of(nextRowId + 5));
- }
-
- @ParameterizedTest
- @CsvSource({"global-sort,parquet", "local-sort,parquet", "global-sort,row", "local-sort,row"})
- public void testIncrementalClusteringPreservesInlineVectors(String mode, String format)
- throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- sql(
- "CREATE TABLE vectors (id INT, embedding ARRAY<FLOAT>) WITH ("
- + "'row-tracking.enabled' = 'true', "
- + "'data-evolution.enabled' = 'true', "
- + "'file.format' = '"
- + format
- + "', "
- + "'vector-field' = 'embedding', 'field.embedding.vector-dim' = '3', "
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', 'clustering.columns' = 'id', "
- + "'clustering.incremental.mode' = '"
- + mode
- + "', 'scan.parallelism' = '2', 'source.split.target-size' = '1 b', "
- + "'sink.parallelism' = '1')");
- sql(
- "INSERT INTO vectors VALUES "
- + "(3, CAST(ARRAY[0, 3, 4] AS ARRAY<FLOAT>)), "
- + "(1, CAST(ARRAY[0, 1, 2] AS ARRAY<FLOAT>)), "
- + "(5, CAST(ARRAY[0, 5, 6] AS ARRAY<FLOAT>))");
- sql(
- "INSERT INTO vectors VALUES "
- + "(6, CAST(ARRAY[0, 6, 7] AS ARRAY<FLOAT>)), "
- + "(4, CAST(ARRAY[0, 4, 5] AS ARRAY<FLOAT>)), "
- + "(2, CAST(NULL AS ARRAY<FLOAT>))");
- FileStoreTable table = paimonTable("vectors");
- long nextRowId = table.latestSnapshot().get().nextRowId();
-
- sql("CALL sys.compact(`table` => 'default.vectors', compact_strategy => 'full')");
-
- List<String> rows = new ArrayList<>();
- try (RecordReader<InternalRow> reader =
- table.newRead().createReader(table.newScan().plan())) {
- reader.forEachRemaining(
- row ->
- rows.add(
- row.getInt(0)
- + "|"
- + (row.isNullAt(1)
- ? "null"
- : Arrays.toString(
- row.getVector(1).toFloatArray()))));
- }
- Collections.sort(rows);
- assertThat(rows)
- .containsExactly(
- "1|[0.0, 1.0, 2.0]",
- "2|null",
- "3|[0.0, 3.0, 4.0]",
- "4|[0.0, 4.0, 5.0]",
- "5|[0.0, 5.0, 6.0]",
- "6|[0.0, 6.0, 7.0]");
- assertThat(sql("SELECT _ROW_ID FROM vectors$row_tracking ORDER BY _ROW_ID"))
- .containsExactly(
- Row.of(nextRowId),
- Row.of(nextRowId + 1),
- Row.of(nextRowId + 2),
- Row.of(nextRowId + 3),
- Row.of(nextRowId + 4),
- Row.of(nextRowId + 5));
- }
-
- @Test
- public void testIncrementalClusteringRemovesFullyDeletedPartition() throws Exception {
- tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
- createTable();
- sql(
- "ALTER TABLE T SET ("
- + "'clustering.incremental' = 'true', "
- + "'clustering.strategy' = 'order', "
- + "'clustering.columns' = 'id', "
- + "'scan.parallelism' = '2')");
- sql("INSERT INTO T VALUES (3, 'three', 'A'), (1, 'one', 'A'), (2, 'two', 'A')");
- sql("INSERT INTO T VALUES (4, 'four', 'B')");
- createGlobalIndex();
- sql("DELETE FROM T WHERE id IN (1, 2, 3)");
- FileStoreTable table = paimonTable("T");
- assertThat(deletionVectorCardinality(table)).isEqualTo(3L);
- long nextRowId = table.latestSnapshot().get().nextRowId();
-
- sql(
- "CALL sys.compact(`table` => 'default.T', partitions => 'dt=A', "
- + "compact_strategy => 'full')");
-
- assertThat(sql("SELECT id, name, dt FROM T")).containsExactly(Row.of(4, "four", "B"));
- assertThat(table.newScan().plan().splits())
- .allSatisfy(
- split ->
- assertThat(((DataSplit) split).partition().getString(0).toString())
- .isEqualTo("B"));
- assertThat(deletionVectorCardinality(table)).isZero();
- assertThat(globalIndexPartitions(table)).containsExactly("B");
- assertThat(table.latestSnapshot().get().nextRowId()).isEqualTo(nextRowId);
- }
-
@Test
public void testMaterializeDeletionVectorsForEmptyTable() throws Exception {
tEnv.getConfig().set(TableConfigOptions.TABLE_DML_SYNC, true);
@@ -859,114 +238,12 @@ public class DataEvolutionDeleteSqlITCase extends CatalogITCaseBase {
assertThat(sql("SELECT id, name, dt FROM T")).containsExactly(Row.of(1, "old", "A"));
}
- private static void updateFirstGroupNames(FileStoreTable table) throws Exception {
- List<String> names = new ArrayList<>();
- for (int id = 39; id >= 1; id -= 2) {
- names.add("updated-" + id);
- }
- writePartialGroup(table, "name", 0L, names.toArray());
- }
-
- @SuppressWarnings("unchecked")
- private static void writePartialGroup(
- FileStoreTable table, String column, long firstRowId, Object... values)
- throws Exception {
- BatchWriteBuilder builder = table.newBatchWriteBuilder();
- BinaryRow partition = ((DataSplit) table.newScan().plan().splits().get(0)).partition();
- try (BatchTableWrite write =
- builder.newWrite()
- .withWriteType(
- table.rowType().project(Arrays.asList(column, "dt")));
- BatchTableCommit commit = builder.newCommit()) {
- // Partial rows do not contain the partition field at its full-schema position.
- FileStoreWrite<InternalRow> fileStoreWrite =
- ((TableWriteImpl<InternalRow>) write).getWrite();
- for (Object value : values) {
- fileStoreWrite.write(
- partition,
- 0,
- GenericRow.of(
- value instanceof String
- ? BinaryString.fromString((String) value)
- : value,
- BinaryString.fromString("A")));
- }
- List<CommitMessage> messages = write.prepareCommit();
- for (CommitMessage message : messages) {
- List<DataFileMeta> files =
- ((CommitMessageImpl) message).newFilesIncrement().newFiles();
- List<DataFileMeta> assigned =
- files.stream()
- .map(file -> file.assignFirstRowId(firstRowId))
- .collect(Collectors.toList());
- files.clear();
- files.addAll(assigned);
- }
- commit.commit(messages);
- }
- }
-
- private static void assertClusteredFileOrder(FileStoreTable table, String mode)
- throws Exception {
- List<DataFileMeta> files = new ArrayList<>();
- List<List<Integer>> fileIds = new ArrayList<>();
- for (Split split : table.newScan().plan().splits()) {
- DataSplit dataSplit = (DataSplit) split;
- if (!dataSplit.partition().getString(0).toString().equals("A")) {
- continue;
- }
- for (DataFileMeta file : dataSplit.dataFiles()) {
- files.add(file);
- DataSplit fileSplit =
- DataSplit.builder()
- .withSnapshot(dataSplit.snapshotId())
- .withPartition(dataSplit.partition())
- .withBucket(dataSplit.bucket())
- .withBucketPath(dataSplit.bucketPath())
- .withDataFiles(Collections.singletonList(file))
- .build();
- List<Integer> ids = new ArrayList<>();
- try (RecordReader<InternalRow> reader = table.newRead().createReader(fileSplit)) {
- reader.forEachRemaining(row -> ids.add(row.getInt(0)));
- }
- assertThat(ids).isNotEmpty().isSorted();
- fileIds.add(ids);
- }
- }
- // Verify ordering across actual output files, as well as within each file.
- assertThat(files).hasSizeGreaterThanOrEqualTo(2);
- assertThat(files).allSatisfy(file -> assertThat(file.level()).isGreaterThan(0));
- fileIds.sort(Comparator.comparingInt(ids -> ids.get(0)));
- boolean overlappingRanges = false;
- for (int i = 1; i < fileIds.size(); i++) {
- List<Integer> previous = fileIds.get(i - 1);
- overlappingRanges |= previous.get(previous.size() - 1) >= fileIds.get(i).get(0);
- }
- if (mode.equals("global-sort")) {
- assertThat(overlappingRanges)
- .as("global-sort output file ranges must not overlap")
- .isFalse();
- } else {
- assertThat(overlappingRanges)
- .as("local-sort preserves the interleaved input ranges across readers")
- .isTrue();
- }
- }
-
private void createGlobalIndex() {
sql(
"CALL sys.create_global_index(`table` => 'default.T', "
+ "index_column => 'name', index_type => 'btree')");
}
- private void createIdIndex(String indexType) {
- sql(
- "CALL sys.create_global_index(`table` => 'default.T', "
- + "index_column => 'id', index_type => '"
- + indexType
- + "')");
- }
-
private void createTable() {
sql(
"CREATE TABLE T (id INT, name STRING, dt STRING) PARTITIONED BY (dt) WITH ("
@@ -1002,15 +279,6 @@ public class DataEvolutionDeleteSqlITCase extends CatalogITCaseBase {
.collect(Collectors.toList());
}
- private static List<String> scalarIndexFiles(
- FileStoreTable table, String indexType, String partition) {
- return table.store().newIndexFileHandler().scan(indexType).stream()
- .filter(entry -> entry.partition().getString(0).toString().equals(partition))
- .map(entry -> entry.indexFile().fileName())
- .sorted()
- .collect(Collectors.toList());
- }
-
private static void assertDeleteSnapshot(
FileStoreTable table, List<String> originalFiles, long deletedRows) {
assertThat(table.latestSnapshot().get().operation()).isEqualTo(Snapshot.Operation.DELETE);
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRowDataWithVectorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRowDataWithVectorTest.java
index 63317299bd..ba8542d395 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRowDataWithVectorTest.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkRowDataWithVectorTest.java
@@ -18,23 +18,13 @@
package org.apache.paimon.flink;
-import org.apache.paimon.data.BinaryRow;
import org.apache.paimon.data.BinaryVector;
-import org.apache.paimon.data.GenericArray;
-import org.apache.paimon.data.GenericMap;
import org.apache.paimon.data.GenericRow;
import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.data.serializer.InternalRowSerializer;
-import org.apache.paimon.types.DataTypes;
-import org.apache.paimon.types.RowType;
-import org.apache.paimon.utils.KeyProjectedRow;
import org.apache.flink.table.data.ArrayData;
-import org.apache.flink.table.data.MapData;
import org.junit.jupiter.api.Test;
-import java.util.Collections;
-
import static org.assertj.core.api.Assertions.assertThat;
/** Tests for {@link FlinkRowData} and vector type. */
@@ -51,71 +41,6 @@ public class FlinkRowDataWithVectorTest {
assertThat(arrayData.toFloatArray()).isEqualTo(values);
}
- @Test
- public void testBinaryVectorAndArrayAsArrayData() {
- float[] vector = new float[] {0.0f, 1.0f, 2.0f};
- float[] array = new float[] {3.0f, 4.0f};
- RowType rowType =
- DataTypes.ROW(
- DataTypes.VECTOR(3, DataTypes.FLOAT()), DataTypes.ARRAY(DataTypes.FLOAT()));
- BinaryRow row =
- new InternalRowSerializer(rowType)
- .toBinaryRow(
- GenericRow.of(
- BinaryVector.fromPrimitiveArray(vector),
- new GenericArray(array)));
-
- FlinkRowData rowData = new FlinkRowData(row, rowType);
-
- assertThat(rowData.getArray(0).toFloatArray()).isEqualTo(vector);
- assertThat(rowData.getArray(1).toFloatArray()).isEqualTo(array);
- }
-
- @Test
- public void testProjectedBinaryVectorAsArrayData() {
- float[] vector = new float[] {0.0f, 1.0f, 2.0f};
- RowType valueType = DataTypes.ROW(DataTypes.VECTOR(3, DataTypes.FLOAT()));
- BinaryRow sortedRow =
- new InternalRowSerializer(DataTypes.INT(), valueType.getTypeAt(0))
- .toBinaryRow(GenericRow.of(5, BinaryVector.fromPrimitiveArray(vector)));
- KeyProjectedRow projected = new KeyProjectedRow(new int[] {1}).replaceRow(sortedRow);
-
- assertThat(new FlinkRowData(projected, valueType).getArray(0).toFloatArray())
- .isEqualTo(vector);
- }
-
- @Test
- public void testNestedBinaryVectorsAsArrayData() {
- float[] values = new float[] {0.0f, 1.0f, 2.0f};
- BinaryVector vector = BinaryVector.fromPrimitiveArray(values);
- RowType nestedRowType = DataTypes.ROW(DataTypes.VECTOR(3, DataTypes.FLOAT()));
- RowType rowType =
- DataTypes.ROW(
- nestedRowType,
- DataTypes.ARRAY(nestedRowType),
- DataTypes.MAP(
- nestedRowType.getTypeAt(0),
- DataTypes.ARRAY(nestedRowType.getTypeAt(0))));
- BinaryRow row =
- new InternalRowSerializer(rowType)
- .toBinaryRow(
- GenericRow.of(
- GenericRow.of(vector),
- new GenericArray(new Object[] {GenericRow.of(vector)}),
- new GenericMap(
- Collections.singletonMap(
- vector,
- new GenericArray(new Object[] {vector})))));
-
- FlinkRowData rowData = new FlinkRowData(row, rowType);
-
- assertThat(rowData.getRow(0, 1).getArray(0).toFloatArray()).isEqualTo(values);
- assertThat(rowData.getArray(1).getRow(0, 1).getArray(0).toFloatArray()).isEqualTo(values);
- MapData map = rowData.getMap(2);
- assertThat(map.keyArray().getArray(0).toFloatArray()).isEqualTo(values);
- assertThat(map.valueArray().getArray(0).getArray(0).toFloatArray()).isEqualTo(values);
- }
-
@Test
public void testNullVector() {
InternalRow row = GenericRow.of(1, null);
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java
index a9da8ef0f3..9914bc68d3 100644
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java
+++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/IncrementalClusterActionITCase.java
@@ -64,23 +64,10 @@ import java.util.stream.Collectors;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatCode;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** IT cases for incremental clustering action. */
public class IncrementalClusterActionITCase extends ActionITCaseBase {
- @Test
- public void testStreamingDataEvolutionClusteringIsRejected() throws Exception {
- Map<String, String> options = commonOptions();
- options.put(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
- options.put(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
- createTable(null, options);
-
- assertThatThrownBy(() -> runAction(true, Collections.emptyList()))
- .isInstanceOf(IllegalArgumentException.class)
- .hasMessage("Data evolution incremental clustering only supports batch mode yet.");
- }
-
@Test
public void testClusterUnpartitionedTable() throws Exception {
FileStoreTable table = createTable(null);
diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/cluster/DataEvolutionClusterWriteOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/cluster/DataEvolutionClusterWriteOperatorTest.java
deleted file mode 100644
index bfaf23bfe7..0000000000
--- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/cluster/DataEvolutionClusterWriteOperatorTest.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.paimon.flink.cluster;
-
-import org.apache.paimon.CoreOptions;
-import org.apache.paimon.append.dataevolution.ClusteringRowIdMapping;
-import org.apache.paimon.data.BinaryRow;
-import org.apache.paimon.data.BinaryString;
-import org.apache.paimon.data.GenericRow;
-import org.apache.paimon.data.InternalRow;
-import org.apache.paimon.flink.sink.Committable;
-import org.apache.paimon.flink.sink.CommittableTypeInfo;
-import org.apache.paimon.fs.Path;
-import org.apache.paimon.fs.PositionOutputStream;
-import org.apache.paimon.fs.local.LocalFileIO;
-import org.apache.paimon.io.DataFileMeta;
-import org.apache.paimon.io.DataFilePathFactory;
-import org.apache.paimon.reader.RecordReader;
-import org.apache.paimon.schema.FileSystemSchemaManager;
-import org.apache.paimon.schema.Schema;
-import org.apache.paimon.schema.SchemaManager;
-import org.apache.paimon.table.FileStoreTable;
-import org.apache.paimon.table.FileStoreTableFactory;
-import org.apache.paimon.table.sink.CommitMessageImpl;
-import org.apache.paimon.table.source.DataSplit;
-import org.apache.paimon.types.DataTypes;
-import org.apache.paimon.utils.CloseableIterator;
-
-import org.apache.flink.api.common.ExecutionConfig;
-import org.apache.flink.streaming.runtime.streamrecord.StreamRecord;
-import org.apache.flink.streaming.util.OneInputStreamOperatorTestHarness;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
-import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.CsvSource;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.List;
-import java.util.stream.Collectors;
-import java.util.stream.IntStream;
-
-import static org.assertj.core.api.Assertions.assertThat;
-import static org.assertj.core.api.Assertions.assertThatThrownBy;
-
-/** Tests that old row IDs follow actual output boundaries, including buffered spills. */
-class DataEvolutionClusterWriteOperatorTest {
-
- @TempDir java.nio.file.Path tempDir;
-
- @ParameterizedTest
- @CsvSource({"false,false", "true,false", "true,true"})
- void testMappingsFollowWrittenRowsAcrossFileRolls(boolean buffered, boolean spillable)
- throws Exception {
- FileStoreTable table = createTable(buffered, spillable);
- DataEvolutionClusterWriteOperator operator =
- new DataEvolutionClusterWriteOperator(table, BinaryRow.EMPTY_ROW, "cluster-test");
- List<DataFileMeta> files;
- int rowCount = 250;
- char[] payload = new char[2048];
- java.util.Arrays.fill(payload, 'x');
- try (OneInputStreamOperatorTestHarness<InternalRow, Committable> harness =
- new OneInputStreamOperatorTestHarness<>(operator)) {
- harness.setup(new CommittableTypeInfo().createSerializer(new ExecutionConfig()));
- harness.open();
- for (int id = 0; id < rowCount; id++) {
- harness.processElement(
- new StreamRecord<>(
- GenericRow.of(
- id,
- BinaryString.fromString(String.valueOf(payload)),
- oldRowId(id, rowCount))));
- }
- operator.endInput();
- files =
- harness.extractOutputValues().stream()
- .flatMap(
- committable ->
- ((CommitMessageImpl) committable.commitMessage())
- .newFilesIncrement().newFiles().stream())
- .collect(Collectors.toList());
- }
-
- assertThat(files).hasSizeGreaterThan(2);
- assertThat(files).allSatisfy(file -> assertThat(file.firstRowId()).isNull());
- assertThat(table.latestSnapshot()).isEmpty();
- DataFilePathFactory pathFactory =
- table.store().pathFactory().createDataFilePathFactory(BinaryRow.EMPTY_ROW, 0);
- List<Integer> writtenIds = new ArrayList<>();
- for (DataFileMeta file : files) {
- List<Integer> ids = readIds(table, file);
- assertThat(ids).hasSize((int) file.rowCount());
- writtenIds.addAll(ids);
- String mappingName = ClusteringRowIdMapping.fileName(file.fileName());
- assertThat(file.extraFiles()).contains(mappingName);
- ClusteringRowIdMapping.read(
- table.fileIO(),
- pathFactory.toAlignedPath(mappingName, file),
- file.rowCount(),
- (ordinal, oldId) ->
- assertThat(oldId)
- .isEqualTo(oldRowId(ids.get((int) ordinal), rowCount)));
- }
- assertThat(writtenIds)
- .containsExactlyElementsOf(
- IntStream.range(0, rowCount).boxed().collect(Collectors.toList()));
- }
-
- @Test
- void testEmptyWriterProducesNoMappingsOrCommit() throws Exception {
- FileStoreTable table = createTable(true, true);
- DataEvolutionClusterWriteOperator operator =
- new DataEvolutionClusterWriteOperator(table, BinaryRow.EMPTY_ROW, "cluster-test");
- try (OneInputStreamOperatorTestHarness<InternalRow, Committable> harness =
- new OneInputStreamOperatorTestHarness<>(operator)) {
- harness.setup(new CommittableTypeInfo().createSerializer(new ExecutionConfig()));
- harness.open();
- operator.endInput();
- assertThat(harness.extractOutputValues()).isEmpty();
- }
- assertThat(table.latestSnapshot()).isEmpty();
- }
-
- @Test
- void testMappingFailureCleansPreparedDataAndEarlierMappings() throws Exception {
- FailingMappingFileIO fileIO = new FailingMappingFileIO();
- FileStoreTable table = createTable(true, true, fileIO);
- fileIO.recordOutputs = true;
- DataEvolutionClusterWriteOperator operator =
- new DataEvolutionClusterWriteOperator(table, BinaryRow.EMPTY_ROW, "cluster-test");
- try (OneInputStreamOperatorTestHarness<InternalRow, Committable> harness =
- new OneInputStreamOperatorTestHarness<>(operator)) {
- harness.setup(new CommittableTypeInfo().createSerializer(new ExecutionConfig()));
- harness.open();
- for (int id = 0; id < 40; id++) {
- harness.processElement(
- new StreamRecord<>(
- GenericRow.of(id, BinaryString.fromString("payload"), (long) id)));
- }
- assertThatThrownBy(operator::endInput)
- .isInstanceOf(IOException.class)
- .hasMessageContaining("injected mapping failure");
- assertThat(harness.extractOutputValues()).isEmpty();
- assertThat(fileIO.mappingCount).isEqualTo(2);
- assertThat(
- fileIO.outputs.stream()
- .filter(
- path ->
- !ClusteringRowIdMapping.isMappingFile(
- path.getName())))
- .isNotEmpty();
- }
- for (Path output : fileIO.outputs) {
- assertThat(fileIO.exists(output)).as("aborted output %s", output).isFalse();
- }
- assertThat(table.latestSnapshot()).isEmpty();
- }
-
- private FileStoreTable createTable(boolean buffered, boolean spillable) throws Exception {
- return createTable(buffered, spillable, LocalFileIO.create());
- }
-
- private FileStoreTable createTable(boolean buffered, boolean spillable, LocalFileIO fileIO)
- throws Exception {
- Schema schema =
- Schema.newBuilder()
- .column("id", DataTypes.INT())
- .column("payload", DataTypes.STRING())
- .option(CoreOptions.BUCKET.key(), "-1")
- .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true")
- .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true")
- .option(CoreOptions.WRITE_ONLY.key(), "true")
- .option(CoreOptions.WRITE_BUFFER_FOR_APPEND.key(), String.valueOf(buffered))
- .option(CoreOptions.WRITE_BUFFER_SPILLABLE.key(), String.valueOf(spillable))
- .option(CoreOptions.WRITE_BUFFER_SIZE.key(), "16 kb")
- .option(CoreOptions.PAGE_SIZE.key(), "4 kb")
- .option(CoreOptions.TARGET_FILE_ROW_NUM.key(), "17")
- .build();
- Path path = new Path(tempDir.toString());
- SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, path);
- return FileStoreTableFactory.create(fileIO, path, schemaManager.createTable(schema));
- }
-
- private List<Integer> readIds(FileStoreTable table, DataFileMeta file) throws Exception {
- DataSplit split =
- DataSplit.builder()
- .withPartition(BinaryRow.EMPTY_ROW)
- .withBucket(0)
- .withTotalBuckets(1)
- .withBucketPath(
- table.store()
- .pathFactory()
- .bucketPath(BinaryRow.EMPTY_ROW, 0)
- .toString())
- .withDataFiles(Collections.singletonList(file.assignFirstRowId(0L)))
- .rawConvertible(false)
- .build();
- List<Integer> result = new ArrayList<>();
- try (RecordReader<InternalRow> reader = table.newRead().createReader(split);
- CloseableIterator<InternalRow> rows = reader.toCloseableIterator()) {
- while (rows.hasNext()) {
- InternalRow row = rows.next();
- assertThat(row.getFieldCount()).isEqualTo(2);
- result.add(row.getInt(0));
- }
- }
- return result;
- }
-
- private static long oldRowId(int id, int rowCount) {
- return 1000L + (rowCount - id - 1) * 3L;
- }
-
- private static class FailingMappingFileIO extends LocalFileIO {
-
- private static final long serialVersionUID = 1L;
- private final List<Path> outputs = new ArrayList<>();
- private boolean recordOutputs;
- private int mappingCount;
-
- @Override
- public PositionOutputStream newOutputStream(Path path, boolean overwrite)
- throws IOException {
- if (recordOutputs) {
- outputs.add(path);
- if (ClusteringRowIdMapping.isMappingFile(path.getName()) && ++mappingCount == 2) {
- throw new IOException("injected mapping failure");
- }
- }
- return super.newOutputStream(path, overwrite);
- }
- }
-}
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
index fbed108b36..218dc6d0a9 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/CompactProcedure.java
@@ -23,9 +23,7 @@ import org.apache.paimon.CoreOptions.OrderType;
import org.apache.paimon.Snapshot;
import org.apache.paimon.append.AppendCompactCoordinator;
import org.apache.paimon.append.AppendCompactTask;
-import org.apache.paimon.append.cluster.DataEvolutionClusterCompaction;
import org.apache.paimon.append.cluster.IncrementalClusterManager;
-import org.apache.paimon.append.dataevolution.ClusteringCommitChangesRewriter;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactCoordinator;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
import org.apache.paimon.compact.CompactUnit;
@@ -106,9 +104,9 @@ import java.util.function.Predicate;
import java.util.stream.Collectors;
import scala.collection.JavaConverters;
+import scala.collection.Seq;
import static org.apache.paimon.CoreOptions.createCommitUser;
-import static org.apache.paimon.compact.CompactOptions.REWRITE_OPTIONS;
import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
import static org.apache.paimon.spark.utils.SparkProcedureUtils.readParallelism;
import static org.apache.paimon.types.VectorType.isVectorStoreFile;
@@ -237,9 +235,7 @@ public class CompactProcedure extends BaseProcedure {
ProcedureUtils.putIfNotEmpty(
dynamicOptions, CoreOptions.WRITE_ONLY.key(), "false");
ProcedureUtils.putAllOptions(dynamicOptions, options);
- dynamicOptions.putAll(REWRITE_OPTIONS);
table = table.copy(dynamicOptions);
- relation = createRelation(tableIdent, sparkTable.copy(table));
if (coreOptions.clusteringIncrementalEnabled()
&& (!OrderType.NONE.name().equals(sortType))) {
throw new IllegalArgumentException(
@@ -332,20 +328,16 @@ public class CompactProcedure extends BaseProcedure {
javaSparkContext);
break;
case BUCKET_UNAWARE:
- if (clusterIncrementalEnabled) {
- clusterIncrementalUnAwareBucketTable(
- table,
- partitionPredicate,
- partitionIdleTime,
- fullCompact,
- relation);
- } else if (table.coreOptions().dataEvolutionEnabled()) {
+ if (table.coreOptions().dataEvolutionEnabled()) {
compactDataEvolutionTable(
table,
relation,
partitionPredicate,
partitionIdleTime,
javaSparkContext);
+ } else if (clusterIncrementalEnabled) {
+ clusterIncrementalUnAwareBucketTable(
+ table, partitionPredicate, fullCompact, relation);
} else {
compactUnAwareBucketTable(
table, partitionPredicate, partitionIdleTime, javaSparkContext);
@@ -849,37 +841,18 @@ public class CompactProcedure extends BaseProcedure {
private void clusterIncrementalUnAwareBucketTable(
FileStoreTable table,
@Nullable PartitionPredicate partitionPredicate,
- @Nullable Duration partitionIdleTime,
boolean fullCompaction,
DataSourceV2Relation relation) {
- Snapshot snapshot = table.snapshotManager().latestSnapshot();
- if (snapshot == null) {
- return;
- }
IncrementalClusterManager incrementalClusterManager =
- new IncrementalClusterManager(table, partitionPredicate, snapshot);
+ new IncrementalClusterManager(table, partitionPredicate);
Map<BinaryRow, CompactUnit> compactUnits =
incrementalClusterManager.createCompactUnits(fullCompaction);
- List<DataEvolutionCompactTask> compactionTasks =
- DataEvolutionClusterCompaction.plan(
- table, snapshot, partitionPredicate, compactUnits);
- if (partitionIdleTime != null) {
- Set<BinaryRow> idlePartitions =
- getPartitionsToCompact(
- table.newSnapshotReader().withSnapshot(snapshot), partitionIdleTime);
- compactUnits.keySet().retainAll(idlePartitions);
- compactionTasks.removeIf(task -> !idlePartitions.contains(task.partition()));
- }
LOG.info(
"Planned {} compact units to incrementally cluster for table {}, full compaction {}.",
compactUnits.size(),
table.fullName(),
fullCompaction);
- if (compactUnits.isEmpty() && compactionTasks.isEmpty()) {
- return;
- }
-
Map<BinaryRow, Pair<List<DataSplit>, CommitMessage>> partitionSplits =
incrementalClusterManager.toSplitsAndRewriteDvFiles(compactUnits);
@@ -902,7 +875,6 @@ public class CompactProcedure extends BaseProcedure {
Dataset<Row> datasetForWrite =
partitionSplits.values().stream()
.map(Pair::getKey)
- .filter(splits -> !splits.isEmpty())
.map(
splits -> {
Dataset<Row> dataset =
@@ -920,90 +892,73 @@ public class CompactProcedure extends BaseProcedure {
})
.reduce(Dataset::union)
.orElse(null);
- // set to write only to prevent invoking compaction
- // do not use overwrite, we don't need to overwrite the whole partition
- PaimonSparkWriter writer = PaimonSparkWriter.apply(table).writeOnly();
- List<CommitMessage> commitMessages =
- datasetForWrite == null
- ? Collections.emptyList()
- : JavaConverters.seqAsJavaList(writer.write(datasetForWrite));
- if (LOG.isDebugEnabled()) {
- LOG.debug("Commit messages after writing:{}", commitMessages);
- }
-
- // re-organize the commit messages to generate the compact messages
- Map<BinaryRow, List<DataFileMeta>> partitionClustered = new HashMap<>();
- for (CommitMessage commitMessage : commitMessages) {
- checkArgument(commitMessage.bucket() == 0);
- partitionClustered
- .computeIfAbsent(commitMessage.partition(), k -> new ArrayList<>())
- .addAll(((CommitMessageImpl) commitMessage).newFilesIncrement().newFiles());
- }
-
- List<CommitMessage> clusterMessages = new ArrayList<>();
- for (Map.Entry<BinaryRow, CompactUnit> entry : compactUnits.entrySet()) {
- BinaryRow partition = entry.getKey();
- CommitMessageImpl dvCommitMessage =
- (CommitMessageImpl) partitionSplits.get(partition).getValue();
- List<DataFileMeta> clusterBefore = entry.getValue().files();
- // upgrade the clustered file to outputLevel
- List<DataFileMeta> clusterAfter =
- IncrementalClusterManager.upgrade(
- partitionClustered.getOrDefault(partition, Collections.emptyList()),
- entry.getValue().outputLevel());
- LOG.info(
- "Partition {}: upgrade file level to {}",
- partition,
- compactUnits.get(partition).outputLevel());
-
- List<IndexFileMeta> newIndexFiles = new ArrayList<>();
- List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
- if (dvCommitMessage != null) {
- newIndexFiles = dvCommitMessage.compactIncrement().newIndexFiles();
- deletedIndexFiles = dvCommitMessage.compactIncrement().deletedIndexFiles();
+ if (datasetForWrite != null) {
+ // set to write only to prevent invoking compaction
+ // do not use overwrite, we don't need to overwrite the whole partition
+ PaimonSparkWriter writer = PaimonSparkWriter.apply(table).writeOnly();
+ Seq<CommitMessage> commitMessages = writer.write(datasetForWrite);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Commit messages after writing:{}", commitMessages);
}
- // get the dv index messages
- CompactIncrement compactIncrement =
- new CompactIncrement(
- clusterBefore,
- clusterAfter,
- Collections.emptyList(),
- newIndexFiles,
- deletedIndexFiles);
- clusterMessages.add(
- new CommitMessageImpl(
- partition,
- // bucket 0 is bucket for unaware-bucket table
- // for compatibility with the old design
- 0,
- table.coreOptions().bucket(),
- DataIncrement.emptyIncrement(),
- compactIncrement));
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Commit messages after reorganizing:{}", clusterMessages);
- }
+ // re-organize the commit messages to generate the compact messages
+ Map<BinaryRow, List<DataFileMeta>> partitionClustered = new HashMap<>();
+ for (CommitMessage commitMessage : JavaConverters.seqAsJavaList(commitMessages)) {
+ checkArgument(commitMessage.bucket() == 0);
+ partitionClustered
+ .computeIfAbsent(commitMessage.partition(), k -> new ArrayList<>())
+ .addAll(((CommitMessageImpl) commitMessage).newFilesIncrement().newFiles());
+ }
- if (table.coreOptions().dataEvolutionEnabled()) {
- String commitUser = createCommitUser(table.coreOptions().toConfiguration());
- try (TableCommitImpl commit = table.newCommit(commitUser).materializeDvRowIdCheck()) {
- commit.withCommitChangesRewriter(new ClusteringCommitChangesRewriter(table));
- clusterMessages.addAll(
- DataEvolutionRewriteExecutor.executeTasks(
- table,
- compactionTasks,
- new JavaSparkContext(spark().sparkContext()),
- spark(),
- commitUser));
- List<CommitMessage> preparedMessages =
- incrementalClusterManager.prepareCommitMessages(clusterMessages);
- commit.commit(preparedMessages);
- } catch (Exception e) {
- throw new RuntimeException("Commit data evolution clustering failed.", e);
+ List<CommitMessage> clusterMessages = new ArrayList<>();
+ for (Map.Entry<BinaryRow, List<DataFileMeta>> entry : partitionClustered.entrySet()) {
+ BinaryRow partition = entry.getKey();
+ CommitMessageImpl dvCommitMessage =
+ (CommitMessageImpl) partitionSplits.get(partition).getValue();
+ List<DataFileMeta> clusterBefore = compactUnits.get(partition).files();
+ // upgrade the clustered file to outputLevel
+ List<DataFileMeta> clusterAfter =
+ IncrementalClusterManager.upgrade(
+ entry.getValue(), compactUnits.get(partition).outputLevel());
+ LOG.info(
+ "Partition {}: upgrade file level to {}",
+ partition,
+ compactUnits.get(partition).outputLevel());
+
+ List<IndexFileMeta> newIndexFiles = new ArrayList<>();
+ List<IndexFileMeta> deletedIndexFiles = new ArrayList<>();
+ if (dvCommitMessage != null) {
+ newIndexFiles = dvCommitMessage.compactIncrement().newIndexFiles();
+ deletedIndexFiles = dvCommitMessage.compactIncrement().deletedIndexFiles();
+ }
+
+ // get the dv index messages
+ CompactIncrement compactIncrement =
+ new CompactIncrement(
+ clusterBefore,
+ clusterAfter,
+ Collections.emptyList(),
+ newIndexFiles,
+ deletedIndexFiles);
+ clusterMessages.add(
+ new CommitMessageImpl(
+ partition,
+ // bucket 0 is bucket for unaware-bucket table
+ // for compatibility with the old design
+ 0,
+ table.coreOptions().bucket(),
+ DataIncrement.emptyIncrement(),
+ compactIncrement));
}
- } else {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Commit messages after reorganizing:{}", clusterMessages);
+ }
+
writer.commit(JavaConverters.asScalaBuffer(clusterMessages).toSeq());
+ } else {
+ LOG.info(
+ "No data split to incrementally cluster for table {}, skip this compact job.",
+ table.fullName());
}
}
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java
index cd942f9ab2..a6610bfff8 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/DataEvolutionRewriteExecutor.java
@@ -105,6 +105,7 @@ final class DataEvolutionRewriteExecutor {
CommitConfigurer commitConfigurer,
@Nullable CommitMessageRewriter commitMessageRewriter,
@Nullable Consumer<List<CommitMessage>> commitObserver) {
+ CommitMessageSerializer messageSerializer = new CommitMessageSerializer();
String commitUser = createCommitUser(table.coreOptions().toConfiguration());
Snapshot preparationSnapshot = initialSnapshot;
int round = 0;
@@ -121,14 +122,64 @@ final class DataEvolutionRewriteExecutor {
table.fullName());
continue;
}
+ boolean containsMaterializeDeletion =
+ compactionTasks.stream()
+ .anyMatch(
+ task ->
+ task.type()
+ == DataEvolutionCompactTask.TaskType
+ .MATERIALIZE_DELETION);
+
+ DataEvolutionCompactTaskSerializer serializer =
+ new DataEvolutionCompactTaskSerializer();
+ List<byte[]> serializedTasks = new ArrayList<>();
+ try {
+ for (DataEvolutionCompactTask compactionTask : compactionTasks) {
+ serializedTasks.add(serializer.serialize(compactionTask));
+ }
+ } catch (IOException e) {
+ throw new RuntimeException("Serialize data evolution rewrite task failed.", e);
+ }
+
+ int readParallelism = readParallelism(serializedTasks, sparkSession);
+ LOG.info(
+ "Starting to execute {} data evolution rewrite tasks of table {} in round {} "
+ + "with read parallelism {}, contains materialize deletion {}.",
+ serializedTasks.size(),
+ table.fullName(),
+ round,
+ readParallelism,
+ containsMaterializeDeletion);
+ JavaRDD<byte[]> commitMessageJavaRDD =
+ javaSparkContext
+ .parallelize(serializedTasks, readParallelism)
+ .mapPartitions(
+ (FlatMapFunction<Iterator<byte[]>, byte[]>)
+ taskIterator -> {
+ DataEvolutionCompactTaskSerializer ser =
+ new DataEvolutionCompactTaskSerializer();
+ List<byte[]> messagesBytes = new ArrayList<>();
+ CommitMessageSerializer messageSer =
+ new CommitMessageSerializer();
+ while (taskIterator.hasNext()) {
+ DataEvolutionCompactTask task =
+ ser.deserialize(
+ ser.getVersion(),
+ taskIterator.next());
+ messagesBytes.add(
+ messageSer.serialize(
+ task.doCompact(
+ table,
+ commitUser)));
+ }
+ return messagesBytes.iterator();
+ });
+
+ List<byte[]> serializedMessages = new ArrayList<>(commitMessageJavaRDD.collect());
try {
List<CommitMessage> compactMessages =
- executeTasks(
- table,
- compactionTasks,
- javaSparkContext,
- sparkSession,
- commitUser);
+ deserializeCommitMessagesAndReleaseSerializedBytes(
+ messageSerializer, serializedMessages);
Snapshot committedSnapshot =
commitWithMergeConflictRetry(
table,
@@ -158,54 +209,6 @@ final class DataEvolutionRewriteExecutor {
}
}
- /** Executes one task batch without publishing it, so callers can share a commit. */
- static List<CommitMessage> executeTasks(
- FileStoreTable table,
- List<DataEvolutionCompactTask> tasks,
- JavaSparkContext javaSparkContext,
- SparkSession sparkSession,
- String commitUser)
- throws IOException {
- if (tasks.isEmpty()) {
- return Collections.emptyList();
- }
- DataEvolutionCompactTaskSerializer serializer = new DataEvolutionCompactTaskSerializer();
- List<byte[]> serializedTasks = new ArrayList<>(tasks.size());
- for (DataEvolutionCompactTask task : tasks) {
- serializedTasks.add(serializer.serialize(task));
- }
- int parallelism = readParallelism(serializedTasks, sparkSession);
- LOG.info(
- "Executing {} data evolution rewrite tasks for table {} with parallelism {}.",
- tasks.size(),
- table.fullName(),
- parallelism);
- JavaRDD<byte[]> serializedMessages =
- javaSparkContext
- .parallelize(serializedTasks, parallelism)
- .mapPartitions(
- (FlatMapFunction<Iterator<byte[]>, byte[]>)
- taskIterator -> {
- DataEvolutionCompactTaskSerializer taskSerializer =
- new DataEvolutionCompactTaskSerializer();
- CommitMessageSerializer messageSerializer =
- new CommitMessageSerializer();
- List<byte[]> messages = new ArrayList<>();
- while (taskIterator.hasNext()) {
- DataEvolutionCompactTask task =
- taskSerializer.deserialize(
- taskSerializer.getVersion(),
- taskIterator.next());
- messages.add(
- messageSerializer.serialize(
- task.doCompact(table, commitUser)));
- }
- return messages.iterator();
- });
- return deserializeCommitMessagesAndReleaseSerializedBytes(
- new CommitMessageSerializer(), new ArrayList<>(serializedMessages.collect()));
- }
-
private static Snapshot commitWithMergeConflictRetry(
FileStoreTable table,
Snapshot taskSnapshot,
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MaterializeDeletionVectorsProcedure.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MaterializeDeletionVectorsProcedure.java
index c742a673bf..7377d8fb6e 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MaterializeDeletionVectorsProcedure.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/procedure/MaterializeDeletionVectorsProcedure.java
@@ -22,7 +22,6 @@ import org.apache.paimon.CoreOptions;
import org.apache.paimon.Snapshot;
import org.apache.paimon.append.dataevolution.DataEvolutionCompactTask;
import org.apache.paimon.append.dataevolution.DataEvolutionDeletionVectorMaterializeCoordinator;
-import org.apache.paimon.append.dataevolution.MaterializeDeletionVectorsCommitChangesRewriter;
import org.apache.paimon.partition.PartitionPredicate;
import org.apache.paimon.spark.utils.SparkProcedureUtils;
import org.apache.paimon.table.BucketMode;
@@ -177,8 +176,6 @@ public class MaterializeDeletionVectorsProcedure extends BaseProcedure {
@Override
public void configure(TableCommitImpl commit) {
commit.materializeDvRowIdCheck();
- commit.withCommitChangesRewriter(
- new MaterializeDeletionVectorsCommitChangesRewriter(table));
}
... 434 lines suppressed ...