You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by ju...@apache.org on 2026/07/28 04:37:11 UTC
(paimon) branch native-reassign created (now 82b9cf4f47)
This is an automated email from the ASF dual-hosted git repository.
leaves12138 pushed a change to branch native-reassign
in repository https://gitbox.apache.org/repos/asf/paimon.git
at 82b9cf4f47 Update reassign for java
This branch includes the following new commits:
new 82b9cf4f47 Update reassign for java
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(paimon) 01/01: Update reassign for java
Posted by ju...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
leaves12138 pushed a commit to branch native-reassign
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 82b9cf4f477c06f5b220e6ee0c8ca764d85cdab3
Author: 仟弋 <ye...@alibaba-inc.com>
AuthorDate: Tue Jul 28 00:49:52 2026 +0800
Update reassign for java
---
.../DataEvolutionRowIdReassigner.java | 2102 +++++++++++++++++---
.../append/dataevolution/RowRangeMappingIndex.java | 122 +-
...ataEvolutionRowIdReassignerCompactPlanTest.java | 237 +++
.../DataEvolutionRowIdReassignerTest.java | 625 ++++++
.../LegacyDataEvolutionRowIdReassigner.java} | 13 +-
.../dataevolution/RowRangeMappingIndexTest.java | 65 +
6 files changed, 2806 insertions(+), 358 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java
index d81d98ed70..c37a7ddff4 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java
@@ -24,9 +24,14 @@ import org.apache.paimon.annotation.VisibleForTesting;
import org.apache.paimon.codegen.CodeGenUtils;
import org.apache.paimon.codegen.RecordComparator;
import org.apache.paimon.data.BinaryRow;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.InternalArray;
+import org.apache.paimon.data.InternalRow;
+import org.apache.paimon.format.FileFormat;
+import org.apache.paimon.format.FormatReaderFactory;
import org.apache.paimon.index.GlobalIndexMeta;
import org.apache.paimon.index.IndexFileMeta;
-import org.apache.paimon.manifest.FileEntry;
+import org.apache.paimon.io.DataFileMeta;
import org.apache.paimon.manifest.FileKind;
import org.apache.paimon.manifest.IndexManifestEntry;
import org.apache.paimon.manifest.IndexManifestFile;
@@ -34,23 +39,32 @@ import org.apache.paimon.manifest.ManifestEntry;
import org.apache.paimon.manifest.ManifestFile;
import org.apache.paimon.manifest.ManifestFileMeta;
import org.apache.paimon.manifest.ManifestList;
+import org.apache.paimon.memory.MemorySegmentUtils;
import org.apache.paimon.operation.FileStoreCommitImpl;
import org.apache.paimon.options.Options;
import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.reader.RecordReader;
import org.apache.paimon.stats.SimpleStats;
import org.apache.paimon.table.FileStoreTable;
import org.apache.paimon.table.SpecialFields;
+import org.apache.paimon.types.DataField;
+import org.apache.paimon.types.RowType;
+import org.apache.paimon.utils.FileUtils;
import org.apache.paimon.utils.Filter;
import org.apache.paimon.utils.Pair;
import org.apache.paimon.utils.Range;
-import org.apache.paimon.utils.RangeHelper;
+import org.apache.paimon.utils.SerializationUtils;
+import org.apache.paimon.utils.VersionedObjectSerializer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import javax.annotation.Nullable;
+import java.io.IOException;
+import java.io.UncheckedIOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashMap;
@@ -61,8 +75,6 @@ import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import static org.apache.paimon.format.blob.BlobFileFormat.isBlobFile;
-import static org.apache.paimon.types.VectorType.isVectorStoreFile;
import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkState;
@@ -130,7 +142,7 @@ public class DataEvolutionRowIdReassigner {
ManifestFile manifestFile = table.store().manifestFileFactory().create();
ManifestList manifestList = table.store().manifestListFactory().create();
Optional<AssignmentPlan> optionalPlan =
- planAssignment(manifestList.readDataManifests(latest), manifestFile);
+ planAssignment(manifestList.readDataManifests(latest));
if (!optionalPlan.isPresent()) {
LOG.info(
"Skip reassigning row IDs for table {} because no partition requires reassignment.",
@@ -186,48 +198,43 @@ public class DataEvolutionRowIdReassigner {
throw new IllegalStateException("Unreachable retry state while reassigning row IDs.");
}
- private Optional<AssignmentPlan> planAssignment(
- List<ManifestFileMeta> manifestMetas, ManifestFile manifestFile) {
+ private Optional<AssignmentPlan> planAssignment(List<ManifestFileMeta> manifestMetas) {
List<List<ManifestFileMeta>> manifestGroups = manifestGroupsByPartition(manifestMetas);
- Map<BinaryRow, List<ManifestEntry>> entriesToReassignByPartition = new LinkedHashMap<>();
-
+ List<List<ManifestFileMeta>> includedGroups = new ArrayList<>();
for (List<ManifestFileMeta> manifestGroup : manifestGroups) {
- if (skipManifestGroupByPartitionFilter(manifestGroup)) {
- continue;
- }
-
- List<ManifestEntry> currentEntries = currentEntries(manifestGroup, manifestFile);
- if (currentEntries.isEmpty()) {
- continue;
- }
-
- Map<BinaryRow, List<ManifestEntry>> entriesByPartition =
- entriesByPartition(currentEntries);
- Set<BinaryRow> partitionsToReassign = partitionsToReassign(entriesByPartition);
- if (partitionsToReassign.isEmpty()) {
- continue;
- }
-
- for (ManifestEntry entry : currentEntries) {
- if (partitionsToReassign.contains(entry.partition())) {
- entriesToReassignByPartition
- .computeIfAbsent(entry.partition(), ignored -> new ArrayList<>())
- .add(entry);
- }
+ if (!skipManifestGroupByPartitionFilter(manifestGroup)) {
+ includedGroups.add(manifestGroup);
}
}
- if (entriesToReassignByPartition.isEmpty()) {
+ PlanningState planningState =
+ new PlanningState(table, partitionPredicate, new ArrayList<>(manifestMetas));
+ planningState.validateGroups(includedGroups);
+ for (List<ManifestFileMeta> manifestGroup : includedGroups) {
+ planningState.planGroup(manifestGroup);
+ }
+ PlanningState.Result compactPlan = planningState.buildResult();
+ if (compactPlan.isEmpty()) {
return Optional.empty();
}
- RelativeRowIdMappings relativeRowIdMappings =
- createRelativeRowIdMappings(entriesToReassignByPartition);
+ List<ManifestFileMeta> manifestMetasToRewrite =
+ new ArrayList<>(compactPlan.manifestOrdinals.length);
+ for (int ordinal : compactPlan.manifestOrdinals) {
+ manifestMetasToRewrite.add(manifestMetas.get(ordinal));
+ }
+
+ Map<BinaryRow, RowRangeMappingIndex> mappings = new LinkedHashMap<>();
+ for (PlanningState.PartitionMapping mapping : compactPlan.partitionMappings) {
+ mappings.put(
+ mapping.partition,
+ RowRangeMappingIndex.createFromOwnedArrays(
+ mapping.oldStarts, mapping.oldEnds, mapping.newRelativeStarts));
+ }
return Optional.of(
new AssignmentPlan(
- findManifestMetasToRewrite(
- manifestMetas, relativeRowIdMappings, manifestFile),
- relativeRowIdMappings));
+ manifestMetasToRewrite,
+ new RelativeRowIdMappings(mappings, compactPlan.totalOffset)));
}
private List<List<ManifestFileMeta>> manifestGroupsByPartition(
@@ -371,49 +378,6 @@ public class DataEvolutionRowIdReassigner {
return false;
}
- private List<ManifestEntry> currentEntries(
- List<ManifestFileMeta> manifestMetas, ManifestFile manifestFile) {
- Set<FileEntry.Identifier> deletedIdentifiers =
- deletedIdentifiers(manifestFile, manifestMetas);
-
- List<ManifestEntry> currentEntries = new ArrayList<>();
- for (ManifestFileMeta manifestMeta : manifestMetas) {
- if (manifestMeta.numAddedFiles() <= 0) {
- continue;
- }
- List<ManifestEntry> entries = readPlanningManifestEntries(manifestFile, manifestMeta);
- for (ManifestEntry entry : entries) {
- if (entry.kind() == FileKind.ADD
- && partitionIncluded(entry.partition())
- && !deletedIdentifiers.contains(entry.identifier())) {
- currentEntries.add(entry);
- }
- }
- }
- return currentEntries;
- }
-
- private Set<FileEntry.Identifier> deletedIdentifiers(
- ManifestFile manifestFile, List<ManifestFileMeta> manifestMetas) {
- Set<FileEntry.Identifier> deletedIdentifiers = new HashSet<>();
- for (ManifestFileMeta manifestMeta : manifestMetas) {
- if (manifestMeta.numDeletedFiles() <= 0) {
- continue;
- }
- List<ManifestEntry> entries = readPlanningManifestEntries(manifestFile, manifestMeta);
- for (ManifestEntry entry : entries) {
- if (entry.kind() == FileKind.DELETE && partitionIncluded(entry.partition())) {
- deletedIdentifiers.add(entry.identifier());
- }
- }
- }
- return deletedIdentifiers;
- }
-
- private boolean partitionIncluded(BinaryRow partition) {
- return !partitionFilterEnabled() || partitionPredicate.test(partition);
- }
-
private boolean partitionFilterEnabled() {
return partitionPredicate != null;
}
@@ -549,53 +513,6 @@ public class DataEvolutionRowIdReassigner {
return false;
}
- private boolean manifestGroupMayContainPartitions(
- List<ManifestFileMeta> manifestGroup, PartitionPredicate effectivePartitionPredicate) {
- int partitionFieldCount = table.schema().logicalPartitionType().getFieldCount();
- for (ManifestFileMeta manifestMeta : manifestGroup) {
- if (!containsPartitionStats(manifestMeta, partitionFieldCount)) {
- return true;
- }
-
- SimpleStats partitionStats = manifestMeta.partitionStats();
- if (effectivePartitionPredicate.test(
- manifestMeta.numAddedFiles() + manifestMeta.numDeletedFiles(),
- partitionStats.minValues(),
- partitionStats.maxValues(),
- partitionStats.nullCounts())) {
- return true;
- }
- }
- return false;
- }
-
- private List<ManifestFileMeta> findManifestMetasToRewrite(
- List<ManifestFileMeta> manifestMetas,
- RelativeRowIdMappings relativeRowIdMappings,
- ManifestFile manifestFile) {
- PartitionPredicate plannedPartitionPredicate =
- PartitionPredicate.fromMultiple(
- table.schema().logicalPartitionType(),
- relativeRowIdMappings.mappings.keySet());
- checkState(plannedPartitionPredicate != null, "Planned partition predicate is null.");
- List<ManifestFileMeta> result = new ArrayList<>();
- for (ManifestFileMeta manifestMeta : manifestMetas) {
- if (!manifestGroupMayContainPartitions(
- Collections.singletonList(manifestMeta), plannedPartitionPredicate)) {
- continue;
- }
- for (ManifestEntry entry : readPlanningManifestEntries(manifestFile, manifestMeta)) {
- RowRangeMappingIndex mapping =
- relativeRowIdMappings.mappings.get(entry.partition());
- if (mapping != null && mapping.map(entry.file().nonNullRowIdRange()).isPresent()) {
- result.add(manifestMeta);
- break;
- }
- }
- }
- return result;
- }
-
private Pair<String, Long> writeBaseManifestList(
List<ManifestFileMeta> manifestMetas,
Map<String, List<ManifestFileMeta>> rewrittenManifestMetas,
@@ -648,20 +565,6 @@ public class DataEvolutionRowIdReassigner {
return new RewrittenDataManifests(rewrittenManifestMetas, fileCount);
}
- private Map<BinaryRow, List<ManifestEntry>> entriesByPartition(List<ManifestEntry> entries) {
- Comparator<ManifestEntry> comparator = entryComparator();
- Collections.sort(entries, comparator);
-
- Map<BinaryRow, List<ManifestEntry>> entriesByPartition = new LinkedHashMap<>();
- for (ManifestEntry entry : entries) {
- validatePlanningEntry(entry);
- entriesByPartition
- .computeIfAbsent(entry.partition(), k -> new ArrayList<>())
- .add(entry);
- }
- return entriesByPartition;
- }
-
private void validatePlanningEntry(ManifestEntry entry) {
List<String> writeCols = entry.file().writeCols();
checkState(
@@ -675,138 +578,6 @@ public class DataEvolutionRowIdReassigner {
table.name());
}
- private Set<BinaryRow> partitionsToReassign(
- Map<BinaryRow, List<ManifestEntry>> entriesByPartition) {
- Set<BinaryRow> partitionsToReassign = new HashSet<>();
- for (Map.Entry<BinaryRow, List<ManifestEntry>> entry : entriesByPartition.entrySet()) {
- if (!partitionRowIdsAreContiguous(entry.getValue())) {
- partitionsToReassign.add(entry.getKey());
- }
- }
- return partitionsToReassign;
- }
-
- private boolean partitionRowIdsAreContiguous(List<ManifestEntry> entries) {
- for (ManifestEntry entry : entries) {
- validatePlanningEntry(entry);
- }
- List<Range> logicalRanges = logicalRanges(entries);
- if (logicalRanges.size() <= 1) {
- return true;
- }
-
- Collections.sort(
- logicalRanges,
- (left, right) -> {
- int result = Long.compare(left.from, right.from);
- return result == 0 ? Long.compare(left.to, right.to) : result;
- });
- long previousEnd = logicalRanges.get(0).to;
- for (int i = 1; i < logicalRanges.size(); i++) {
- Range current = logicalRanges.get(i);
- if (current.from != previousEnd + 1) {
- return false;
- }
- previousEnd = current.to;
- }
- return true;
- }
-
- private List<Range> logicalRanges(List<ManifestEntry> entries) {
- RangeHelper<ManifestEntry> rangeHelper =
- new RangeHelper<>(entry -> entry.file().nonNullRowIdRange());
- List<List<ManifestEntry>> groups = rangeHelper.mergeOverlappingRanges(entries);
- List<Range> logicalRanges = new ArrayList<>(groups.size());
- for (List<ManifestEntry> group : groups) {
- logicalRanges.add(oldLogicalRange(group));
- }
- return logicalRanges;
- }
-
- private RelativeRowIdMappings createRelativeRowIdMappings(
- Map<BinaryRow, List<ManifestEntry>> entriesByPartition) {
- Map<BinaryRow, List<Range>> logicalRangesByPartition = new LinkedHashMap<>();
- for (Map.Entry<BinaryRow, List<ManifestEntry>> partition : entriesByPartition.entrySet()) {
- for (ManifestEntry entry : partition.getValue()) {
- validatePlanningEntry(entry);
- }
- logicalRangesByPartition.put(partition.getKey(), logicalRanges(partition.getValue()));
- }
- return createRelativeRowIdMappingsFromRanges(logicalRangesByPartition);
- }
-
- private RelativeRowIdMappings createRelativeRowIdMappingsFromRanges(
- Map<BinaryRow, List<Range>> logicalRangesByPartition) {
- List<BinaryRow> partitions = new ArrayList<>(logicalRangesByPartition.keySet());
- RecordComparator partitionComparator = partitionComparator();
- Collections.sort(partitions, partitionComparator);
-
- Map<BinaryRow, RowRangeMappingIndex> result = new LinkedHashMap<>();
- long nextOffset = 0L;
- for (BinaryRow partition : partitions) {
- List<Range> ranges = new ArrayList<>(logicalRangesByPartition.get(partition));
- Collections.sort(
- ranges,
- (left, right) -> {
- int compare = Long.compare(left.from, right.from);
- return compare == 0 ? Long.compare(left.to, right.to) : compare;
- });
- List<RowRangeMappingIndex.Mapping> mappings = new ArrayList<>(ranges.size());
- for (Range range : ranges) {
- mappings.add(RowRangeMappingIndex.mapping(range.from, range.to, nextOffset));
- nextOffset = Math.addExact(nextOffset, range.count());
- }
- result.put(partition, RowRangeMappingIndex.create(mappings));
- }
- return new RelativeRowIdMappings(result, nextOffset);
- }
-
- private Range oldLogicalRange(List<ManifestEntry> group) {
- List<ManifestEntry> dataFiles = new ArrayList<>();
- for (ManifestEntry entry : group) {
- if (!isSpecialFile(entry)) {
- dataFiles.add(entry);
- }
- }
-
- Range logicalRange;
- if (dataFiles.isEmpty()) {
- logicalRange = spanningRange(group);
- } else {
- logicalRange = dataFiles.get(0).file().nonNullRowIdRange();
- for (ManifestEntry dataFile : dataFiles) {
- Range current = dataFile.file().nonNullRowIdRange();
- checkState(
- logicalRange.from == current.from && logicalRange.to == current.to,
- "Data files in one overlapping row-id group must have the same row-id range, but found %s and %s.",
- logicalRange,
- current);
- }
- }
-
- for (ManifestEntry entry : group) {
- Range range = entry.file().nonNullRowIdRange();
- checkState(
- range.from >= logicalRange.from && range.to <= logicalRange.to,
- "File '%s' row-id range %s is outside logical row-id range %s.",
- entry.file().fileName(),
- range,
- logicalRange);
- }
- return logicalRange;
- }
-
- private Range spanningRange(List<ManifestEntry> group) {
- long min = Long.MAX_VALUE;
- long max = Long.MIN_VALUE;
- for (ManifestEntry entry : group) {
- Range range = entry.file().nonNullRowIdRange();
- min = Math.min(min, range.from);
- max = Math.max(max, range.to);
- }
- return new Range(min, max);
- }
-
private RewrittenIndexManifest rewriteIndexManifest(Assignment assignment) {
if (assignment.snapshot.indexManifest() == null) {
return new RewrittenIndexManifest(null, 0);
@@ -888,55 +659,1766 @@ public class DataEvolutionRowIdReassigner {
ManifestEntry::copyWithoutStats);
}
- private Comparator<ManifestEntry> entryComparator() {
- RecordComparator partitionComparator = partitionComparator();
- Comparator<ManifestEntry> withoutPartition = entryComparatorWithoutPartition();
- return (left, right) -> {
- int partitionCompare = partitionComparator.compare(left.partition(), right.partition());
- if (partitionCompare != 0) {
- return partitionCompare;
- }
- return withoutPartition.compare(left, right);
- };
- }
-
private RecordComparator partitionComparator() {
return CodeGenUtils.newRecordComparator(
table.schema().logicalPartitionType().getFieldTypes());
}
- private Comparator<ManifestEntry> entryComparatorWithoutPartition() {
- return (left, right) -> {
- int result =
- Long.compare(left.file().nonNullFirstRowId(), right.file().nonNullFirstRowId());
- if (result != 0) {
- return result;
+ /**
+ * Per-invocation state for {@link #planAssignment(List)}. Each pass projects only the manifest
+ * fields it needs, and no state escapes the plan invocation.
+ */
+ static final class PlanningState {
+
+ private static final int MANIFEST_ENTRY_VERSION = 2;
+ private static final int EXCLUDED_PARTITION_CACHE_SIZE = 1024;
+ private static final int CURRENT_ENTRY_WORDS = 3;
+ private static final int MAX_INITIAL_CURRENT_ENTRIES = 1 << 24;
+ private static final long CURRENT_SPECIAL = 1L << 32;
+ private static final BinaryString ROW_ID_FIELD =
+ BinaryString.fromString(SpecialFields.ROW_ID.name());
+ private static final BinaryString BLOB_FILE_SUFFIX = BinaryString.fromString(".blob");
+ private static final BinaryString VECTOR_FILE_MARKER = BinaryString.fromString(".vector.");
+
+ private final FileStoreTable table;
+ private final @Nullable PartitionPredicate partitionPredicate;
+ private final List<ManifestFileMeta> manifestMetas;
+ private final Map<String, Integer> manifestOrdinals;
+ private final boolean[] rewrittenManifests;
+ private final Projection deleteProjection;
+ private final Projection addIdentifierProjection;
+ private final Projection compactAddProjection;
+ private final Projection rewriteProjection;
+ private final Map<ByteArrayKey, SelectedPartition> selectedPartitions;
+ private long nextManifestGroupOrdinal;
+ private long nextRetainedAddScanOrdinal;
+
+ PlanningState(
+ FileStoreTable table,
+ @Nullable PartitionPredicate partitionPredicate,
+ List<ManifestFileMeta> manifestMetas) {
+ this.table = table;
+ this.partitionPredicate = partitionPredicate;
+ this.manifestMetas = manifestMetas;
+ this.manifestOrdinals = manifestOrdinals(manifestMetas);
+ this.rewrittenManifests = new boolean[manifestMetas.size()];
+ this.deleteProjection =
+ Projection.create(
+ table,
+ true,
+ "_FILE_NAME",
+ "_LEVEL",
+ "_EXTRA_FILES",
+ "_EMBEDDED_FILE_INDEX",
+ "_EXTERNAL_PATH");
+ this.addIdentifierProjection =
+ Projection.create(
+ table,
+ true,
+ "_FILE_NAME",
+ "_ROW_COUNT",
+ "_LEVEL",
+ "_EXTRA_FILES",
+ "_EMBEDDED_FILE_INDEX",
+ "_EXTERNAL_PATH",
+ "_FIRST_ROW_ID",
+ "_WRITE_COLS",
+ "_MAX_SEQUENCE_NUMBER");
+ this.compactAddProjection =
+ Projection.create(
+ table,
+ false,
+ "_FILE_NAME",
+ "_ROW_COUNT",
+ "_FIRST_ROW_ID",
+ "_WRITE_COLS",
+ "_MAX_SEQUENCE_NUMBER");
+ this.rewriteProjection =
+ Projection.create(table, false, "_ROW_COUNT", "_FIRST_ROW_ID", "_WRITE_COLS");
+ this.selectedPartitions = new LinkedHashMap<>();
+ }
+
+ void planGroup(List<ManifestFileMeta> manifestGroup) {
+ long manifestGroupOrdinal = nextManifestGroupOrdinal;
+ nextManifestGroupOrdinal = Math.addExact(nextManifestGroupOrdinal, 1L);
+ GroupState group =
+ new GroupState(
+ table.schema().logicalPartitionType().getFieldCount(),
+ partitionPredicate,
+ EXCLUDED_PARTITION_CACHE_SIZE,
+ partitionPredicate == null
+ ? initialCurrentEntryCapacity(manifestGroup)
+ : 0);
+ IdentifierScratch identifier = new IdentifierScratch();
+ long[] rowRangeScratch = new long[2];
+
+ for (ManifestFileMeta manifestMeta : manifestGroup) {
+ if (manifestMeta.numDeletedFiles() <= 0) {
+ continue;
+ }
+ scan(
+ manifestMeta,
+ deleteProjection,
+ row -> {
+ if (kind(row) != FileKind.DELETE.toByteValue()) {
+ return true;
+ }
+ PartitionState partition = group.internPartition(row.getBinary(2));
+ if (partition == null) {
+ return true;
+ }
+ InternalRow file = file(row, deleteProjection);
+ identifier.encode(row, file, deleteProjection);
+ group.deletedIdentifiers.add(
+ partition.id, identifier.bytes(), identifier.length());
+ return true;
+ });
+ }
+
+ Projection addProjection =
+ group.deletedIdentifiers.isEmpty()
+ ? compactAddProjection
+ : addIdentifierProjection;
+ for (ManifestFileMeta manifestMeta : manifestGroup) {
+ if (manifestMeta.numAddedFiles() <= 0) {
+ continue;
+ }
+ int manifestOrdinal = ordinal(manifestMeta);
+ scan(
+ manifestMeta,
+ addProjection,
+ row -> {
+ if (kind(row) != FileKind.ADD.toByteValue()) {
+ return true;
+ }
+ PartitionState partition = group.internPartition(row.getBinary(2));
+ if (partition == null) {
+ return true;
+ }
+ InternalRow file = file(row, addProjection);
+ if (!group.deletedIdentifiers.isEmpty()) {
+ identifier.encode(row, file, addProjection);
+ if (group.deletedIdentifiers.contains(
+ partition.id, identifier.bytes(), identifier.length())) {
+ return true;
+ }
+ }
+
+ BinaryString fileName = requiredFileName(file, addProjection);
+ readRowRange(
+ file,
+ addProjection,
+ manifestOrdinal,
+ fileName,
+ rowRangeScratch);
+ checkState(
+ !writesRowId(file, addProjection),
+ "Cannot reassign row IDs for file '%s' because it physically stores the row-id field.",
+ fileName);
+ long maxSequenceNumber = requiredMaxSequenceNumber(file, addProjection);
+ long retainedAddScanOrdinal = nextRetainedAddScanOrdinal;
+ nextRetainedAddScanOrdinal =
+ Math.addExact(nextRetainedAddScanOrdinal, 1L);
+ partition.considerLegacyOrderKey(
+ manifestGroupOrdinal,
+ rowRangeScratch[0],
+ fileOrder(fileName),
+ maxSequenceNumber,
+ fileName,
+ retainedAddScanOrdinal);
+ group.currentEntries.add(
+ partition.id,
+ isSpecialFile(fileName),
+ rowRangeScratch[0],
+ inclusiveRangeCount(rowRangeScratch[0], rowRangeScratch[1]));
+ return true;
+ });
+ }
+ identifier.release();
+ group.releaseDeletedIdentifiers();
+
+ GroupSelection[] groupSelections = group.finishAddPass();
+ boolean selected = false;
+ for (GroupSelection selection : groupSelections) {
+ if (selection == null) {
+ continue;
+ }
+ selected = true;
+ mergeSelectedPartition(selection);
}
- result = Integer.compare(fileOrder(left), fileOrder(right));
- if (result != 0) {
- return result;
+ if (!selected) {
+ return;
}
- result =
- Long.compare(right.file().maxSequenceNumber(), left.file().maxSequenceNumber());
- if (result != 0) {
- return result;
+
+ for (ManifestFileMeta manifestMeta : manifestGroup) {
+ int manifestOrdinal = ordinal(manifestMeta);
+ scan(
+ manifestMeta,
+ rewriteProjection,
+ row -> {
+ PartitionState partition = group.internPartition(row.getBinary(2));
+ if (partition == null) {
+ return true;
+ }
+ if (partition.id >= groupSelections.length) {
+ return true;
+ }
+ GroupSelection selection = groupSelections[partition.id];
+ if (selection == null) {
+ return true;
+ }
+ InternalRow file = file(row, rewriteProjection);
+ readRowRange(
+ file,
+ rewriteProjection,
+ manifestOrdinal,
+ null,
+ rowRangeScratch);
+ if (!rangesFullyCover(
+ rowRangeScratch[0],
+ rowRangeScratch[1],
+ selection.logicalRanges)) {
+ return true;
+ }
+ checkState(
+ !writesRowId(file, rewriteProjection),
+ "Cannot reassign an entry in manifest '%s' because it physically stores the row-id field.",
+ manifestMeta.fileName());
+ rewrittenManifests[manifestOrdinal] = true;
+ return false;
+ });
}
- return left.file().fileName().compareTo(right.file().fileName());
- };
- }
+ }
+
+ Result buildResult() {
+ if (selectedPartitions.isEmpty()) {
+ return new Result(new int[0], Collections.emptyList(), 0L);
+ }
+
+ List<SelectedPartition> partitions = new ArrayList<>(selectedPartitions.values());
+ RecordComparator typedComparator =
+ CodeGenUtils.newRecordComparator(
+ table.schema().logicalPartitionType().getFieldTypes());
+ partitions.sort(
+ (left, right) -> {
+ int comparison = typedComparator.compare(left.partition, right.partition);
+ return comparison != 0
+ ? comparison
+ : left.legacyOrderKey.compareTo(right.legacyOrderKey);
+ });
+
+ List<PartitionMapping> mappings = new ArrayList<>(partitions.size());
+ long nextOffset = 0L;
+ for (SelectedPartition partition : partitions) {
+ partition.logicalRanges.normalizeOverlapping();
+ int rangeCount = partition.logicalRanges.size();
+ checkState(rangeCount > 0, "Selected partition has no logical row-id ranges.");
+ OwnedPrimitiveRanges ownedRanges = partition.logicalRanges.takeOwned();
+ long[] oldStarts = ownedRanges.starts;
+ long[] oldEnds = ownedRanges.ends;
+ long[] newStarts = new long[rangeCount];
+ for (int i = 0; i < rangeCount; i++) {
+ newStarts[i] = nextOffset;
+ nextOffset =
+ Math.addExact(
+ nextOffset, inclusiveRangeCount(oldStarts[i], oldEnds[i]));
+ }
+ mappings.add(
+ new PartitionMapping(partition.partition, oldStarts, oldEnds, newStarts));
+ }
+
+ int rewrittenCount = 0;
+ for (boolean rewritten : rewrittenManifests) {
+ if (rewritten) {
+ rewrittenCount++;
+ }
+ }
+ int[] ordinals = new int[rewrittenCount];
+ int position = 0;
+ for (int i = 0; i < rewrittenManifests.length; i++) {
+ if (rewrittenManifests[i]) {
+ ordinals[position++] = i;
+ }
+ }
+ checkState(
+ ordinals.length > 0,
+ "Selected row-id mappings do not reference any manifest file.");
+ return new Result(ordinals, mappings, nextOffset);
+ }
- private int fileOrder(ManifestEntry entry) {
- if (isBlobFile(entry.file().fileName())) {
- return 1;
+ private void mergeSelectedPartition(GroupSelection selection) {
+ ByteArrayLookupKey lookup = new ByteArrayLookupKey(selection.partition.serialized);
+ SelectedPartition selected = selectedPartitions.get(lookup);
+ if (selected == null) {
+ LegacyPartitionOrderKey legacyOrderKey =
+ selection.partition.requiredLegacyOrderKey();
+ selected =
+ new SelectedPartition(
+ selection.partition.serialized,
+ selection.partition.partition,
+ legacyOrderKey,
+ selection.logicalRanges);
+ selectedPartitions.put(new ByteArrayKey(selection.partition.serialized), selected);
+ return;
+ }
+ LegacyPartitionOrderKey incomingOrderKey = selection.partition.requiredLegacyOrderKey();
+ if (incomingOrderKey.compareTo(selected.legacyOrderKey) < 0) {
+ selected.legacyOrderKey = incomingOrderKey;
+ }
+ selected.logicalRanges.append(selection.logicalRanges);
+ selected.logicalRanges.normalizeOverlapping();
}
- if (isVectorStoreFile(entry.file().fileName())) {
- return 2;
+
+ private void scan(
+ ManifestFileMeta manifestMeta, Projection projection, ProjectedRowVisitor visitor) {
+ try (RecordReader<InternalRow> reader =
+ FileUtils.createFormatReader(
+ table.fileIO(),
+ projection.readerFactory,
+ table.store().pathFactory().toManifestFilePath(manifestMeta.fileName()),
+ manifestMeta.fileSize())) {
+ boolean keepReading = true;
+ while (keepReading) {
+ RecordReader.RecordIterator<InternalRow> batch = reader.readBatch();
+ if (batch == null) {
+ break;
+ }
+ try {
+ InternalRow row;
+ while ((row = batch.next()) != null) {
+ validateManifestRow(row);
+ if (!visitor.visit(row)) {
+ keepReading = false;
+ break;
+ }
+ }
+ } finally {
+ batch.releaseBatch();
+ }
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException(
+ "Failed to read manifest file " + manifestMeta.fileName(), e);
+ }
+ }
+
+ private static void validateManifestRow(InternalRow row) {
+ checkState(
+ row.getInt(0) == MANIFEST_ENTRY_VERSION,
+ "Unsupported manifest entry version %s.",
+ row.getInt(0));
+ byte kind = row.getByte(1);
+ checkState(
+ kind == FileKind.ADD.toByteValue() || kind == FileKind.DELETE.toByteValue(),
+ "Unsupported manifest file kind %s.",
+ kind);
+ }
+
+ private static byte kind(InternalRow row) {
+ return row.getByte(1);
+ }
+
+ private static InternalRow file(InternalRow row, Projection projection) {
+ InternalRow file = row.getRow(projection.outerFilePosition, projection.fileFieldCount);
+ checkState(file != null, "Manifest data file metadata cannot be null.");
+ return file;
+ }
+
+ private static BinaryString requiredFileName(InternalRow file, Projection projection) {
+ checkState(
+ projection.fileNamePosition >= 0,
+ "The selected projection does not contain file name.");
+ BinaryString fileName = file.getString(projection.fileNamePosition);
+ checkState(fileName != null, "Manifest file name cannot be null.");
+ return fileName;
+ }
+
+ private static void readRowRange(
+ InternalRow file,
+ Projection projection,
+ int manifestOrdinal,
+ @Nullable BinaryString fileName,
+ long[] result) {
+ checkState(
+ projection.rowCountPosition >= 0 && projection.firstRowIdPosition >= 0,
+ "The selected projection does not contain a row-id range.");
+ checkState(
+ !file.isNullAt(projection.firstRowIdPosition),
+ fileName == null
+ ? "Manifest %s contains a file without first row id."
+ : "File '%s' does not have first row id.",
+ fileName == null ? manifestOrdinal : fileName);
+ long firstRowId = file.getLong(projection.firstRowIdPosition);
+ long rowCount = file.getLong(projection.rowCountPosition);
+ checkState(
+ rowCount > 0,
+ "Manifest %s contains a file with non-positive row count %s.",
+ manifestOrdinal,
+ rowCount);
+ result[0] = firstRowId;
+ result[1] = Math.addExact(firstRowId, rowCount - 1L);
+ }
+
+ private static boolean writesRowId(InternalRow file, Projection projection) {
+ checkState(
+ projection.writeColsPosition >= 0,
+ "The selected projection does not contain write columns.");
+ if (file.isNullAt(projection.writeColsPosition)) {
+ return false;
+ }
+ InternalArray writeCols = file.getArray(projection.writeColsPosition);
+ checkState(writeCols != null, "Manifest write columns cannot be null.");
+ for (int i = 0; i < writeCols.size(); i++) {
+ checkState(!writeCols.isNullAt(i), "Manifest write column cannot be null.");
+ if (ROW_ID_FIELD.equals(writeCols.getString(i))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private static long requiredMaxSequenceNumber(InternalRow file, Projection projection) {
+ checkState(
+ projection.maxSequenceNumberPosition >= 0,
+ "The selected projection does not contain max sequence number.");
+ checkState(
+ !file.isNullAt(projection.maxSequenceNumberPosition),
+ "Manifest max sequence number cannot be null.");
+ return file.getLong(projection.maxSequenceNumberPosition);
+ }
+
+ private static int fileOrder(BinaryString fileName) {
+ if (fileName.endsWith(BLOB_FILE_SUFFIX)) {
+ return 1;
+ }
+ if (fileName.contains(VECTOR_FILE_MARKER)) {
+ return 2;
+ }
+ return 0;
+ }
+
+ private static boolean isSpecialFile(BinaryString fileName) {
+ return fileOrder(fileName) != 0;
+ }
+
+ private int ordinal(ManifestFileMeta manifestMeta) {
+ Integer ordinal = manifestOrdinals.get(manifestMeta.fileName());
+ checkArgument(
+ ordinal != null,
+ "Planning group references unknown manifest '%s'.",
+ manifestMeta.fileName());
+ return ordinal;
+ }
+
+ private static int initialCurrentEntryCapacity(List<ManifestFileMeta> manifestGroup) {
+ long addedCount = 0L;
+ long deletedCount = 0L;
+ for (ManifestFileMeta manifestMeta : manifestGroup) {
+ checkState(
+ manifestMeta.numAddedFiles() >= 0 && manifestMeta.numDeletedFiles() >= 0,
+ "Manifest file counts cannot be negative.");
+ addedCount = Math.addExact(addedCount, manifestMeta.numAddedFiles());
+ deletedCount = Math.addExact(deletedCount, manifestMeta.numDeletedFiles());
+ }
+ return initialCurrentEntryCapacity(addedCount, deletedCount);
+ }
+
+ static int initialCurrentEntryCapacity(long addedCount, long deletedCount) {
+ checkArgument(addedCount >= 0, "Added entry count cannot be negative.");
+ checkArgument(deletedCount >= 0, "Deleted entry count cannot be negative.");
+ // Counts are only a sizing hint: DELETE entries may be duplicated or may not match an
+ // ADD
+ // in this group. Estimate the live set, cap the eager allocation, and let
+ // CurrentEntries
+ // grow if the actual number of retained ADD entries is larger.
+ long estimatedLiveCount = addedCount > deletedCount ? addedCount - deletedCount : 0L;
+ return (int) Math.min(estimatedLiveCount, MAX_INITIAL_CURRENT_ENTRIES);
+ }
+
+ void validateGroups(List<List<ManifestFileMeta>> groups) {
+ boolean[] seen = new boolean[manifestMetas.size()];
+ for (List<ManifestFileMeta> group : groups) {
+ checkArgument(group != null && !group.isEmpty(), "Manifest group cannot be empty.");
+ for (ManifestFileMeta manifestMeta : group) {
+ checkArgument(manifestMeta != null, "Manifest meta cannot be null.");
+ int ordinal = ordinal(manifestMeta);
+ checkArgument(
+ !seen[ordinal],
+ "Manifest '%s' occurs in more than one planning group.",
+ manifestMeta.fileName());
+ seen[ordinal] = true;
+ }
+ }
+ }
+
+ private static Map<String, Integer> manifestOrdinals(List<ManifestFileMeta> manifestMetas) {
+ Map<String, Integer> result = new HashMap<>();
+ for (int i = 0; i < manifestMetas.size(); i++) {
+ ManifestFileMeta manifestMeta = manifestMetas.get(i);
+ checkArgument(manifestMeta != null, "Manifest meta cannot be null.");
+ checkArgument(
+ result.put(manifestMeta.fileName(), i) == null,
+ "Duplicate manifest file '%s'.",
+ manifestMeta.fileName());
+ }
+ return result;
+ }
+
+ private static boolean rangesFullyCover(
+ long rangeStart, long rangeEnd, PrimitiveRangeBuffer mappings) {
+ long cursor = rangeStart;
+ for (int i = 0; i < mappings.size(); i++) {
+ long mappingStart = mappings.start(i);
+ long mappingEnd = mappings.end(i);
+ if (mappingEnd < cursor) {
+ continue;
+ }
+ if (mappingStart > cursor) {
+ return false;
+ }
+ long segmentEnd = Math.min(mappingEnd, rangeEnd);
+ if (segmentEnd == rangeEnd) {
+ return true;
+ }
+ if (segmentEnd == Long.MAX_VALUE) {
+ return false;
+ }
+ cursor = segmentEnd + 1L;
+ }
+ return false;
+ }
+
+ private static long inclusiveRangeCount(long start, long end) {
+ return Math.addExact(Math.subtractExact(end, start), 1L);
+ }
+
+ /**
+ * Compact planner result. All new row starts are relative to the snapshot's next row ID.
+ */
+ static final class Result {
+
+ final int[] manifestOrdinals;
+ final List<PartitionMapping> partitionMappings;
+ final long totalOffset;
+
+ private Result(
+ int[] manifestOrdinals,
+ List<PartitionMapping> partitionMappings,
+ long totalOffset) {
+ this.manifestOrdinals = manifestOrdinals;
+ this.partitionMappings =
+ Collections.unmodifiableList(new ArrayList<>(partitionMappings));
+ this.totalOffset = totalOffset;
+ }
+
+ boolean isEmpty() {
+ return partitionMappings.isEmpty();
+ }
+ }
+
+ /** Mapping arrays for one partition. Elements at the same index form one mapping. */
+ static final class PartitionMapping {
+
+ final BinaryRow partition;
+ final long[] oldStarts;
+ final long[] oldEnds;
+ final long[] newRelativeStarts;
+
+ private PartitionMapping(
+ BinaryRow partition,
+ long[] oldStarts,
+ long[] oldEnds,
+ long[] newRelativeStarts) {
+ this.partition = partition;
+ this.oldStarts = oldStarts;
+ this.oldEnds = oldEnds;
+ this.newRelativeStarts = newRelativeStarts;
+ }
+ }
+
+ private interface ProjectedRowVisitor {
+
+ boolean visit(InternalRow row);
}
- return 0;
- }
- private boolean isSpecialFile(ManifestEntry entry) {
- return isBlobFile(entry.file().fileName()) || isVectorStoreFile(entry.file().fileName());
+ private static final class Projection {
+
+ private final FormatReaderFactory readerFactory;
+ private final int outerFilePosition;
+ private final int bucketPosition;
+ private final int fileFieldCount;
+ private final int fileNamePosition;
+ private final int rowCountPosition;
+ private final int levelPosition;
+ private final int extraFilesPosition;
+ private final int embeddedFileIndexPosition;
+ private final int externalPathPosition;
+ private final int firstRowIdPosition;
+ private final int writeColsPosition;
+ private final int maxSequenceNumberPosition;
+
+ private Projection(
+ FormatReaderFactory readerFactory,
+ int outerFilePosition,
+ int bucketPosition,
+ int fileFieldCount,
+ int fileNamePosition,
+ int rowCountPosition,
+ int levelPosition,
+ int extraFilesPosition,
+ int embeddedFileIndexPosition,
+ int externalPathPosition,
+ int firstRowIdPosition,
+ int writeColsPosition,
+ int maxSequenceNumberPosition) {
+ this.readerFactory = readerFactory;
+ this.outerFilePosition = outerFilePosition;
+ this.bucketPosition = bucketPosition;
+ this.fileFieldCount = fileFieldCount;
+ this.fileNamePosition = fileNamePosition;
+ this.rowCountPosition = rowCountPosition;
+ this.levelPosition = levelPosition;
+ this.extraFilesPosition = extraFilesPosition;
+ this.embeddedFileIndexPosition = embeddedFileIndexPosition;
+ this.externalPathPosition = externalPathPosition;
+ this.firstRowIdPosition = firstRowIdPosition;
+ this.writeColsPosition = writeColsPosition;
+ this.maxSequenceNumberPosition = maxSequenceNumberPosition;
+ }
+
+ private static Projection create(
+ FileStoreTable table, boolean includeBucket, String... projectedFileFields) {
+ RowType fullType = VersionedObjectSerializer.versionType(ManifestEntry.SCHEMA);
+ RowType projectedFileType = DataFileMeta.SCHEMA.project(projectedFileFields);
+ List<DataField> fields = new ArrayList<>();
+ fields.add(fullType.getField("_VERSION"));
+ fields.add(fullType.getField("_KIND"));
+ fields.add(fullType.getField("_PARTITION"));
+ int bucketPosition = -1;
+ if (includeBucket) {
+ bucketPosition = fields.size();
+ fields.add(fullType.getField("_BUCKET"));
+ }
+ int outerFilePosition = fields.size();
+ fields.add(fullType.getField("_FILE").newType(projectedFileType));
+ RowType projectedType = new RowType(false, fields);
+ FileFormat format = FileFormat.manifestFormat(table.coreOptions());
+ FormatReaderFactory readerFactory =
+ format.createReaderFactory(
+ fullType, projectedType, Collections.emptyList());
+ return new Projection(
+ readerFactory,
+ outerFilePosition,
+ bucketPosition,
+ projectedFileFields.length,
+ position(projectedFileFields, "_FILE_NAME"),
+ position(projectedFileFields, "_ROW_COUNT"),
+ position(projectedFileFields, "_LEVEL"),
+ position(projectedFileFields, "_EXTRA_FILES"),
+ position(projectedFileFields, "_EMBEDDED_FILE_INDEX"),
+ position(projectedFileFields, "_EXTERNAL_PATH"),
+ position(projectedFileFields, "_FIRST_ROW_ID"),
+ position(projectedFileFields, "_WRITE_COLS"),
+ position(projectedFileFields, "_MAX_SEQUENCE_NUMBER"));
+ }
+
+ private static int position(String[] fields, String field) {
+ for (int i = 0; i < fields.length; i++) {
+ if (field.equals(fields[i])) {
+ return i;
+ }
+ }
+ return -1;
+ }
+ }
+
+ private static final class GroupState {
+
+ private final GroupPartitionDictionary partitions;
+ private final DeletedIdentifierSet deletedIdentifiers = new DeletedIdentifierSet();
+ private final CurrentEntries currentEntries;
+
+ private GroupState(
+ int partitionArity,
+ @Nullable PartitionPredicate partitionPredicate,
+ int excludedPartitionCacheSize,
+ int expectedAddEntryCount) {
+ this.partitions =
+ new GroupPartitionDictionary(
+ partitionArity, partitionPredicate, excludedPartitionCacheSize);
+ this.currentEntries = new CurrentEntries(expectedAddEntryCount);
+ }
+
+ private @Nullable PartitionState internPartition(byte[] serialized) {
+ return partitions.intern(serialized);
+ }
+
+ private void releaseDeletedIdentifiers() {
+ deletedIdentifiers.release();
+ }
+
+ private GroupSelection[] finishAddPass() {
+ currentEntries.sort();
+ GroupSelection[] selections = new GroupSelection[partitions.partitionCount()];
+ long[] rangeScratch = new long[2];
+ int groupStart = 0;
+ while (groupStart < currentEntries.size()) {
+ int partitionId = currentEntries.partitionId(groupStart);
+ int groupEnd = groupStart + 1;
+ while (groupEnd < currentEntries.size()
+ && currentEntries.partitionId(groupEnd) == partitionId) {
+ groupEnd++;
+ }
+ int rangeScan =
+ currentEntries.scanLogicalRanges(groupStart, groupEnd, rangeScratch);
+ if (rangeScan > 0) {
+ PrimitiveRangeBuffer logicalRanges =
+ currentEntries.materializeLogicalRanges(
+ groupStart, groupEnd, rangeScan, rangeScratch);
+ selections[partitionId] =
+ new GroupSelection(
+ partitions.partition(partitionId), logicalRanges);
+ }
+ groupStart = groupEnd;
+ }
+ currentEntries.release();
+ return selections;
+ }
+ }
+
+ private static final class GroupPartitionDictionary {
+
+ private final int expectedArity;
+ private final @Nullable PartitionPredicate partitionPredicate;
+ private final int excludedCacheSize;
+ private final Map<ByteArrayKey, PartitionState> included = new HashMap<>();
+ private final LinkedHashMap<ByteArrayKey, Boolean> excluded;
+ private final ByteArrayLookupKey lookup = new ByteArrayLookupKey();
+ private final List<PartitionState> partitions = new ArrayList<>();
+
+ private GroupPartitionDictionary(
+ int expectedArity,
+ @Nullable PartitionPredicate partitionPredicate,
+ int excludedCacheSize) {
+ checkArgument(excludedCacheSize >= 0, "Excluded cache size cannot be negative.");
+ this.expectedArity = expectedArity;
+ this.partitionPredicate = partitionPredicate;
+ this.excludedCacheSize = excludedCacheSize;
+ this.excluded =
+ new LinkedHashMap<ByteArrayKey, Boolean>(16, 0.75f, true) {
+ @Override
+ protected boolean removeEldestEntry(
+ Map.Entry<ByteArrayKey, Boolean> eldest) {
+ return size() > GroupPartitionDictionary.this.excludedCacheSize;
+ }
+ };
+ }
+
+ private @Nullable PartitionState intern(byte[] serialized) {
+ checkArgument(serialized != null, "Serialized partition cannot be null.");
+ lookup.reset(serialized);
+ try {
+ PartitionState existing = included.get(lookup);
+ if (existing != null) {
+ return existing;
+ }
+ if (partitionPredicate != null && excluded.get(lookup) != null) {
+ return null;
+ }
+
+ byte[] canonical = copyValidatedPartition(expectedArity, serialized);
+ BinaryRow partition = SerializationUtils.deserializeBinaryRow(canonical);
+ if (partitionPredicate != null && !partitionPredicate.test(partition)) {
+ if (excludedCacheSize > 0) {
+ excluded.put(new ByteArrayKey(canonical), Boolean.TRUE);
+ }
+ return null;
+ }
+
+ checkState(
+ partitions.size() < Integer.MAX_VALUE,
+ "Too many partitions in one manifest group.");
+ PartitionState created =
+ new PartitionState(partitions.size(), canonical, partition);
+ partitions.add(created);
+ included.put(new ByteArrayKey(canonical), created);
+ return created;
+ } finally {
+ lookup.clear();
+ }
+ }
+
+ private int partitionCount() {
+ return partitions.size();
+ }
+
+ private PartitionState partition(int partitionId) {
+ return partitions.get(partitionId);
+ }
+ }
+
+ private static byte[] copyValidatedPartition(int expectedArity, byte[] serialized) {
+ checkArgument(serialized.length >= 4, "Serialized partition is truncated.");
+ int arity =
+ ((serialized[0] & 0xFF) << 24)
+ | ((serialized[1] & 0xFF) << 16)
+ | ((serialized[2] & 0xFF) << 8)
+ | (serialized[3] & 0xFF);
+ checkArgument(
+ arity == expectedArity,
+ "Serialized partition has arity %s, expected %s.",
+ arity,
+ expectedArity);
+ int fixedLength = BinaryRow.calculateFixPartSizeInBytes(arity);
+ checkArgument(
+ serialized.length - 4 >= fixedLength,
+ "Serialized partition payload is truncated.");
+ return Arrays.copyOf(serialized, serialized.length);
+ }
+
+ private static final class PartitionState {
+
+ private final int id;
+ private final byte[] serialized;
+ private final BinaryRow partition;
+ private @Nullable LegacyPartitionOrderKey legacyOrderKey;
+
+ private PartitionState(int id, byte[] serialized, BinaryRow partition) {
+ this.id = id;
+ this.serialized = serialized;
+ this.partition = partition;
+ }
+
+ private void considerLegacyOrderKey(
+ long manifestGroupOrdinal,
+ long firstRowId,
+ int fileOrder,
+ long maxSequenceNumber,
+ BinaryString fileName,
+ long retainedAddScanOrdinal) {
+ if (legacyOrderKey == null) {
+ legacyOrderKey =
+ new LegacyPartitionOrderKey(
+ manifestGroupOrdinal,
+ firstRowId,
+ fileOrder,
+ maxSequenceNumber,
+ fileName.toString(),
+ retainedAddScanOrdinal);
+ return;
+ }
+
+ int comparison =
+ Long.compare(manifestGroupOrdinal, legacyOrderKey.manifestGroupOrdinal);
+ if (comparison == 0) {
+ comparison = Long.compare(firstRowId, legacyOrderKey.firstRowId);
+ }
+ if (comparison == 0) {
+ comparison = Integer.compare(fileOrder, legacyOrderKey.fileOrder);
+ }
+ if (comparison == 0) {
+ comparison = Long.compare(legacyOrderKey.maxSequenceNumber, maxSequenceNumber);
+ }
+
+ String stableFileName = null;
+ if (comparison == 0) {
+ stableFileName = fileName.toString();
+ comparison = stableFileName.compareTo(legacyOrderKey.fileName);
+ }
+ if (comparison == 0) {
+ comparison =
+ Long.compare(
+ retainedAddScanOrdinal, legacyOrderKey.retainedAddScanOrdinal);
+ }
+ if (comparison < 0) {
+ if (stableFileName == null) {
+ stableFileName = fileName.toString();
+ }
+ legacyOrderKey =
+ new LegacyPartitionOrderKey(
+ manifestGroupOrdinal,
+ firstRowId,
+ fileOrder,
+ maxSequenceNumber,
+ stableFileName,
+ retainedAddScanOrdinal);
+ }
+ }
+
+ private LegacyPartitionOrderKey requiredLegacyOrderKey() {
+ checkState(
+ legacyOrderKey != null,
+ "Selected partition does not have a retained ADD ordering key.");
+ return legacyOrderKey;
+ }
+ }
+
+ private static final class LegacyPartitionOrderKey
+ implements Comparable<LegacyPartitionOrderKey> {
+
+ private final long manifestGroupOrdinal;
+ private final long firstRowId;
+ private final int fileOrder;
+ private final long maxSequenceNumber;
+ private final String fileName;
+ private final long retainedAddScanOrdinal;
+
+ private LegacyPartitionOrderKey(
+ long manifestGroupOrdinal,
+ long firstRowId,
+ int fileOrder,
+ long maxSequenceNumber,
+ String fileName,
+ long retainedAddScanOrdinal) {
+ this.manifestGroupOrdinal = manifestGroupOrdinal;
+ this.firstRowId = firstRowId;
+ this.fileOrder = fileOrder;
+ this.maxSequenceNumber = maxSequenceNumber;
+ this.fileName = fileName;
+ this.retainedAddScanOrdinal = retainedAddScanOrdinal;
+ }
+
+ @Override
+ public int compareTo(LegacyPartitionOrderKey other) {
+ int comparison = Long.compare(manifestGroupOrdinal, other.manifestGroupOrdinal);
+ if (comparison != 0) {
+ return comparison;
+ }
+ comparison = Long.compare(firstRowId, other.firstRowId);
+ if (comparison != 0) {
+ return comparison;
+ }
+ comparison = Integer.compare(fileOrder, other.fileOrder);
+ if (comparison != 0) {
+ return comparison;
+ }
+ comparison = Long.compare(other.maxSequenceNumber, maxSequenceNumber);
+ if (comparison != 0) {
+ return comparison;
+ }
+ comparison = fileName.compareTo(other.fileName);
+ return comparison != 0
+ ? comparison
+ : Long.compare(retainedAddScanOrdinal, other.retainedAddScanOrdinal);
+ }
+ }
+
+ private static final class GroupSelection {
+
+ private final PartitionState partition;
+ private final PrimitiveRangeBuffer logicalRanges;
+
+ private GroupSelection(PartitionState partition, PrimitiveRangeBuffer logicalRanges) {
+ this.partition = partition;
+ this.logicalRanges = logicalRanges;
+ }
+ }
+
+ private static final class SelectedPartition {
+
+ private final byte[] serializedPartition;
+ private final BinaryRow partition;
+ private LegacyPartitionOrderKey legacyOrderKey;
+ private final PrimitiveRangeBuffer logicalRanges;
+
+ private SelectedPartition(
+ byte[] serializedPartition,
+ BinaryRow partition,
+ LegacyPartitionOrderKey legacyOrderKey,
+ PrimitiveRangeBuffer logicalRanges) {
+ this.serializedPartition = serializedPartition;
+ this.partition = partition;
+ this.legacyOrderKey = legacyOrderKey;
+ this.logicalRanges = logicalRanges;
+ }
+ }
+
+ /** Object-free storage for current entries. */
+ static final class CurrentEntries {
+
+ private long[] words;
+ private int size;
+
+ CurrentEntries() {
+ this(0);
+ }
+
+ CurrentEntries(int expectedEntries) {
+ checkArgument(
+ expectedEntries >= 0, "Expected current entry count cannot be negative.");
+ this.words = new long[Math.multiplyExact(expectedEntries, CURRENT_ENTRY_WORDS)];
+ }
+
+ void add(int partitionId, boolean special, long firstRowId, long rowCount) {
+ checkArgument(partitionId >= 0, "Partition id cannot be negative.");
+ checkArgument(rowCount > 0, "Row count must be positive.");
+ Math.addExact(firstRowId, rowCount - 1L);
+ ensureCapacity(Math.addExact(size, 1));
+ int offset = size * CURRENT_ENTRY_WORDS;
+ words[offset] =
+ Integer.toUnsignedLong(partitionId) | (special ? CURRENT_SPECIAL : 0L);
+ words[offset + 1] = firstRowId;
+ words[offset + 2] = rowCount;
+ size++;
+ }
+
+ int size() {
+ return size;
+ }
+
+ int retainedWordCount() {
+ return words.length;
+ }
+
+ int usedWordCount() {
+ return size * CURRENT_ENTRY_WORDS;
+ }
+
+ private void release() {
+ words = new long[0];
+ size = 0;
+ }
+
+ int partitionId(int index) {
+ return (int) words[offset(index)];
+ }
+
+ private boolean special(int index) {
+ return (words[offset(index)] & CURRENT_SPECIAL) != 0;
+ }
+
+ private long firstRowId(int index) {
+ return words[offset(index) + 1];
+ }
+
+ private long rowCount(int index) {
+ return words[offset(index) + 2];
+ }
+
+ private long lastRowId(int index) {
+ return firstRowId(index) + rowCount(index) - 1L;
+ }
+
+ private int offset(int index) {
+ checkArgument(index >= 0 && index < size, "Current entry index is out of bounds.");
+ return index * CURRENT_ENTRY_WORDS;
+ }
+
+ private void ensureCapacity(int requiredEntries) {
+ long requiredWords = (long) requiredEntries * CURRENT_ENTRY_WORDS;
+ checkState(
+ requiredWords <= Integer.MAX_VALUE,
+ "Too many current entries in one manifest group.");
+ if (requiredWords <= words.length) {
+ return;
+ }
+ int newLength = Math.max(48, words.length);
+ while (newLength < requiredWords) {
+ int grown = newLength + (newLength >>> 1);
+ if (grown <= newLength || grown > Integer.MAX_VALUE) {
+ newLength = (int) requiredWords;
+ break;
+ }
+ newLength = grown;
+ }
+ words = Arrays.copyOf(words, newLength);
+ }
+
+ private void sort() {
+ if (size > 1) {
+ sort(0, size - 1);
+ }
+ }
+
+ private void sort(int left, int right) {
+ while (left < right) {
+ int middle = left + ((right - left) >>> 1);
+ long pivotPartition = words[middle * CURRENT_ENTRY_WORDS];
+ long pivotFirst = words[middle * CURRENT_ENTRY_WORDS + 1];
+ long pivotCount = words[middle * CURRENT_ENTRY_WORDS + 2];
+ int lower = left;
+ int current = left;
+ int upper = right;
+ while (current <= upper) {
+ int comparison = compare(current, pivotPartition, pivotFirst, pivotCount);
+ if (comparison < 0) {
+ swap(lower++, current++);
+ } else if (comparison > 0) {
+ swap(current, upper--);
+ } else {
+ current++;
+ }
+ }
+
+ if (lower - left < right - upper) {
+ if (left < lower - 1) {
+ sort(left, lower - 1);
+ }
+ left = upper + 1;
+ } else {
+ if (upper + 1 < right) {
+ sort(upper + 1, right);
+ }
+ right = lower - 1;
+ }
+ }
+ }
+
+ private int compare(int index, long pivotPartition, long pivotFirst, long pivotCount) {
+ int offset = index * CURRENT_ENTRY_WORDS;
+ int result =
+ Long.compare(words[offset] & 0xFFFF_FFFFL, pivotPartition & 0xFFFF_FFFFL);
+ if (result != 0) {
+ return result;
+ }
+ result = Long.compare(words[offset + 1], pivotFirst);
+ if (result != 0) {
+ return result;
+ }
+ long end = words[offset + 1] + words[offset + 2] - 1L;
+ long pivotEnd = pivotFirst + pivotCount - 1L;
+ return Long.compare(end, pivotEnd);
+ }
+
+ private void swap(int left, int right) {
+ if (left == right) {
+ return;
+ }
+ int leftOffset = left * CURRENT_ENTRY_WORDS;
+ int rightOffset = right * CURRENT_ENTRY_WORDS;
+ for (int i = 0; i < CURRENT_ENTRY_WORDS; i++) {
+ long value = words[leftOffset + i];
+ words[leftOffset + i] = words[rightOffset + i];
+ words[rightOffset + i] = value;
+ }
+ }
+
+ /**
+ * Scans logical ranges without retaining one object (or even one primitive pair) per
+ * range.
+ *
+ * <p>The absolute return value is the number of logical ranges. A negative result means
+ * that all logical ranges are contiguous and therefore this partition does not need a
+ * plan. A positive result means that the ranges are fragmented and need
+ * materialization.
+ */
+ private int scanLogicalRanges(int from, int to, long[] rangeScratch) {
+ checkArgument(from >= 0 && from < to && to <= size, "Invalid entry slice.");
+ int overlapStart = from;
+ long currentEnd = lastRowId(from);
+ int rangeCount = 0;
+ boolean contiguous = true;
+ boolean hasPrevious = false;
+ long previousEnd = 0L;
+ for (int i = from + 1; i < to; i++) {
+ if (firstRowId(i) <= currentEnd) {
+ currentEnd = Math.max(currentEnd, lastRowId(i));
+ } else {
+ computeLogicalRange(overlapStart, i, rangeScratch);
+ rangeCount++;
+ if (hasPrevious
+ && (previousEnd == Long.MAX_VALUE
+ || rangeScratch[0] != previousEnd + 1L)) {
+ contiguous = false;
+ }
+ previousEnd = rangeScratch[1];
+ hasPrevious = true;
+ overlapStart = i;
+ currentEnd = lastRowId(i);
+ }
+ }
+ computeLogicalRange(overlapStart, to, rangeScratch);
+ rangeCount++;
+ if (hasPrevious
+ && (previousEnd == Long.MAX_VALUE || rangeScratch[0] != previousEnd + 1L)) {
+ contiguous = false;
+ }
+ return contiguous ? -rangeCount : rangeCount;
+ }
+
+ private PrimitiveRangeBuffer materializeLogicalRanges(
+ int from, int to, int expectedRangeCount, long[] rangeScratch) {
+ checkArgument(
+ from >= 0 && from < to && to <= size && expectedRangeCount > 0,
+ "Invalid fragmented entry slice.");
+ PrimitiveRangeBuffer ranges = new PrimitiveRangeBuffer(expectedRangeCount);
+ int overlapStart = from;
+ long currentEnd = lastRowId(from);
+ for (int i = from + 1; i < to; i++) {
+ if (firstRowId(i) <= currentEnd) {
+ currentEnd = Math.max(currentEnd, lastRowId(i));
+ } else {
+ computeLogicalRange(overlapStart, i, rangeScratch);
+ ranges.add(rangeScratch[0], rangeScratch[1]);
+ overlapStart = i;
+ currentEnd = lastRowId(i);
+ }
+ }
+ computeLogicalRange(overlapStart, to, rangeScratch);
+ ranges.add(rangeScratch[0], rangeScratch[1]);
+ checkState(
+ ranges.size() == expectedRangeCount,
+ "Logical range count changed between scan and materialization.");
+ return ranges;
+ }
+
+ private void computeLogicalRange(int from, int to, long[] result) {
+ boolean hasOrdinary = false;
+ long ordinaryStart = 0L;
+ long ordinaryEnd = 0L;
+ long spanningStart = Long.MAX_VALUE;
+ long spanningEnd = Long.MIN_VALUE;
+ for (int i = from; i < to; i++) {
+ long start = firstRowId(i);
+ long end = lastRowId(i);
+ spanningStart = Math.min(spanningStart, start);
+ spanningEnd = Math.max(spanningEnd, end);
+ if (!special(i)) {
+ checkState(
+ !hasOrdinary || (ordinaryStart == start && ordinaryEnd == end),
+ "Data files in one overlapping row-id group must have the same row-id range.");
+ ordinaryStart = start;
+ ordinaryEnd = end;
+ hasOrdinary = true;
+ }
+ }
+ long logicalStart = hasOrdinary ? ordinaryStart : spanningStart;
+ long logicalEnd = hasOrdinary ? ordinaryEnd : spanningEnd;
+ for (int i = from; i < to; i++) {
+ checkState(
+ firstRowId(i) >= logicalStart && lastRowId(i) <= logicalEnd,
+ "File row-id range is outside its logical row-id range.");
+ }
+ result[0] = logicalStart;
+ result[1] = logicalEnd;
+ }
+
+ @Nullable
+ PrimitiveRangeBuffer selectedRangesForTesting() {
+ checkState(size > 0, "Cannot inspect an empty current-entry buffer.");
+ sort();
+ int partitionId = partitionId(0);
+ for (int i = 1; i < size; i++) {
+ checkState(
+ partitionId(i) == partitionId,
+ "The structural range test helper requires one partition.");
+ }
+ long[] rangeScratch = new long[2];
+ int rangeScan = scanLogicalRanges(0, size, rangeScratch);
+ return rangeScan < 0
+ ? null
+ : materializeLogicalRanges(0, size, rangeScan, rangeScratch);
+ }
+ }
+
+ /** Compact, collision-safe set backed by primitive arrays and one identifier byte arena. */
+ static final class DeletedIdentifierSet {
+
+ private static final float LOAD_FACTOR = 0.75f;
+
+ private int[] buckets = filledWithMinusOne(16);
+ private long[] hashes = new long[16];
+ private int[] partitionIds = new int[16];
+ private int[] offsets = new int[16];
+ private int[] lengths = new int[16];
+ private int[] next = new int[16];
+ private byte[] arena = new byte[256];
+ private int arenaSize;
+ private int size;
+
+ boolean isEmpty() {
+ return size == 0;
+ }
+
+ int size() {
+ return size;
+ }
+
+ int retainedIdentifierBytes() {
+ return arenaSize;
+ }
+
+ private void release() {
+ buckets = filledWithMinusOne(16);
+ hashes = new long[0];
+ partitionIds = new int[0];
+ offsets = new int[0];
+ lengths = new int[0];
+ next = new int[0];
+ arena = new byte[0];
+ arenaSize = 0;
+ size = 0;
+ }
+
+ void add(int partitionId, byte[] identifier, int length) {
+ checkIdentifier(identifier, length);
+ long hash = hash(partitionId, identifier, length);
+ if (contains(partitionId, identifier, length, hash)) {
+ return;
+ }
+ if (size + 1 > (int) (buckets.length * LOAD_FACTOR)) {
+ growBuckets();
+ }
+ ensureEntryCapacity(size + 1);
+ ensureArenaCapacity(length);
+ int offset = arenaSize;
+ System.arraycopy(identifier, 0, arena, offset, length);
+ arenaSize = Math.addExact(arenaSize, length);
+
+ int bucket = bucket(hash);
+ hashes[size] = hash;
+ partitionIds[size] = partitionId;
+ offsets[size] = offset;
+ lengths[size] = length;
+ next[size] = buckets[bucket];
+ buckets[bucket] = size;
+ size++;
+ }
+
+ boolean contains(int partitionId, byte[] identifier, int length) {
+ checkIdentifier(identifier, length);
+ return contains(
+ partitionId, identifier, length, hash(partitionId, identifier, length));
+ }
+
+ private boolean contains(int partitionId, byte[] identifier, int length, long hash) {
+ for (int entry = buckets[bucket(hash)]; entry >= 0; entry = next[entry]) {
+ if (hashes[entry] == hash
+ && partitionIds[entry] == partitionId
+ && lengths[entry] == length
+ && bytesEqual(arena, offsets[entry], identifier, length)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ private void growBuckets() {
+ checkState(
+ buckets.length < (1 << 30),
+ "Too many deleted identifiers in one manifest group.");
+ int[] grown = filledWithMinusOne(buckets.length << 1);
+ for (int entry = 0; entry < size; entry++) {
+ int bucket = bucket(hashes[entry], grown.length);
+ next[entry] = grown[bucket];
+ grown[bucket] = entry;
+ }
+ buckets = grown;
+ }
+
+ private void ensureEntryCapacity(int required) {
+ if (required <= hashes.length) {
+ return;
+ }
+ int grown = Math.max(required, hashes.length + (hashes.length >>> 1));
+ hashes = Arrays.copyOf(hashes, grown);
+ partitionIds = Arrays.copyOf(partitionIds, grown);
+ offsets = Arrays.copyOf(offsets, grown);
+ lengths = Arrays.copyOf(lengths, grown);
+ next = Arrays.copyOf(next, grown);
+ }
+
+ private void ensureArenaCapacity(int additional) {
+ int required;
+ try {
+ required = Math.addExact(arenaSize, additional);
+ } catch (ArithmeticException e) {
+ throw new IllegalStateException(
+ "Deleted identifier arena exceeds the Java array limit.", e);
+ }
+ if (required <= arena.length) {
+ return;
+ }
+ int grown = Math.max(required, arena.length + (arena.length >>> 1));
+ if (grown < 0) {
+ grown = required;
+ }
+ arena = Arrays.copyOf(arena, grown);
+ }
+
+ private int bucket(long hash) {
+ return bucket(hash, buckets.length);
+ }
+
+ private static int bucket(long hash, int bucketCount) {
+ return ((int) (hash ^ (hash >>> 32))) & (bucketCount - 1);
+ }
+
+ private static long hash(int partitionId, byte[] bytes, int length) {
+ long hash = 0xcbf29ce484222325L;
+ hash ^= Integer.toUnsignedLong(partitionId);
+ hash *= 0x100000001b3L;
+ for (int i = 0; i < length; i++) {
+ hash ^= bytes[i] & 0xFFL;
+ hash *= 0x100000001b3L;
+ }
+ return hash;
+ }
+
+ private static boolean bytesEqual(
+ byte[] left, int leftOffset, byte[] right, int length) {
+ for (int i = 0; i < length; i++) {
+ if (left[leftOffset + i] != right[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static void checkIdentifier(byte[] identifier, int length) {
+ checkArgument(identifier != null, "Identifier bytes cannot be null.");
+ checkArgument(
+ length >= 0 && length <= identifier.length,
+ "Invalid identifier length %s.",
+ length);
+ }
+
+ private static int[] filledWithMinusOne(int length) {
+ int[] values = new int[length];
+ Arrays.fill(values, -1);
+ return values;
+ }
+ }
+
+ private static final class IdentifierScratch {
+
+ private byte[] bytes = new byte[256];
+ private int length;
+
+ private void encode(InternalRow outer, InternalRow file, Projection projection) {
+ checkState(
+ projection.bucketPosition >= 0
+ && projection.fileNamePosition >= 0
+ && projection.levelPosition >= 0
+ && projection.extraFilesPosition >= 0
+ && projection.embeddedFileIndexPosition >= 0
+ && projection.externalPathPosition >= 0,
+ "The selected projection does not contain a complete file identifier.");
+ length = 0;
+ putInt(outer.getInt(projection.bucketPosition));
+ putInt(file.getInt(projection.levelPosition));
+ putString(requiredFileName(file, projection));
+
+ InternalArray extraFiles = file.getArray(projection.extraFilesPosition);
+ checkState(extraFiles != null, "Manifest extra files cannot be null.");
+ putInt(extraFiles.size());
+ for (int i = 0; i < extraFiles.size(); i++) {
+ checkState(!extraFiles.isNullAt(i), "Extra file name cannot be null.");
+ putString(extraFiles.getString(i));
+ }
+
+ if (file.isNullAt(projection.embeddedFileIndexPosition)) {
+ putInt(-1);
+ } else {
+ putBytes(file.getBinary(projection.embeddedFileIndexPosition));
+ }
+ if (file.isNullAt(projection.externalPathPosition)) {
+ putInt(-1);
+ } else {
+ putString(file.getString(projection.externalPathPosition));
+ }
+ }
+
+ private byte[] bytes() {
+ return bytes;
+ }
+
+ private int length() {
+ return length;
+ }
+
+ private void release() {
+ bytes = new byte[0];
+ length = 0;
+ }
+
+ private void putString(BinaryString value) {
+ checkState(value != null, "Manifest string field cannot be null.");
+ int valueLength = value.getSizeInBytes();
+ putInt(valueLength);
+ ensureCapacity(valueLength);
+ MemorySegmentUtils.copyToBytes(
+ value.getSegments(), value.getOffset(), bytes, length, valueLength);
+ length += valueLength;
+ }
+
+ private void putBytes(byte[] value) {
+ checkState(value != null, "Manifest binary field cannot be null.");
+ putInt(value.length);
+ ensureCapacity(value.length);
+ System.arraycopy(value, 0, bytes, length, value.length);
+ length += value.length;
+ }
+
+ private void putInt(int value) {
+ ensureCapacity(Integer.BYTES);
+ bytes[length++] = (byte) (value >>> 24);
+ bytes[length++] = (byte) (value >>> 16);
+ bytes[length++] = (byte) (value >>> 8);
+ bytes[length++] = (byte) value;
+ }
+
+ private void ensureCapacity(int additional) {
+ int required = Math.addExact(length, additional);
+ if (required <= bytes.length) {
+ return;
+ }
+ int grown = Math.max(required, bytes.length + (bytes.length >>> 1));
+ bytes = Arrays.copyOf(bytes, grown);
+ }
+ }
+
+ /**
+ * Object-free logical range storage.
+ *
+ * <p>Starts and ends are kept in separate primitive arrays so that the planner can transfer
+ * ownership of both arrays directly into the final mapping. The common one-group path
+ * allocates the exact range count and requires no copy during that transfer.
+ */
+ static final class PrimitiveRangeBuffer {
+
+ private long[] starts;
+ private long[] ends;
+ private int size;
+ private boolean sorted = true;
+
+ private PrimitiveRangeBuffer(int expectedRanges) {
+ checkArgument(expectedRanges >= 0, "Expected range count cannot be negative.");
+ starts = new long[expectedRanges];
+ ends = new long[expectedRanges];
+ }
+
+ int size() {
+ return size;
+ }
+
+ int retainedWordCount() {
+ return Math.addExact(starts.length, ends.length);
+ }
+
+ long start(int index) {
+ checkIndex(index);
+ return starts[index];
+ }
+
+ long end(int index) {
+ checkIndex(index);
+ return ends[index];
+ }
+
+ private void add(long start, long end) {
+ checkArgument(start <= end, "Invalid row-id range [%s, %s].", start, end);
+ ensureCapacity(Math.addExact(size, 1));
+ if (size > 0 && compare(starts[size - 1], ends[size - 1], start, end) > 0) {
+ sorted = false;
+ }
+ starts[size] = start;
+ ends[size] = end;
+ size++;
+ }
+
+ private void append(PrimitiveRangeBuffer other) {
+ checkArgument(other != null, "Ranges to append cannot be null.");
+ if (other.size == 0) {
+ return;
+ }
+ int oldSize = size;
+ int combinedSize = Math.addExact(size, other.size);
+ ensureCapacity(combinedSize);
+ if (oldSize > 0
+ && compare(
+ starts[oldSize - 1],
+ ends[oldSize - 1],
+ other.starts[0],
+ other.ends[0])
+ > 0) {
+ sorted = false;
+ }
+ sorted &= other.sorted;
+ System.arraycopy(other.starts, 0, starts, oldSize, other.size);
+ System.arraycopy(other.ends, 0, ends, oldSize, other.size);
+ size = combinedSize;
+ }
+
+ private void normalizeOverlapping() {
+ if (size <= 1) {
+ sorted = true;
+ return;
+ }
+ if (!sorted) {
+ sort(0, size - 1);
+ sorted = true;
+ }
+ int writeIndex = 0;
+ for (int readIndex = 1; readIndex < size; readIndex++) {
+ if (starts[readIndex] <= ends[writeIndex]) {
+ ends[writeIndex] = Math.max(ends[writeIndex], ends[readIndex]);
+ } else {
+ writeIndex++;
+ starts[writeIndex] = starts[readIndex];
+ ends[writeIndex] = ends[readIndex];
+ }
+ }
+ size = writeIndex + 1;
+ }
+
+ private OwnedPrimitiveRanges takeOwned() {
+ long[] ownedStarts = starts.length == size ? starts : Arrays.copyOf(starts, size);
+ long[] ownedEnds = ends.length == size ? ends : Arrays.copyOf(ends, size);
+ starts = new long[0];
+ ends = new long[0];
+ size = 0;
+ sorted = true;
+ return new OwnedPrimitiveRanges(ownedStarts, ownedEnds);
+ }
+
+ private void ensureCapacity(int required) {
+ if (required <= starts.length) {
+ return;
+ }
+ int grown = Math.max(16, starts.length);
+ while (grown < required) {
+ int next = grown + (grown >>> 1);
+ if (next <= grown || next < 0) {
+ grown = required;
+ break;
+ }
+ grown = next;
+ }
+ starts = Arrays.copyOf(starts, grown);
+ ends = Arrays.copyOf(ends, grown);
+ }
+
+ private void sort(int left, int right) {
+ while (left < right) {
+ int middle = left + ((right - left) >>> 1);
+ long pivotStart = starts[middle];
+ long pivotEnd = ends[middle];
+ int lower = left;
+ int current = left;
+ int upper = right;
+ while (current <= upper) {
+ int comparison =
+ compare(starts[current], ends[current], pivotStart, pivotEnd);
+ if (comparison < 0) {
+ swap(lower++, current++);
+ } else if (comparison > 0) {
+ swap(current, upper--);
+ } else {
+ current++;
+ }
+ }
+
+ if (lower - left < right - upper) {
+ if (left < lower - 1) {
+ sort(left, lower - 1);
+ }
+ left = upper + 1;
+ } else {
+ if (upper + 1 < right) {
+ sort(upper + 1, right);
+ }
+ right = lower - 1;
+ }
+ }
+ }
+
+ private void swap(int left, int right) {
+ if (left == right) {
+ return;
+ }
+ long start = starts[left];
+ long end = ends[left];
+ starts[left] = starts[right];
+ ends[left] = ends[right];
+ starts[right] = start;
+ ends[right] = end;
+ }
+
+ private void checkIndex(int index) {
+ checkArgument(index >= 0 && index < size, "Logical range index is out of bounds.");
+ }
+
+ private static int compare(
+ long leftStart, long leftEnd, long rightStart, long rightEnd) {
+ int result = Long.compare(leftStart, rightStart);
+ return result == 0 ? Long.compare(leftEnd, rightEnd) : result;
+ }
+ }
+
+ private static final class OwnedPrimitiveRanges {
+
+ private final long[] starts;
+ private final long[] ends;
+
+ private OwnedPrimitiveRanges(long[] starts, long[] ends) {
+ this.starts = starts;
+ this.ends = ends;
+ }
+ }
+
+ private static final class ByteArrayKey {
+
+ private final byte[] bytes;
+ private final int hash;
+
+ private ByteArrayKey(byte[] bytes) {
+ this.bytes = bytes;
+ this.hash = Arrays.hashCode(bytes);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj == this
+ || (obj instanceof ByteArrayKey
+ && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes))
+ || (obj instanceof ByteArrayLookupKey
+ && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes));
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+ }
+
+ private static final class ByteArrayLookupKey {
+
+ private @Nullable byte[] bytes;
+ private int hash;
+
+ private ByteArrayLookupKey() {}
+
+ private ByteArrayLookupKey(byte[] bytes) {
+ reset(bytes);
+ }
+
+ private void reset(byte[] bytes) {
+ this.bytes = bytes;
+ this.hash = Arrays.hashCode(bytes);
+ }
+
+ private void clear() {
+ bytes = null;
+ hash = 0;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ return obj == this
+ || (bytes != null
+ && obj instanceof ByteArrayKey
+ && Arrays.equals(bytes, ((ByteArrayKey) obj).bytes))
+ || (bytes != null
+ && obj instanceof ByteArrayLookupKey
+ && Arrays.equals(bytes, ((ByteArrayLookupKey) obj).bytes));
+ }
+
+ @Override
+ public int hashCode() {
+ return hash;
+ }
+ }
}
/** Result of row-id reassignment. */
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndex.java b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndex.java
index 7977c9295c..c4c0f07b8c 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndex.java
+++ b/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndex.java
@@ -31,16 +31,17 @@ import static org.apache.paimon.utils.Preconditions.checkArgument;
/** Index for row-range mappings. */
final class RowRangeMappingIndex {
- private final List<Mapping> mappings;
+ private final long[] oldStarts;
private final long[] oldEnds;
-
- private RowRangeMappingIndex(List<Mapping> mappings) {
- this.mappings = mappings;
- this.oldEnds = new long[mappings.size()];
- for (int i = 0; i < mappings.size(); i++) {
- Mapping mapping = mappings.get(i);
- oldEnds[i] = mapping.oldEnd;
- }
+ private final long[] newStarts;
+ private final long newStartOffset;
+
+ private RowRangeMappingIndex(
+ long[] oldStarts, long[] oldEnds, long[] newStarts, long newStartOffset) {
+ this.oldStarts = oldStarts;
+ this.oldEnds = oldEnds;
+ this.newStarts = newStarts;
+ this.newStartOffset = newStartOffset;
}
static RowRangeMappingIndex create(List<Mapping> mappings) {
@@ -49,21 +50,48 @@ final class RowRangeMappingIndex {
List<Mapping> sorted = new ArrayList<>(mappings);
Collections.sort(sorted, Comparator.comparingLong(mapping -> mapping.oldStart));
- Mapping previous = null;
- for (Mapping mapping : sorted) {
+ long[] oldStarts = new long[sorted.size()];
+ long[] oldEnds = new long[sorted.size()];
+ long[] newStarts = new long[sorted.size()];
+ for (int i = 0; i < sorted.size(); i++) {
+ Mapping mapping = sorted.get(i);
+ oldStarts[i] = mapping.oldStart;
+ oldEnds[i] = mapping.oldEnd;
+ newStarts[i] = mapping.newStart;
+ }
+ validate(oldStarts, oldEnds, newStarts);
+ return new RowRangeMappingIndex(oldStarts, oldEnds, newStarts, 0L);
+ }
+
+ /** Creates an index by taking ownership of the three arrays. */
+ static RowRangeMappingIndex createFromOwnedArrays(
+ long[] oldStarts, long[] oldEnds, long[] newStarts) {
+ checkArgument(oldStarts != null, "Old row range starts cannot be null.");
+ checkArgument(oldEnds != null, "Old row range ends cannot be null.");
+ checkArgument(newStarts != null, "New row range starts cannot be null.");
+
+ validate(oldStarts, oldEnds, newStarts);
+ return new RowRangeMappingIndex(oldStarts, oldEnds, newStarts, 0L);
+ }
+
+ private static void validate(long[] oldStarts, long[] oldEnds, long[] newStarts) {
+ checkArgument(
+ oldStarts.length == oldEnds.length && oldStarts.length == newStarts.length,
+ "Row range mapping arrays must have the same length.");
+ checkArgument(oldStarts.length > 0, "Row range mappings cannot be empty.");
+
+ for (int i = 0; i < oldStarts.length; i++) {
checkArgument(
- mapping.oldStart <= mapping.oldEnd,
+ oldStarts[i] <= oldEnds[i],
"Invalid old row range [%s, %s].",
- mapping.oldStart,
- mapping.oldEnd);
- if (previous != null) {
+ oldStarts[i],
+ oldEnds[i]);
+ if (i > 0) {
checkArgument(
- previous.oldEnd < mapping.oldStart,
- "Old row range mappings cannot overlap.");
+ oldEnds[i - 1] < oldStarts[i],
+ "Old row range mappings cannot overlap or be out of order.");
}
- previous = mapping;
}
- return new RowRangeMappingIndex(Collections.unmodifiableList(sorted));
}
static Mapping mapping(long oldStart, long oldEnd, long newStart) {
@@ -71,15 +99,26 @@ final class RowRangeMappingIndex {
}
RowRangeMappingIndex shiftNewStarts(long offset) {
- List<Mapping> shifted = new ArrayList<>(mappings.size());
- for (Mapping mapping : mappings) {
- shifted.add(
- mapping(
- mapping.oldStart,
- mapping.oldEnd,
- Math.addExact(mapping.newStart, offset)));
+ long combinedOffset;
+ try {
+ combinedOffset = Math.addExact(newStartOffset, offset);
+ } catch (ArithmeticException ignored) {
+ // The offsets themselves may overflow even though every shifted row ID remains valid.
+ // Materialize the effective starts in that uncommon case.
+ long[] shiftedNewStarts = new long[newStarts.length];
+ for (int i = 0; i < newStarts.length; i++) {
+ shiftedNewStarts[i] =
+ Math.addExact(Math.addExact(newStarts[i], newStartOffset), offset);
+ }
+ return new RowRangeMappingIndex(oldStarts, oldEnds, shiftedNewStarts, 0L);
}
- return create(shifted);
+
+ // Preserve the old eager implementation's contract: shift fails immediately if any
+ // individual new row ID overflows, even when that mapping is never queried later.
+ for (long newStart : newStarts) {
+ Math.addExact(Math.addExact(newStart, newStartOffset), offset);
+ }
+ return new RowRangeMappingIndex(oldStarts, oldEnds, newStarts, combinedOffset);
}
Optional<Range> map(Range oldRange) {
@@ -91,33 +130,32 @@ final class RowRangeMappingIndex {
long newTo = Long.MIN_VALUE;
boolean mapped = false;
- for (int i = lowerBound(oldEnds, cursor); i < mappings.size(); i++) {
- Mapping mapping = mappings.get(i);
- if (mapping.oldStart > cursor) {
+ for (int i = lowerBound(oldEnds, cursor); i < oldStarts.length; i++) {
+ if (oldStarts[i] > cursor) {
break;
}
- long segmentTo = Math.min(mapping.oldEnd, oldRange.to);
- long segmentNewFrom = mapping.newStart + cursor - mapping.oldStart;
- long segmentNewTo = mapping.newStart + segmentTo - mapping.oldStart;
+ long segmentTo = Math.min(oldEnds[i], oldRange.to);
+ long shiftedNewStart = Math.addExact(newStarts[i], newStartOffset);
+ long segmentNewFrom =
+ Math.addExact(shiftedNewStart, Math.subtractExact(cursor, oldStarts[i]));
+ long segmentNewTo =
+ Math.addExact(shiftedNewStart, Math.subtractExact(segmentTo, oldStarts[i]));
if (!mapped) {
newFrom = segmentNewFrom;
mapped = true;
- } else if (newTo + 1 != segmentNewFrom) {
+ } else if (Math.addExact(newTo, 1L) != segmentNewFrom) {
return Optional.empty();
}
newTo = segmentNewTo;
- cursor = segmentTo + 1;
- if (cursor > oldRange.to) {
- break;
+ if (segmentTo == oldRange.to) {
+ return Optional.of(new Range(newFrom, newTo));
}
+ cursor = Math.addExact(segmentTo, 1L);
}
- if (cursor <= oldRange.to) {
- return Optional.empty();
- }
- return Optional.of(new Range(newFrom, newTo));
+ return Optional.empty();
}
boolean overlaps(Range oldRange) {
@@ -125,7 +163,7 @@ final class RowRangeMappingIndex {
checkArgument(oldRange.from <= oldRange.to, "Invalid old row range %s.", oldRange);
int index = lowerBound(oldEnds, oldRange.from);
- return index < mappings.size() && mappings.get(index).oldStart <= oldRange.to;
+ return index < oldStarts.length && oldStarts[index] <= oldRange.to;
}
private static int lowerBound(long[] sorted, long target) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerCompactPlanTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerCompactPlanTest.java
new file mode 100644
index 0000000000..ec972cdba5
--- /dev/null
+++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerCompactPlanTest.java
@@ -0,0 +1,237 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.paimon.append.dataevolution;
+
+import org.apache.paimon.CoreOptions;
+import org.apache.paimon.Snapshot;
+import org.apache.paimon.data.BinaryString;
+import org.apache.paimon.data.GenericRow;
+import org.apache.paimon.manifest.ManifestFileMeta;
+import org.apache.paimon.manifest.ManifestList;
+import org.apache.paimon.partition.PartitionPredicate;
+import org.apache.paimon.schema.Schema;
+import org.apache.paimon.table.FileStoreTable;
+import org.apache.paimon.table.TableTestBase;
+import org.apache.paimon.table.sink.BatchTableCommit;
+import org.apache.paimon.table.sink.BatchTableWrite;
+import org.apache.paimon.table.sink.BatchWriteBuilder;
+import org.apache.paimon.types.DataTypes;
+
+import org.junit.jupiter.api.Test;
+
+import java.util.Collections;
+import java.util.List;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/** Tests for the compact planning state embedded in {@link DataEvolutionRowIdReassigner}. */
+public class DataEvolutionRowIdReassignerCompactPlanTest extends TableTestBase {
+
+ @Test
+ public void testProjectedPlannerBuildsExpectedPlan() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeOneRow(table, "a", 0);
+ writeOneRow(table, "b", 1);
+ writeOneRow(table, "a", 2);
+
+ DataEvolutionRowIdReassigner.PlanningState.Result result = plan(table, null);
+
+ assertThat(result.isEmpty()).isFalse();
+ assertThat(result.totalOffset).isEqualTo(2L);
+ assertThat(result.manifestOrdinals).isNotEmpty().isSorted();
+ assertThat(result.partitionMappings).hasSize(1);
+ DataEvolutionRowIdReassigner.PlanningState.PartitionMapping mapping =
+ result.partitionMappings.get(0);
+ assertThat(mapping.partition.getString(0).toString()).isEqualTo("a");
+ assertThat(mapping.oldStarts).containsExactly(0L, 2L);
+ assertThat(mapping.oldEnds).containsExactly(0L, 2L);
+ assertThat(mapping.newRelativeStarts).containsExactly(0L, 1L);
+ }
+
+ @Test
+ public void testProjectedPlannerAppliesPartitionPredicate() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeOneRow(table, "a", 0);
+ writeOneRow(table, "b", 1);
+ writeOneRow(table, "a", 2);
+ PartitionPredicate onlyB =
+ PartitionPredicate.fromMaps(
+ table.schema().logicalPartitionType(),
+ Collections.singletonList(Collections.singletonMap("pt", "b")),
+ table.coreOptions().partitionDefaultName());
+
+ DataEvolutionRowIdReassigner.PlanningState.Result result = plan(table, onlyB);
+
+ assertThat(result.isEmpty()).isTrue();
+ assertThat(result.manifestOrdinals).isEmpty();
+ assertThat(result.totalOffset).isZero();
+ }
+
+ @Test
+ public void testProjectedPlannerReadsOrcManifest() throws Exception {
+ catalog.createTable(identifier(), schema("orc"), true);
+ FileStoreTable table = getTableDefault();
+ writeOneRow(table, "a", 0);
+ writeOneRow(table, "b", 1);
+ writeOneRow(table, "a", 2);
+
+ DataEvolutionRowIdReassigner.PlanningState.Result result = plan(table, null);
+
+ assertThat(result.partitionMappings).hasSize(1);
+ assertThat(result.partitionMappings.get(0).oldStarts).containsExactly(0L, 2L);
+ assertThat(result.totalOffset).isEqualTo(2L);
+ }
+
+ @Test
+ public void testCurrentEntryHotStateUsesThreePrimitiveWords() {
+ DataEvolutionRowIdReassigner.PlanningState.CurrentEntries entries =
+ new DataEvolutionRowIdReassigner.PlanningState.CurrentEntries(10_000);
+
+ for (int i = 0; i < 10_000; i++) {
+ entries.add(i % 7, (i & 1) == 0, i * 10L, 3L);
+ }
+
+ assertThat(entries.size()).isEqualTo(10_000);
+ assertThat(entries.usedWordCount()).isEqualTo(30_000);
+ assertThat(entries.retainedWordCount()).isEqualTo(30_000);
+ }
+
+ @Test
+ public void testInitialCapacityEstimatesLiveEntriesAfterDeletes() {
+ assertThat(
+ DataEvolutionRowIdReassigner.PlanningState.initialCurrentEntryCapacity(
+ 13_572_157L, 0L))
+ .isEqualTo(13_572_157);
+ assertThat(
+ DataEvolutionRowIdReassigner.PlanningState.initialCurrentEntryCapacity(
+ 800_000_000L, 799_999_990L))
+ .isEqualTo(10);
+ assertThat(
+ DataEvolutionRowIdReassigner.PlanningState.initialCurrentEntryCapacity(
+ 800_000_000L, 0L))
+ .isEqualTo(1 << 24);
+ }
+
+ @Test
+ public void testContiguousEntriesDoNotMaterializePerEntryRanges() {
+ DataEvolutionRowIdReassigner.PlanningState.CurrentEntries entries =
+ new DataEvolutionRowIdReassigner.PlanningState.CurrentEntries(10_000);
+ for (int i = 0; i < 10_000; i++) {
+ entries.add(0, false, i, 1L);
+ }
+
+ DataEvolutionRowIdReassigner.PlanningState.PrimitiveRangeBuffer ranges =
+ entries.selectedRangesForTesting();
+
+ assertThat(ranges).isNull();
+ assertThat(entries.usedWordCount()).isEqualTo(30_000);
+ assertThat(entries.retainedWordCount()).isEqualTo(30_000);
+ assertThat(DataEvolutionRowIdReassigner.PlanningState.class.getDeclaredClasses())
+ .extracting(Class::getSimpleName)
+ .doesNotContain("RowRange");
+ }
+
+ @Test
+ public void testFragmentedEntriesUsePrimitiveLogicalRanges() {
+ DataEvolutionRowIdReassigner.PlanningState.CurrentEntries entries =
+ new DataEvolutionRowIdReassigner.PlanningState.CurrentEntries(10_000);
+ for (int i = 0; i < 10_000; i++) {
+ entries.add(0, false, i * 2L, 1L);
+ }
+
+ DataEvolutionRowIdReassigner.PlanningState.PrimitiveRangeBuffer ranges =
+ entries.selectedRangesForTesting();
+
+ assertThat(ranges).isNotNull();
+ assertThat(ranges.size()).isEqualTo(10_000);
+ assertThat(ranges.retainedWordCount()).isEqualTo(20_000);
+ assertThat(ranges.start(0)).isZero();
+ assertThat(ranges.end(0)).isZero();
+ assertThat(ranges.start(9_999)).isEqualTo(19_998L);
+ assertThat(ranges.end(9_999)).isEqualTo(19_998L);
+ assertThat(entries.usedWordCount()).isEqualTo(30_000);
+ assertThat(entries.retainedWordCount()).isEqualTo(30_000);
+ }
+
+ @Test
+ public void testDeletedIdentifiersUseCompactArenaAndIncludePartition() {
+ DataEvolutionRowIdReassigner.PlanningState.DeletedIdentifierSet identifiers =
+ new DataEvolutionRowIdReassigner.PlanningState.DeletedIdentifierSet();
+ byte[] identifier = {1, 2, 3, 4};
+
+ identifiers.add(3, identifier, identifier.length);
+ identifiers.add(3, identifier, identifier.length);
+
+ assertThat(identifiers.size()).isOne();
+ assertThat(identifiers.retainedIdentifierBytes()).isEqualTo(identifier.length);
+ assertThat(identifiers.contains(3, identifier, identifier.length)).isTrue();
+ assertThat(identifiers.contains(4, identifier, identifier.length)).isFalse();
+
+ identifiers.add(4, identifier, identifier.length);
+ assertThat(identifiers.size()).isEqualTo(2);
+ assertThat(identifiers.retainedIdentifierBytes()).isEqualTo(identifier.length * 2);
+ }
+
+ private DataEvolutionRowIdReassigner.PlanningState.Result plan(
+ FileStoreTable table, PartitionPredicate predicate) {
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ ManifestList manifestList = table.store().manifestListFactory().create();
+ List<ManifestFileMeta> manifestMetas = manifestList.readDataManifests(snapshot);
+ DataEvolutionRowIdReassigner.PlanningState state =
+ new DataEvolutionRowIdReassigner.PlanningState(table, predicate, manifestMetas);
+ List<List<ManifestFileMeta>> groups = Collections.singletonList(manifestMetas);
+ state.validateGroups(groups);
+ state.planGroup(manifestMetas);
+ return state.buildResult();
+ }
+
+ private void writeOneRow(FileStoreTable table, String partition, int id) throws Exception {
+ BatchWriteBuilder builder = table.newBatchWriteBuilder();
+ try (BatchTableWrite write = builder.newWrite();
+ BatchTableCommit commit = builder.newCommit()) {
+ write.write(
+ GenericRow.of(
+ BinaryString.fromString(partition),
+ id,
+ BinaryString.fromString("v" + id)));
+ commit.commit(write.prepareCommit());
+ }
+ }
+
+ @Override
+ protected Schema schemaDefault() {
+ return schema(null);
+ }
+
+ private Schema schema(String manifestFormat) {
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ schemaBuilder.column("pt", DataTypes.STRING());
+ schemaBuilder.column("id", DataTypes.INT());
+ schemaBuilder.column("payload", DataTypes.STRING());
+ schemaBuilder.partitionKeys("pt");
+ schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+ if (manifestFormat != null) {
+ schemaBuilder.option(CoreOptions.MANIFEST_FORMAT.key(), manifestFormat);
+ }
+ return schemaBuilder.build();
+ }
+}
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
index 77cb9802f3..37fcf1a1bd 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassignerTest.java
@@ -68,6 +68,8 @@ import org.apache.paimon.utils.SnapshotManager;
import org.junit.jupiter.api.Test;
+import java.lang.reflect.Field;
+import java.lang.reflect.Method;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -77,6 +79,7 @@ import java.util.LinkedHashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -123,6 +126,372 @@ public class DataEvolutionRowIdReassignerTest extends TableTestBase {
return schemaBuilder.build();
}
+ @Test
+ public void testCompactAndLegacyPlansMatchOnSameHistoricalSnapshot() throws Exception {
+ FileStoreTable table = createTableWithInterleavedPartitions();
+ FileEntry.Identifier historicalFile = compactOneFile(table, "pt=a/");
+ assertAddDeleteEntriesConsistent(table, historicalFile);
+
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ ManifestList manifestList = table.store().manifestListFactory().create();
+ List<ManifestFileMeta> manifestMetas = manifestList.readDataManifests(snapshot);
+ List<String> beforeManifestFiles = dataManifestFileNames(table);
+ int[] historicalManifestOrdinals =
+ manifestOrdinalsContaining(table, manifestMetas, historicalFile);
+ assertThat(historicalManifestOrdinals).hasSize(2);
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ assertThat(compactPlan.get().manifestOrdinals).contains(historicalManifestOrdinals);
+
+ assertThat(table.snapshotManager().latestSnapshot().id()).isEqualTo(snapshot.id());
+ assertThat(dataManifestFileNames(table)).containsExactlyElementsOf(beforeManifestFiles);
+ }
+
+ @Test
+ public void testCompactAndLegacyEmptyPlansMatch() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ writeRows(table, "a", 0, 1);
+ writeRows(table, "a", 2);
+ writeRows(table, "b", 3, 4);
+
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ ManifestList manifestList = table.store().manifestListFactory().create();
+ List<ManifestFileMeta> manifestMetas = manifestList.readDataManifests(snapshot);
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isEmpty();
+ assertThat(table.snapshotManager().latestSnapshot().id()).isEqualTo(snapshot.id());
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchWithPartitionPredicate() throws Exception {
+ FileStoreTable table = createTableWithInterleavedPartitions();
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ ManifestList manifestList = table.store().manifestListFactory().create();
+ List<ManifestFileMeta> manifestMetas = manifestList.readDataManifests(snapshot);
+ PartitionPredicate onlyA = partitionPredicate(table, "a");
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, onlyA);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, onlyA);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ assertThat(compactPlan.get().partitionMappings).hasSize(1);
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchWithoutPartitionStats() throws Exception {
+ FileStoreTable table = createTableWithInterleavedPartitions();
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ List<ManifestFileMeta> manifestMetas =
+ table.store().manifestListFactory().create().readDataManifests(snapshot);
+ List<ManifestFileMeta> metasWithoutPartitionStats = new ArrayList<>(manifestMetas.size());
+ for (ManifestFileMeta manifestMeta : manifestMetas) {
+ metasWithoutPartitionStats.add(withoutPartitionStats(manifestMeta));
+ }
+
+ Optional<AssignmentPlanView> compactPlan =
+ compactPlanView(table, metasWithoutPartitionStats, null);
+ Optional<AssignmentPlanView> legacyPlan =
+ legacyPlanView(table, metasWithoutPartitionStats, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchForOrcManifests() throws Exception {
+ Schema base = schemaDefault();
+ Map<String, String> options = new HashMap<>(base.options());
+ options.put(CoreOptions.MANIFEST_FORMAT.key(), "orc");
+ catalog.createTable(
+ identifier(),
+ new Schema(
+ base.fields(),
+ base.partitionKeys(),
+ base.primaryKeys(),
+ options,
+ base.comment()),
+ true);
+ FileStoreTable table = getTableDefault();
+ writeOneRow(table, "a", 0);
+ writeOneRow(table, "b", 1);
+ writeOneRow(table, "a", 2);
+
+ Snapshot snapshot = table.snapshotManager().latestSnapshot();
+ ManifestList manifestList = table.store().manifestListFactory().create();
+ List<ManifestFileMeta> manifestMetas = manifestList.readDataManifests(snapshot);
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchForSpecialFileRanges() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ InternalRowSerializer partitionSerializer =
+ new InternalRowSerializer(table.schema().logicalPartitionType());
+ BinaryRow partitionA =
+ partitionSerializer.toBinaryRow(GenericRow.of(BinaryString.fromString("a"))).copy();
+ BinaryRow partitionB =
+ partitionSerializer.toBinaryRow(GenericRow.of(BinaryString.fromString("b"))).copy();
+ BinaryRow partitionC =
+ partitionSerializer.toBinaryRow(GenericRow.of(BinaryString.fromString("c"))).copy();
+
+ List<ManifestEntry> entries =
+ Arrays.asList(
+ planningManifestEntry(partitionA, "ordinary-a-0.parquet", 10L, 10L),
+ planningManifestEntry(partitionA, "ordinary-a-0.blob", 12L, 4L),
+ planningManifestEntry(partitionA, "ordinary-a-0.vector.bin", 16L, 2L),
+ planningManifestEntry(partitionA, "ordinary-a-1.parquet", 30L, 10L),
+ planningManifestEntry(partitionB, "ordinary-b-0.parquet", 0L, 5L),
+ planningManifestEntry(partitionB, "ordinary-b-1.parquet", 5L, 5L),
+ planningManifestEntry(partitionC, "special-c-0.blob", 50L, 6L),
+ planningManifestEntry(partitionC, "special-c-0.vector.bin", 54L, 7L),
+ planningManifestEntry(partitionC, "special-c-1.blob", 80L, 3L),
+ planningManifestEntry(partitionC, "special-c-1.vector.bin", 82L, 4L));
+ List<ManifestFileMeta> manifestMetas =
+ table.store().manifestFileFactory().create().write(entries);
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ assertThat(compactPlan.get().partitionMappings).hasSize(2);
+ PartitionMappingView ordinaryMapping = compactPlan.get().partitionMappings.get(0);
+ assertThat(ordinaryMapping.oldStarts).containsExactly(10L, 30L);
+ assertThat(ordinaryMapping.oldEnds).containsExactly(19L, 39L);
+ assertThat(ordinaryMapping.newStarts).containsExactly(0L, 10L);
+ PartitionMappingView allSpecialMapping = compactPlan.get().partitionMappings.get(1);
+ assertThat(allSpecialMapping.oldStarts).containsExactly(50L, 80L);
+ assertThat(allSpecialMapping.oldEnds).containsExactly(60L, 85L);
+ assertThat(allSpecialMapping.newStarts).containsExactly(20L, 31L);
+ assertThat(compactPlan.get().totalOffset).isEqualTo(37L);
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchForRawNaNPartitionPayloads() throws Exception {
+ Schema.Builder schemaBuilder = Schema.newBuilder();
+ schemaBuilder.column("float_pt", DataTypes.FLOAT());
+ schemaBuilder.column("double_pt", DataTypes.DOUBLE());
+ schemaBuilder.column("id", DataTypes.INT());
+ schemaBuilder.partitionKeys("float_pt", "double_pt");
+ schemaBuilder.option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true");
+ schemaBuilder.option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true");
+ catalog.createTable(identifier(), schemaBuilder.build(), true);
+ FileStoreTable table = getTableDefault();
+
+ InternalRowSerializer partitionSerializer =
+ new InternalRowSerializer(table.schema().logicalPartitionType());
+ float floatNaN1 = Float.intBitsToFloat(0x7fc00001);
+ float floatNaN2 = Float.intBitsToFloat(0x7fc00002);
+ double doubleNaN1 = Double.longBitsToDouble(0x7ff8000000000001L);
+ double doubleNaN2 = Double.longBitsToDouble(0x7ff8000000000002L);
+ BinaryRow doubleScanFirst =
+ partitionSerializer.toBinaryRow(GenericRow.of(0.0f, doubleNaN1)).copy();
+ BinaryRow doubleLegacyFirst =
+ partitionSerializer.toBinaryRow(GenericRow.of(0.0f, doubleNaN2)).copy();
+ BinaryRow floatScanFirst =
+ partitionSerializer.toBinaryRow(GenericRow.of(floatNaN1, 0.0d)).copy();
+ BinaryRow floatLegacyFirst =
+ partitionSerializer.toBinaryRow(GenericRow.of(floatNaN2, 0.0d)).copy();
+
+ assertThat(doubleScanFirst.toBytes()).isNotEqualTo(doubleLegacyFirst.toBytes());
+ assertThat(floatScanFirst.toBytes()).isNotEqualTo(floatLegacyFirst.toBytes());
+
+ List<ManifestEntry> entries =
+ Arrays.asList(
+ planningManifestEntry(doubleScanFirst, "double-scan-100.parquet", 100L, 1L),
+ planningManifestEntry(
+ doubleLegacyFirst, "double-legacy-10.parquet", 10L, 1L),
+ planningManifestEntry(doubleScanFirst, "double-scan-102.parquet", 102L, 1L),
+ planningManifestEntry(
+ doubleLegacyFirst, "double-legacy-12.parquet", 12L, 1L),
+ planningManifestEntry(floatScanFirst, "float-scan-300.parquet", 300L, 1L),
+ planningManifestEntry(
+ floatLegacyFirst, "float-legacy-200.parquet", 200L, 1L),
+ planningManifestEntry(floatScanFirst, "float-scan-302.parquet", 302L, 1L),
+ planningManifestEntry(
+ floatLegacyFirst, "float-legacy-202.parquet", 202L, 1L));
+ List<ManifestFileMeta> manifestMetas =
+ table.store().manifestFileFactory().create().write(entries);
+ assertThat(manifestMetas).hasSize(1);
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+
+ assertThat(compactPlan).isPresent();
+ AssignmentPlanView plan = compactPlan.get();
+ assertThat(plan.manifestOrdinals).containsExactly(0);
+ assertThat(plan.partitionMappings).hasSize(4);
+
+ PartitionMappingView doubleLegacyMapping = plan.partitionMappings.get(0);
+ assertThat(doubleLegacyMapping.partitionBytes).containsExactly(doubleLegacyFirst.toBytes());
+ assertThat(doubleLegacyMapping.oldStarts).containsExactly(10L, 12L);
+ assertThat(doubleLegacyMapping.oldEnds).containsExactly(10L, 12L);
+ assertThat(doubleLegacyMapping.newStarts).containsExactly(0L, 1L);
+
+ PartitionMappingView doubleScanMapping = plan.partitionMappings.get(1);
+ assertThat(doubleScanMapping.partitionBytes).containsExactly(doubleScanFirst.toBytes());
+ assertThat(doubleScanMapping.oldStarts).containsExactly(100L, 102L);
+ assertThat(doubleScanMapping.oldEnds).containsExactly(100L, 102L);
+ assertThat(doubleScanMapping.newStarts).containsExactly(2L, 3L);
+
+ PartitionMappingView floatLegacyMapping = plan.partitionMappings.get(2);
+ assertThat(floatLegacyMapping.partitionBytes).containsExactly(floatLegacyFirst.toBytes());
+ assertThat(floatLegacyMapping.oldStarts).containsExactly(200L, 202L);
+ assertThat(floatLegacyMapping.oldEnds).containsExactly(200L, 202L);
+ assertThat(floatLegacyMapping.newStarts).containsExactly(4L, 5L);
+
+ PartitionMappingView floatScanMapping = plan.partitionMappings.get(3);
+ assertThat(floatScanMapping.partitionBytes).containsExactly(floatScanFirst.toBytes());
+ assertThat(floatScanMapping.oldStarts).containsExactly(300L, 302L);
+ assertThat(floatScanMapping.oldEnds).containsExactly(300L, 302L);
+ assertThat(floatScanMapping.newStarts).containsExactly(6L, 7L);
+ assertThat(plan.totalOffset).isEqualTo(8L);
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchForComplexDeleteIdentifier() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ BinaryRow partition =
+ new InternalRowSerializer(table.schema().logicalPartitionType())
+ .toBinaryRow(GenericRow.of(BinaryString.fromString("a")))
+ .copy();
+ DataFileMeta deletedFile =
+ DataFileMeta.forAppend(
+ "deleted.parquet",
+ 1L,
+ 1L,
+ SimpleStats.EMPTY_STATS,
+ 7L,
+ 9L,
+ 0L,
+ Arrays.asList("extra-0", "extra-1"),
+ new byte[] {1, 2, 3, 4},
+ FileSource.APPEND,
+ null,
+ "/external/deleted.parquet",
+ 1L,
+ null)
+ .upgrade(3);
+ ManifestFile manifestFile = table.store().manifestFileFactory().create();
+ List<ManifestFileMeta> manifestMetas = new ArrayList<>();
+ manifestMetas.addAll(
+ manifestFile.write(
+ Arrays.asList(
+ planningManifestEntry(partition, "current-0.parquet", 0L, 1L),
+ ManifestEntry.create(FileKind.ADD, partition, 7, 8, deletedFile),
+ planningManifestEntry(partition, "current-2.parquet", 2L, 1L))));
+ manifestMetas.addAll(
+ manifestFile.write(
+ Collections.singletonList(
+ ManifestEntry.create(
+ FileKind.DELETE, partition, 7, 8, deletedFile))));
+
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ assertThat(compactPlan.get().partitionMappings).hasSize(1);
+ PartitionMappingView mapping = compactPlan.get().partitionMappings.get(0);
+ assertThat(mapping.oldStarts).containsExactly(0L, 2L);
+ assertThat(mapping.oldEnds).containsExactly(0L, 2L);
+ assertThat(mapping.newStarts).containsExactly(0L, 1L);
+ }
+
+ @Test
+ public void testCompactAndLegacyPlansMatchForDeleteIdentifierBoundaries() throws Exception {
+ createTableDefault();
+ FileStoreTable table = getTableDefault();
+ InternalRowSerializer partitionSerializer =
+ new InternalRowSerializer(table.schema().logicalPartitionType());
+ BinaryRow partitionA =
+ partitionSerializer.toBinaryRow(GenericRow.of(BinaryString.fromString("a"))).copy();
+ BinaryRow partitionB =
+ partitionSerializer.toBinaryRow(GenericRow.of(BinaryString.fromString("b"))).copy();
+
+ List<String> extraFiles = Arrays.asList("extra-0", "extra-1");
+ byte[] embeddedIndex = {1, 2, 3, 4};
+ String externalPath = "/external/identity.parquet";
+ DataFileMeta identityFile =
+ planningDataFile(
+ "identity.parquet", 2L, 3, extraFiles, embeddedIndex, externalPath);
+ DataFileMeta totalBucketsFile =
+ planningDataFile(
+ "total-buckets.parquet", 6L, 3, extraFiles, embeddedIndex, externalPath);
+
+ List<ManifestEntry> entries = new ArrayList<>();
+ entries.add(planningManifestEntry(partitionA, "anchor-0.parquet", 0L, 1L));
+ entries.add(ManifestEntry.create(FileKind.ADD, partitionA, 7, 8, identityFile));
+ entries.add(planningManifestEntry(partitionA, "anchor-4.parquet", 4L, 1L));
+ entries.add(ManifestEntry.create(FileKind.ADD, partitionA, 9, 16, totalBucketsFile));
+
+ // Each DELETE differs from identityFile in exactly one Identifier field.
+ entries.add(ManifestEntry.create(FileKind.DELETE, partitionB, 7, 8, identityFile));
+ entries.add(ManifestEntry.create(FileKind.DELETE, partitionA, 6, 8, identityFile));
+ List<DataFileMeta> fileFieldNearMisses =
+ Arrays.asList(
+ identityFile.upgrade(4),
+ planningDataFile(
+ "different-name.parquet",
+ 2L,
+ 3,
+ extraFiles,
+ embeddedIndex,
+ externalPath),
+ planningDataFile(
+ "identity.parquet",
+ 2L,
+ 3,
+ Arrays.asList("extra-1", "extra-0"),
+ embeddedIndex,
+ externalPath),
+ planningDataFile(
+ "identity.parquet",
+ 2L,
+ 3,
+ extraFiles,
+ new byte[] {1, 2, 3, 5},
+ externalPath),
+ planningDataFile(
+ "identity.parquet",
+ 2L,
+ 3,
+ extraFiles,
+ embeddedIndex,
+ "/external/different.parquet"));
+ for (DataFileMeta nearMiss : fileFieldNearMisses) {
+ entries.add(ManifestEntry.create(FileKind.DELETE, partitionA, 7, 8, nearMiss));
+ }
+
+ // totalBuckets is deliberately different but is not part of FileEntry.Identifier.
+ entries.add(ManifestEntry.create(FileKind.DELETE, partitionA, 9, 32, totalBucketsFile));
+
+ List<ManifestFileMeta> manifestMetas =
+ table.store().manifestFileFactory().create().write(entries);
+ Optional<AssignmentPlanView> compactPlan = compactPlanView(table, manifestMetas, null);
+ Optional<AssignmentPlanView> legacyPlan = legacyPlanView(table, manifestMetas, null);
+
+ assertPlanViewsEqual(compactPlan, legacyPlan);
+ assertThat(compactPlan).isPresent();
+ assertThat(compactPlan.get().partitionMappings).hasSize(1);
+ PartitionMappingView mapping = compactPlan.get().partitionMappings.get(0);
+ assertThat(mapping.oldStarts).containsExactly(0L, 2L, 4L);
+ assertThat(mapping.oldEnds).containsExactly(0L, 2L, 4L);
+ assertThat(mapping.newStarts).containsExactly(0L, 1L, 2L);
+ assertThat(compactPlan.get().totalOffset).isEqualTo(3L);
+ }
+
@Test
public void testReassignRowIdsByPartition() throws Exception {
FileStoreTable table = createTableWithInterleavedPartitions();
@@ -171,6 +540,13 @@ public class DataEvolutionRowIdReassignerTest extends TableTestBase {
.containsEntry("pt=a/", Collections.singletonList(0L))
.containsEntry("pt=z/", Collections.singletonList(2L));
+ Snapshot planningSnapshot = table.snapshotManager().latestSnapshot();
+ List<ManifestFileMeta> planningManifests =
+ table.store().manifestListFactory().create().readDataManifests(planningSnapshot);
+ assertPlanViewsEqual(
+ compactPlanView(table, planningManifests, null),
+ legacyPlanView(table, planningManifests, null));
+
DataEvolutionRowIdReassigner.Result result =
new DataEvolutionRowIdReassigner(table)
.reassign("test-reassign-null-partition-row-id");
@@ -645,6 +1021,13 @@ public class DataEvolutionRowIdReassignerTest extends TableTestBase {
assertThat(table.snapshotManager().latestSnapshot().nextRowId()).isEqualTo(6L);
+ Snapshot planningSnapshot = table.snapshotManager().latestSnapshot();
+ List<ManifestFileMeta> planningManifests =
+ table.store().manifestListFactory().create().readDataManifests(planningSnapshot);
+ assertPlanViewsEqual(
+ compactPlanView(table, planningManifests, null),
+ legacyPlanView(table, planningManifests, null));
+
DataEvolutionRowIdReassigner.Result result =
new DataEvolutionRowIdReassigner(table).reassign("test-reassign-multi-row");
@@ -1226,6 +1609,12 @@ public class DataEvolutionRowIdReassignerTest extends TableTestBase {
ExpectedLargeManifestAssignment expectedAssignment =
expectedLargeManifestAssignment(table, before, entryCount);
+ List<ManifestFileMeta> planningManifests =
+ table.store().manifestListFactory().create().readDataManifests(before);
+ assertPlanViewsEqual(
+ compactPlanView(table, planningManifests, null),
+ legacyPlanView(table, planningManifests, null));
+
DataEvolutionRowIdReassigner.Result result =
new DataEvolutionRowIdReassigner(table)
.reassign("test-reassign-large-out-of-order-row-id");
@@ -1327,6 +1716,68 @@ public class DataEvolutionRowIdReassignerTest extends TableTestBase {
return ManifestEntry.create(FileKind.ADD, partition, 0, 1, file);
}
+ private ManifestEntry planningManifestEntry(
+ BinaryRow partition, String fileName, long firstRowId, long rowCount) {
+ DataFileMeta file =
+ DataFileMeta.forAppend(
+ fileName,
+ 1L,
+ rowCount,
+ SimpleStats.EMPTY_STATS,
+ 0L,
+ 0L,
+ 0L,
+ Collections.emptyList(),
+ null,
+ FileSource.APPEND,
+ null,
+ null,
+ firstRowId,
+ null);
+ return ManifestEntry.create(FileKind.ADD, partition, 0, 1, file);
+ }
+
+ private DataFileMeta planningDataFile(
+ String fileName,
+ long firstRowId,
+ int level,
+ List<String> extraFiles,
+ byte[] embeddedIndex,
+ String externalPath) {
+ return DataFileMeta.forAppend(
+ fileName,
+ 1L,
+ 1L,
+ SimpleStats.EMPTY_STATS,
+ 0L,
+ 0L,
+ 0L,
+ extraFiles,
+ embeddedIndex,
+ FileSource.APPEND,
+ null,
+ externalPath,
+ firstRowId,
+ null)
+ .upgrade(level);
+ }
+
+ private ManifestFileMeta withoutPartitionStats(ManifestFileMeta manifestMeta) {
+ return new ManifestFileMeta(
+ manifestMeta.fileName(),
+ manifestMeta.fileSize(),
+ manifestMeta.numAddedFiles(),
+ manifestMeta.numDeletedFiles(),
+ null,
+ manifestMeta.schemaId(),
+ manifestMeta.minBucket(),
+ manifestMeta.maxBucket(),
+ manifestMeta.minLevel(),
+ manifestMeta.maxLevel(),
+ manifestMeta.minRowId(),
+ manifestMeta.maxRowId());
+ }
+
private void assertReassignedOutOfOrderPartitionEntries(
FileStoreTable table,
int entryCount,
@@ -2183,6 +2634,180 @@ public class DataEvolutionRowIdReassignerTest extends TableTestBase {
return result;
}
+ private Optional<AssignmentPlanView> compactPlanView(
+ FileStoreTable table,
+ List<ManifestFileMeta> manifestMetas,
+ PartitionPredicate partitionPredicate)
+ throws Exception {
+ Method planAssignment =
+ DataEvolutionRowIdReassigner.class.getDeclaredMethod("planAssignment", List.class);
+ planAssignment.setAccessible(true);
+ Object result =
+ planAssignment.invoke(
+ new DataEvolutionRowIdReassigner(table, partitionPredicate), manifestMetas);
+ return assignmentPlanView(result, manifestMetas);
+ }
+
+ private Optional<AssignmentPlanView> legacyPlanView(
+ FileStoreTable table,
+ List<ManifestFileMeta> manifestMetas,
+ PartitionPredicate partitionPredicate)
+ throws Exception {
+ Method planAssignment =
+ LegacyDataEvolutionRowIdReassigner.class.getDeclaredMethod(
+ "planAssignment", List.class, ManifestFile.class);
+ planAssignment.setAccessible(true);
+ Object result =
+ planAssignment.invoke(
+ new LegacyDataEvolutionRowIdReassigner(table, partitionPredicate),
+ manifestMetas,
+ table.store().manifestFileFactory().create());
+ return assignmentPlanView(result, manifestMetas);
+ }
+
+ private Optional<AssignmentPlanView> assignmentPlanView(
+ Object optionalResult, List<ManifestFileMeta> manifestMetas) throws Exception {
+ assertThat(optionalResult).isInstanceOf(Optional.class);
+ Optional<?> optionalPlan = (Optional<?>) optionalResult;
+ if (!optionalPlan.isPresent()) {
+ return Optional.empty();
+ }
+
+ Object assignmentPlan = optionalPlan.get();
+ @SuppressWarnings("unchecked")
+ List<ManifestFileMeta> rewrittenManifests =
+ (List<ManifestFileMeta>) fieldValue(assignmentPlan, "manifestMetasToRewrite");
+ int[] manifestOrdinals = new int[rewrittenManifests.size()];
+ for (int i = 0; i < rewrittenManifests.size(); i++) {
+ manifestOrdinals[i] = manifestOrdinal(manifestMetas, rewrittenManifests.get(i));
+ }
+
+ Object relativeMappings = fieldValue(assignmentPlan, "relativeRowIdMappings");
+ @SuppressWarnings("unchecked")
+ Map<BinaryRow, RowRangeMappingIndex> mappings =
+ (Map<BinaryRow, RowRangeMappingIndex>) fieldValue(relativeMappings, "mappings");
+ List<PartitionMappingView> partitionMappings = new ArrayList<>(mappings.size());
+ for (Map.Entry<BinaryRow, RowRangeMappingIndex> mapping : mappings.entrySet()) {
+ partitionMappings.add(
+ new PartitionMappingView(
+ mapping.getKey().toBytes(),
+ longArrayField(mapping.getValue(), "oldStarts"),
+ longArrayField(mapping.getValue(), "oldEnds"),
+ longArrayField(mapping.getValue(), "newStarts"),
+ (Long) fieldValue(mapping.getValue(), "newStartOffset")));
+ }
+ long totalOffset = (Long) fieldValue(relativeMappings, "totalOffset");
+ return Optional.of(
+ new AssignmentPlanView(manifestOrdinals, partitionMappings, totalOffset));
+ }
+
+ private void assertPlanViewsEqual(
+ Optional<AssignmentPlanView> compact, Optional<AssignmentPlanView> legacy) {
+ assertThat(compact.isPresent()).isEqualTo(legacy.isPresent());
+ if (!compact.isPresent()) {
+ return;
+ }
+
+ AssignmentPlanView compactPlan = compact.get();
+ AssignmentPlanView legacyPlan = legacy.get();
+ assertThat(compactPlan.manifestOrdinals).containsExactly(legacyPlan.manifestOrdinals);
+ assertThat(compactPlan.totalOffset).isEqualTo(legacyPlan.totalOffset);
+ assertThat(compactPlan.partitionMappings).hasSameSizeAs(legacyPlan.partitionMappings);
+ for (int i = 0; i < compactPlan.partitionMappings.size(); i++) {
+ PartitionMappingView compactMapping = compactPlan.partitionMappings.get(i);
+ PartitionMappingView legacyMapping = legacyPlan.partitionMappings.get(i);
+ assertThat(compactMapping.partitionBytes).containsExactly(legacyMapping.partitionBytes);
+ assertThat(compactMapping.oldStarts).containsExactly(legacyMapping.oldStarts);
+ assertThat(compactMapping.oldEnds).containsExactly(legacyMapping.oldEnds);
+ assertThat(compactMapping.newStarts).containsExactly(legacyMapping.newStarts);
+ assertThat(compactMapping.newStartOffset).isEqualTo(legacyMapping.newStartOffset);
+ }
+ }
+
+ private int[] manifestOrdinalsContaining(
+ FileStoreTable table,
+ List<ManifestFileMeta> manifestMetas,
+ FileEntry.Identifier identifier) {
+ ManifestFile manifestFile = table.store().manifestFileFactory().create();
+ List<Integer> ordinals = new ArrayList<>();
+ for (int i = 0; i < manifestMetas.size(); i++) {
+ ManifestFileMeta manifestMeta = manifestMetas.get(i);
+ for (ManifestEntry entry :
+ manifestFile.read(manifestMeta.fileName(), manifestMeta.fileSize())) {
+ if (entry.identifier().equals(identifier)) {
+ ordinals.add(i);
+ break;
+ }
+ }
+ }
+ int[] result = new int[ordinals.size()];
+ for (int i = 0; i < ordinals.size(); i++) {
+ result[i] = ordinals.get(i);
+ }
+ return result;
+ }
+
+ private static int manifestOrdinal(
+ List<ManifestFileMeta> manifestMetas, ManifestFileMeta plannedManifest) {
+ for (int i = 0; i < manifestMetas.size(); i++) {
+ if (manifestMetas.get(i).fileName().equals(plannedManifest.fileName())) {
+ return i;
+ }
+ }
+ throw new AssertionError(
+ "Planned manifest is not present in the source snapshot: "
+ + plannedManifest.fileName());
+ }
+
+ private static long[] longArrayField(Object target, String fieldName) throws Exception {
+ long[] values = (long[]) fieldValue(target, fieldName);
+ return Arrays.copyOf(values, values.length);
+ }
+
+ private static Object fieldValue(Object target, String fieldName) throws Exception {
+ Field field = target.getClass().getDeclaredField(fieldName);
+ field.setAccessible(true);
+ return field.get(target);
+ }
+
+ private static class AssignmentPlanView {
+
+ private final int[] manifestOrdinals;
+ private final List<PartitionMappingView> partitionMappings;
+ private final long totalOffset;
+
+ private AssignmentPlanView(
+ int[] manifestOrdinals,
+ List<PartitionMappingView> partitionMappings,
+ long totalOffset) {
+ this.manifestOrdinals = manifestOrdinals;
+ this.partitionMappings = partitionMappings;
+ this.totalOffset = totalOffset;
+ }
+ }
+
+ private static class PartitionMappingView {
+
+ private final byte[] partitionBytes;
+ private final long[] oldStarts;
+ private final long[] oldEnds;
+ private final long[] newStarts;
+ private final long newStartOffset;
+
+ private PartitionMappingView(
+ byte[] partitionBytes,
+ long[] oldStarts,
+ long[] oldEnds,
+ long[] newStarts,
+ long newStartOffset) {
+ this.partitionBytes = partitionBytes;
+ this.oldStarts = oldStarts;
+ this.oldEnds = oldEnds;
+ this.newStarts = newStarts;
+ this.newStartOffset = newStartOffset;
+ }
+ }
+
private static class ExpectedLargeManifestAssignment {
private final int[] entriesByPartition;
diff --git a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java
similarity index 99%
copy from paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java
copy to paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java
index d81d98ed70..6415414990 100644
--- a/paimon-core/src/main/java/org/apache/paimon/append/dataevolution/DataEvolutionRowIdReassigner.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/LegacyDataEvolutionRowIdReassigner.java
@@ -66,10 +66,11 @@ import static org.apache.paimon.types.VectorType.isVectorStoreFile;
import static org.apache.paimon.utils.Preconditions.checkArgument;
import static org.apache.paimon.utils.Preconditions.checkState;
-/** Reassigns row IDs for data evolution tables by rewriting metadata only. */
-public class DataEvolutionRowIdReassigner {
+/** Frozen legacy planner used only as a differential-test oracle. */
+final class LegacyDataEvolutionRowIdReassigner {
- private static final Logger LOG = LoggerFactory.getLogger(DataEvolutionRowIdReassigner.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LegacyDataEvolutionRowIdReassigner.class);
private static final String COMMIT_USER_PREFIX = "reassign-row-id";
private static final int MAX_COMMIT_ATTEMPTS = 3;
@@ -77,17 +78,17 @@ public class DataEvolutionRowIdReassigner {
private final @Nullable PartitionPredicate partitionPredicate;
private final Runnable beforeCommit;
- public DataEvolutionRowIdReassigner(FileStoreTable table) {
+ public LegacyDataEvolutionRowIdReassigner(FileStoreTable table) {
this(table, null);
}
- public DataEvolutionRowIdReassigner(
+ public LegacyDataEvolutionRowIdReassigner(
FileStoreTable table, @Nullable PartitionPredicate partitionPredicate) {
this(table, partitionPredicate, () -> {});
}
@VisibleForTesting
- DataEvolutionRowIdReassigner(
+ LegacyDataEvolutionRowIdReassigner(
FileStoreTable table,
@Nullable PartitionPredicate partitionPredicate,
Runnable beforeCommit) {
diff --git a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndexTest.java b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndexTest.java
index 266574106c..9ed385bca0 100644
--- a/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndexTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/append/dataevolution/RowRangeMappingIndexTest.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test;
import java.util.Arrays;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
/** Tests for {@link RowRangeMappingIndex}. */
public class RowRangeMappingIndexTest {
@@ -85,6 +86,70 @@ public class RowRangeMappingIndexTest {
assertThat(absolute.map(new Range(20, 24))).hasValue(new Range(105, 109));
}
+ @Test
+ public void testPrimitiveArrayMappingsCanBeShifted() {
+ long[] oldStarts = {20, 30};
+ long[] oldEnds = {24, 34};
+ long[] newStarts = {0, 5};
+ RowRangeMappingIndex relative =
+ RowRangeMappingIndex.createFromOwnedArrays(oldStarts, oldEnds, newStarts);
+
+ RowRangeMappingIndex absolute = relative.shiftNewStarts(40L).shiftNewStarts(2L);
+ assertThat(relative.map(new Range(20, 24))).hasValue(new Range(0, 4));
+ assertThat(absolute.map(new Range(20, 24))).hasValue(new Range(42, 46));
+ assertThat(absolute.map(new Range(30, 34))).hasValue(new Range(47, 51));
+ }
+
+ @Test
+ public void testPrimitiveArrayMappingsAreValidated() {
+ assertThatThrownBy(
+ () ->
+ RowRangeMappingIndex.createFromOwnedArrays(
+ new long[] {10}, new long[] {14, 24}, new long[] {100}))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("same length");
+ assertThatThrownBy(
+ () ->
+ RowRangeMappingIndex.createFromOwnedArrays(
+ new long[] {10, 14},
+ new long[] {14, 24},
+ new long[] {100, 105}))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("cannot overlap");
+ assertThatThrownBy(
+ () ->
+ RowRangeMappingIndex.createFromOwnedArrays(
+ new long[] {20, 10},
+ new long[] {24, 14},
+ new long[] {100, 105}))
+ .isInstanceOf(IllegalArgumentException.class)
+ .hasMessageContaining("out of order");
+ }
+
+ @Test
+ public void testMapFailsOnNewRowIdOverflow() {
+ RowRangeMappingIndex index =
+ RowRangeMappingIndex.createFromOwnedArrays(
+ new long[] {10}, new long[] {11}, new long[] {Long.MAX_VALUE});
+
+ assertThatThrownBy(() -> index.shiftNewStarts(1L)).isInstanceOf(ArithmeticException.class);
+ assertThatThrownBy(() -> index.map(new Range(10, 11)))
+ .isInstanceOf(ArithmeticException.class);
+ assertThatThrownBy(() -> index.shiftNewStarts(Long.MAX_VALUE).shiftNewStarts(1L))
+ .isInstanceOf(ArithmeticException.class);
+ }
+
+ @Test
+ public void testShiftMaterializesWhenOnlyCombinedOffsetOverflows() {
+ RowRangeMappingIndex index =
+ RowRangeMappingIndex.createFromOwnedArrays(
+ new long[] {10}, new long[] {10}, new long[] {Long.MIN_VALUE});
+
+ RowRangeMappingIndex shifted = index.shiftNewStarts(Long.MAX_VALUE).shiftNewStarts(1L);
+
+ assertThat(shifted.map(new Range(10, 10))).hasValue(new Range(0, 0));
+ }
+
@Test
public void testOverlaps() {
RowRangeMappingIndex index =