You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by ht...@apache.org on 2026/10/14 04:57:02 UTC
(asterixdb) branch master updated: [ASTERIXDB-3946][EXT]: Apply all fixed delete keys
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 27ed59b4a8 [ASTERIXDB-3946][EXT]: Apply all fixed delete keys
27ed59b4a8 is described below
commit 27ed59b4a8e9d0720350c151db30289778091a72
Author: Hussain Towaileb <Hu...@Couchbase.com>
AuthorDate: Wed Oct 7 18:58:49 2026 +0300
[ASTERIXDB-3946][EXT]: Apply all fixed delete keys
- user model changes: no
- storage format changes: no
- interface changes: no
Details: Iceberg reads an equality delete file reusing one record for
every row and keeps each row with Record.copy(), which copies binary
values but not the byte[] of a fixed value, while the Parquet fixed
reader refills that same array. Every kept key therefore held the last
key read, and rows deleted by the others were returned. Equality delete
files keyed on a fixed column are now read without reusing containers,
through the delete cache like any other delete file, and on the
variant-pruned read.
Added tests covering fixed keys through the key-type matrix, the loader
with and without the cache, and the pruned read's delete filter,
equal-length binary keys, and a tripwire for the defect in Iceberg's own
loader.
Ext-ref: MB-74470
Generated-by: Claude Opus 5.5 <no...@anthropic.com>
Change-Id: I0f98b516c01165721c7726137105ac353ab4425a
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/22010
Tested-by: Hussain Towaileb <hu...@gmail.com>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ian Maxon <im...@apache.org>
Reviewed-by: Hussain Towaileb <hu...@gmail.com>
---
.../reader/iceberg/FixedKeySafeDeleteLoader.java | 123 ++++++++++++
.../record/reader/iceberg/IcebergDeleteCache.java | 5 +-
.../reader/iceberg/IcebergFileRecordReader.java | 3 +-
.../iceberg/PositionlessGenericDeleteFilter.java | 3 +-
.../iceberg/FixedKeySafeDeleteLoaderTest.java | 218 +++++++++++++++++++++
.../iceberg/IcebergEqualityDeleteKeyTypesTest.java | 10 +-
6 files changed, 352 insertions(+), 10 deletions(-)
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/FixedKeySafeDeleteLoader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/FixedKeySafeDeleteLoader.java
new file mode 100644
index 0000000000..e81fca8a44
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/FixedKeySafeDeleteLoader.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.iceberg;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.function.Function;
+
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.StructLike;
+import org.apache.iceberg.data.BaseDeleteLoader;
+import org.apache.iceberg.data.InternalRecordWrapper;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.InputFile;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.TypeUtil;
+import org.apache.iceberg.util.StructLikeSet;
+
+/**
+ * Iceberg's delete loader, except that equality deletes keyed on a {@code fixed} column are read without reusing
+ * containers.
+ * <p>
+ * Iceberg reads an equality delete file reusing one record for every row and keeps each row through
+ * {@code Record.copy()}, which deep-copies a binary value but not the {@code byte[]} of a fixed one, while the Parquet
+ * fixed reader refills that same array for every row. Every kept key therefore holds the last key read, and every
+ * other deleted row is returned. Reading those files with fresh containers gives each key an array of its own. Delete
+ * files of any other format keep Iceberg's path, as no other format is read here.
+ * <p>
+ * The rows read this way go through {@link #canCache} and {@link #getOrLoad} exactly as Iceberg's own loads do, so a
+ * caching subclass shares them across readers. That is safe for the same reason it is for any other key: each row and
+ * its array are its own and are only read once loaded.
+ */
+class FixedKeySafeDeleteLoader extends BaseDeleteLoader {
+
+ private final Function<DeleteFile, InputFile> loadInputFile;
+
+ FixedKeySafeDeleteLoader(Function<DeleteFile, InputFile> loadInputFile) {
+ super(loadInputFile);
+ this.loadInputFile = loadInputFile;
+ }
+
+ @Override
+ public StructLikeSet loadEqualityDeletes(Iterable<DeleteFile> deleteFiles, Schema projection) {
+ if (!hasFixedField(projection) || !allParquet(deleteFiles)) {
+ return super.loadEqualityDeletes(deleteFiles, projection);
+ }
+ StructLikeSet deleteSet = StructLikeSet.create(projection.asStruct());
+ for (DeleteFile deleteFile : deleteFiles) {
+ long size = estimatedSize(deleteFile, projection);
+ Iterable<StructLike> rows = canCache(size)
+ ? getOrLoad(deleteFile.location(), () -> readEqualityDeletes(deleteFile, projection), size)
+ : readEqualityDeletes(deleteFile, projection);
+ for (StructLike row : rows) {
+ deleteSet.add(row);
+ }
+ }
+ return deleteSet;
+ }
+
+ private List<StructLike> readEqualityDeletes(DeleteFile deleteFile, Schema projection) {
+ // converts the stored representation to the one the data rows are compared in, as Iceberg's own load does
+ InternalRecordWrapper asStructLike = new InternalRecordWrapper(projection.asStruct());
+ List<StructLike> rows = new ArrayList<>();
+ try (CloseableIterable<Record> records = Parquet.read(loadInputFile.apply(deleteFile)).project(projection)
+ .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(projection, fileSchema)).build()) {
+ for (Record record : records) {
+ rows.add(asStructLike.copyFor(record));
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to close equality delete file " + deleteFile.location(), e);
+ }
+ return Collections.unmodifiableList(rows);
+ }
+
+ /** The estimate Iceberg charges for an equality delete file it loads: its rows times the key's estimated size. */
+ private static long estimatedSize(DeleteFile deleteFile, Schema projection) {
+ long rowSize = projection.columns().stream().mapToLong(TypeUtil::estimateSize).sum();
+ try {
+ return Math.multiplyExact(deleteFile.recordCount(), rowSize);
+ } catch (ArithmeticException e) {
+ return Long.MAX_VALUE;
+ }
+ }
+
+ /** @return whether {@code schema} has a {@code fixed} field at any depth */
+ static boolean hasFixedField(Schema schema) {
+ return TypeUtil.indexById(schema.asStruct()).values().stream()
+ .anyMatch(field -> field.type().typeId() == Type.TypeID.FIXED);
+ }
+
+ private static boolean allParquet(Iterable<DeleteFile> deleteFiles) {
+ for (DeleteFile deleteFile : deleteFiles) {
+ if (deleteFile.format() != FileFormat.PARQUET) {
+ return false;
+ }
+ }
+ return true;
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergDeleteCache.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergDeleteCache.java
index 0323e5196b..6843d614f3 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergDeleteCache.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergDeleteCache.java
@@ -28,7 +28,6 @@ import java.util.function.Supplier;
import org.apache.iceberg.DeleteFile;
import org.apache.iceberg.Schema;
-import org.apache.iceberg.data.BaseDeleteLoader;
import org.apache.iceberg.data.DeleteLoader;
import org.apache.iceberg.io.InputFile;
import org.apache.iceberg.util.StructLikeSet;
@@ -68,7 +67,7 @@ final class IcebergDeleteCache {
* @return a delete loader reading through this cache, opening delete files with {@code loadInputFile}
*/
DeleteLoader newLoader(Function<DeleteFile, InputFile> loadInputFile) {
- return new BaseDeleteLoader(loadInputFile) {
+ return new FixedKeySafeDeleteLoader(loadInputFile) {
// Iceberg asks this before it looks the file up, so the total budget is not applied here: a file already
// in the cache is served from it however full the cache is, and a miss that does not fit is loaded
// uncached in getOrLoad.
@@ -88,7 +87,7 @@ final class IcebergDeleteCache {
@Override
public StructLikeSet loadEqualityDeletes(Iterable<DeleteFile> deleteFiles, Schema projection) {
if (hasStructField(projection)) {
- return new BaseDeleteLoader(loadInputFile).loadEqualityDeletes(deleteFiles, projection);
+ return new FixedKeySafeDeleteLoader(loadInputFile).loadEqualityDeletes(deleteFiles, projection);
}
return super.loadEqualityDeletes(deleteFiles, projection);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergFileRecordReader.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergFileRecordReader.java
index 4301fc5fc9..c78cab97d0 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergFileRecordReader.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergFileRecordReader.java
@@ -486,7 +486,8 @@ public class IcebergFileRecordReader implements IRecordReader<Record> {
@Override
protected DeleteLoader newDeleteLoader() {
- return deleteCache != null ? deleteCache.newLoader(this::loadInputFile) : super.newDeleteLoader();
+ return deleteCache != null ? deleteCache.newLoader(this::loadInputFile)
+ : new FixedKeySafeDeleteLoader(this::loadInputFile);
}
};
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/PositionlessGenericDeleteFilter.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/PositionlessGenericDeleteFilter.java
index c7306177da..88bf7eed4b 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/PositionlessGenericDeleteFilter.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/iceberg/PositionlessGenericDeleteFilter.java
@@ -93,6 +93,7 @@ final class PositionlessGenericDeleteFilter extends org.apache.iceberg.data.Dele
@Override
protected DeleteLoader newDeleteLoader() {
- return deleteCache != null ? deleteCache.newLoader(this::loadInputFile) : super.newDeleteLoader();
+ return deleteCache != null ? deleteCache.newLoader(this::loadInputFile)
+ : new FixedKeySafeDeleteLoader(this::loadInputFile);
}
}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/FixedKeySafeDeleteLoaderTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/FixedKeySafeDeleteLoaderTest.java
new file mode 100644
index 0000000000..325345b207
--- /dev/null
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/FixedKeySafeDeleteLoaderTest.java
@@ -0,0 +1,218 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.external.input.record.reader.iceberg;
+
+import java.io.File;
+import java.nio.file.Files;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeSet;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.function.Predicate;
+
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DataFiles;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileFormat;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.data.BaseDeleteLoader;
+import org.apache.iceberg.data.GenericRecord;
+import org.apache.iceberg.data.Record;
+import org.apache.iceberg.data.parquet.GenericParquetReaders;
+import org.apache.iceberg.data.parquet.GenericParquetWriter;
+import org.apache.iceberg.deletes.EqualityDeleteWriter;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.io.FileAppender;
+import org.apache.iceberg.io.FileIO;
+import org.apache.iceberg.parquet.Parquet;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.StructLikeSet;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+/**
+ * Equality deletes keyed on a {@code fixed} column, loaded by {@link FixedKeySafeDeleteLoader}: every deleted key is
+ * kept, where Iceberg's own loader keeps only the last one read. The key-type matrix in
+ * {@code IcebergEqualityDeleteKeyTypesTest} covers the standard read with and without the cache; this covers the
+ * loader on its own and the delete filter of the variant-pruned read.
+ */
+public class FixedKeySafeDeleteLoaderTest {
+
+ private static final Schema SCHEMA = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get()),
+ Types.NestedField.required(2, "k", Types.FixedType.ofLength(16)));
+ private static final Schema KEY = SCHEMA.select("k");
+ private static final int ROWS = 200;
+ private static final int KEYS = 50;
+ private static final int DELETED_KEYS = 20;
+
+ private File warehouse;
+ private Table table;
+ private FileScanTask task;
+ private FileIO io;
+
+ @Before
+ public void createTable() throws Exception {
+ warehouse = Files.createTempDirectory("fixed-key-deletes").toFile();
+ table = new HadoopTables(new org.apache.hadoop.conf.Configuration()).create(SCHEMA,
+ PartitionSpec.unpartitioned(), Map.of(TableProperties.FORMAT_VERSION, "2"), warehouse.toString());
+ table.newAppend().appendFile(writeDataFile()).commit();
+ table.newRowDelta().addDeletes(writeEqualityDelete()).commit();
+ try (CloseableIterable<FileScanTask> planned = table.newScan().planFiles()) {
+ task = planned.iterator().next();
+ }
+ Assert.assertEquals("the equality delete must apply to the data file", 1, task.deletes().size());
+ io = table.io();
+ }
+
+ @After
+ public void deleteWarehouse() throws Exception {
+ org.apache.commons.io.FileUtils.deleteDirectory(warehouse);
+ }
+
+ @Test
+ public void everyFixedKeyIsLoaded() {
+ StructLikeSet deleted = new FixedKeySafeDeleteLoader(this::open).loadEqualityDeletes(task.deletes(), KEY);
+ Assert.assertEquals(DELETED_KEYS, deleted.size());
+ }
+
+ /** Readers of a scan sharing the delete cache load a fixed-keyed delete file once, as they do any other. */
+ @Test
+ public void aFixedKeyedFileIsLoadedOnceThroughTheCache() {
+ AtomicInteger opens = new AtomicInteger();
+ IcebergDeleteCache cache = new IcebergDeleteCache();
+ for (int reader = 0; reader < 2; reader++) {
+ StructLikeSet deleted = cache.newLoader(file -> {
+ opens.incrementAndGet();
+ return open(file);
+ }).loadEqualityDeletes(task.deletes(), KEY);
+ Assert.assertEquals(DELETED_KEYS, deleted.size());
+ }
+ Assert.assertEquals("the second reader must be served from the cache", 1, opens.get());
+ Assert.assertEquals(1, cache.size());
+ }
+
+ @Test
+ public void withoutTheCacheEveryLoadReadsTheFile() {
+ AtomicInteger opens = new AtomicInteger();
+ for (int reader = 0; reader < 2; reader++) {
+ StructLikeSet deleted = new FixedKeySafeDeleteLoader(file -> {
+ opens.incrementAndGet();
+ return open(file);
+ }).loadEqualityDeletes(task.deletes(), KEY);
+ Assert.assertEquals(DELETED_KEYS, deleted.size());
+ }
+ Assert.assertEquals(2, opens.get());
+ }
+
+ /**
+ * Tripwire: Iceberg's own loader keeps one key out of {@link #DELETED_KEYS}. When this fails, Iceberg has fixed
+ * the defect and {@link FixedKeySafeDeleteLoader} can be retired.
+ */
+ @Test
+ public void icebergsOwnLoaderStillKeepsOnlyTheLastKey() {
+ StructLikeSet deleted = new BaseDeleteLoader(this::open).loadEqualityDeletes(task.deletes(), KEY);
+ Assert.assertEquals(1, deleted.size());
+ }
+
+ @Test
+ public void theVariantPrunedReadsDeleteFilterAppliesEveryFixedKey() throws Exception {
+ PositionlessGenericDeleteFilter filter = new PositionlessGenericDeleteFilter(io, task, SCHEMA, SCHEMA);
+ Predicate<Record> keep = filter.eqDeletedRowFilter();
+ Schema required = filter.requiredSchema();
+ Set<Integer> kept = new TreeSet<>();
+ try (CloseableIterable<Record> rows = Parquet.read(io.newInputFile(task.file().location())).project(required)
+ .createReaderFunc(fileSchema -> GenericParquetReaders.buildReader(required, fileSchema)).build()) {
+ for (Record row : rows) {
+ if (keep.test(row)) {
+ kept.add((Integer) row.getField("id"));
+ }
+ }
+ }
+ Assert.assertEquals(expectedIds(), kept);
+ }
+
+ @Test
+ public void aFixedFieldIsFoundAtAnyDepth() {
+ Assert.assertTrue(FixedKeySafeDeleteLoader.hasFixedField(KEY));
+ Assert.assertTrue(FixedKeySafeDeleteLoader.hasFixedField(new Schema(Types.NestedField.optional(3, "st",
+ Types.StructType.of(Types.NestedField.optional(4, "k", Types.FixedType.ofLength(4)))))));
+ Assert.assertFalse(FixedKeySafeDeleteLoader.hasFixedField(SCHEMA.select("id")));
+ }
+
+ private org.apache.iceberg.io.InputFile open(DeleteFile deleteFile) {
+ return io.newInputFile(deleteFile.location(), deleteFile.fileSizeInBytes());
+ }
+
+ private static Set<Integer> expectedIds() {
+ Set<Integer> ids = new TreeSet<>();
+ for (int id = 0; id < ROWS; id++) {
+ if (id % KEYS >= DELETED_KEYS) {
+ ids.add(id);
+ }
+ }
+ return ids;
+ }
+
+ private static byte[] key(int k) {
+ byte[] bytes = new byte[16];
+ bytes[0] = (byte) k;
+ bytes[15] = (byte) (k * 7);
+ return bytes;
+ }
+
+ private DataFile writeDataFile() throws Exception {
+ File file = new File(warehouse, "data/data.parquet");
+ Assert.assertTrue(file.getParentFile().mkdirs() || file.getParentFile().exists());
+ FileAppender<Record> appender = Parquet.write(org.apache.iceberg.Files.localOutput(file)).schema(SCHEMA)
+ .createWriterFunc(GenericParquetWriter::create).build();
+ try (FileAppender<Record> writer = appender) {
+ for (int id = 0; id < ROWS; id++) {
+ Record row = GenericRecord.create(SCHEMA);
+ row.setField("id", id);
+ row.setField("k", key(id % KEYS));
+ writer.add(row);
+ }
+ }
+ return DataFiles.builder(PartitionSpec.unpartitioned()).withInputFile(org.apache.iceberg.Files.localInput(file))
+ .withMetrics(appender.metrics()).withFormat(FileFormat.PARQUET).build();
+ }
+
+ private DeleteFile writeEqualityDelete() throws Exception {
+ File file = new File(warehouse, "data/eq-delete.parquet");
+ EqualityDeleteWriter<Record> writer = Parquet.writeDeletes(org.apache.iceberg.Files.localOutput(file))
+ .forTable(table).rowSchema(KEY).createWriterFunc(GenericParquetWriter::create)
+ .equalityFieldIds(List.of(2)).overwrite().buildEqualityWriter();
+ try (EqualityDeleteWriter<Record> open = writer) {
+ for (int k = 0; k < DELETED_KEYS; k++) {
+ Record row = GenericRecord.create(KEY);
+ row.setField("k", key(k));
+ open.write(row);
+ }
+ }
+ return writer.toDeleteFile();
+ }
+}
diff --git a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergEqualityDeleteKeyTypesTest.java b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergEqualityDeleteKeyTypesTest.java
index e1e782e54d..ea5abac944 100644
--- a/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergEqualityDeleteKeyTypesTest.java
+++ b/asterixdb/asterix-external-data/src/test/java/org/apache/asterix/external/input/record/reader/iceberg/IcebergEqualityDeleteKeyTypesTest.java
@@ -67,7 +67,6 @@ import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -132,6 +131,11 @@ public class IcebergEqualityDeleteKeyTypesTest {
k -> k < 20, nested, 2);
add(cases, "binary", Types.BinaryType.get(),
k -> ByteBuffer.wrap(("bin-" + k).getBytes(StandardCharsets.UTF_8)), k -> k < 20, nested, 2);
+ // Iceberg's binary reader refills the previous row's buffer whenever the next value fits in it, so only
+ // keys of one length exercise that reuse; the case above changes length at k = 10
+ add(cases, "binary_same_length", Types.BinaryType.get(),
+ k -> ByteBuffer.wrap(String.format("bin-%02d", k).getBytes(StandardCharsets.UTF_8)), k -> k < 20,
+ nested, 2);
}
return cases;
}
@@ -188,10 +192,6 @@ public class IcebergEqualityDeleteKeyTypesTest {
@Test
public void withoutCache() throws Exception {
- // Iceberg reads equality delete files reusing its containers, and its fixed-length reader fills the same
- // array for every row it reads, so every loaded fixed key holds the last one read: only that key is deleted,
- // with or without the cache. The cache must still change nothing, which the other tests check.
- Assume.assumeFalse("fixed-length equality keys are mis-loaded by Iceberg itself", "fixed".equals(typeName));
Assert.assertEquals(expectedIds(), readAll(null));
}