You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by hu...@apache.org on 2026/06/14 04:47:14 UTC

(iceberg) branch main updated: Data: Skip equality-delete filter when there are no equality deletes (#16742)

This is an automated email from the ASF dual-hosted git repository.

huaxingao pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new ffb7d24f6f Data: Skip equality-delete filter when there are no equality deletes (#16742)
ffb7d24f6f is described below

commit ffb7d24f6f4e5fa06aeccc06c7590af1d206455c
Author: Vova Kolmakov <wo...@gmail.com>
AuthorDate: Sun Jun 14 11:47:09 2026 +0700

    Data: Skip equality-delete filter when there are no equality deletes (#16742)
    
    Co-authored-by: Vova Kolmakov <wo...@apache.org>
    Co-authored-by: Claude Opus 4.8 (1M context) <no...@anthropic.com>
---
 .../apache/iceberg/data/DeleteFilterBenchmark.java | 235 +++++++++++++++++++++
 .../java/org/apache/iceberg/data/DeleteFilter.java |   4 +
 2 files changed, 239 insertions(+)

diff --git a/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java b/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java
new file mode 100644
index 0000000000..ab918c9902
--- /dev/null
+++ b/data/src/jmh/java/org/apache/iceberg/data/DeleteFilterBenchmark.java
@@ -0,0 +1,235 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.data;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+import static org.apache.iceberg.types.Types.NestedField.required;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.TimeUnit;
+import org.apache.iceberg.DataFile;
+import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileScanTask;
+import org.apache.iceberg.Files;
+import org.apache.iceberg.MetadataColumns;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.Table;
+import org.apache.iceberg.TableProperties;
+import org.apache.iceberg.hadoop.HadoopTables;
+import org.apache.iceberg.io.CloseableIterable;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.Pair;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.TearDown;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+/**
+ * Measures the generic read path through {@link DeleteFilter} when applying position and equality
+ * deletes, plus the no-delete baseline.
+ *
+ * <p>Two views are measured per scenario:
+ *
+ * <ul>
+ *   <li>{@code scan} - end-to-end read via {@link IcebergGenerics} (Parquet decode + delete filter)
+ *   <li>{@code filterOnly} - the delete filter applied to pre-materialized in-memory records,
+ *       isolating the per-row iterator/predicate cost from file decode
+ * </ul>
+ *
+ * The {@code deleteType} parameter covers NONE (plain scan), POSITION (position deletes, the common
+ * merge-on-read / deletion-vector shape) and EQUALITY (a control where the equality-delete path
+ * does real work).
+ */
+@Fork(1)
+@State(Scope.Benchmark)
+@Warmup(iterations = 5)
+@Measurement(iterations = 15)
+@BenchmarkMode(Mode.SingleShotTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+public class DeleteFilterBenchmark {
+
+  private static final Schema SCHEMA =
+      new Schema(
+          required(1, "id", Types.LongType.get()),
+          required(2, "intCol", Types.IntegerType.get()),
+          optional(3, "doubleCol", Types.DoubleType.get()),
+          optional(4, "stringCol", Types.StringType.get()));
+
+  private static final int NUM_ROWS = 2_500_000;
+  private static final long SEED = -1L;
+
+  // delete every 20th row -> 5% delete ratio
+  private static final int DELETE_EVERY = 20;
+
+  @Param({"NONE", "POSITION", "EQUALITY"})
+  private String deleteType;
+
+  private final HadoopTables tables = new HadoopTables();
+
+  private File tableDir;
+  private Table table;
+  private List<Record> filterInput;
+  private GenericDeleteFilter filter;
+
+  @Setup
+  public void setupBenchmark() throws IOException {
+    this.tableDir = java.nio.file.Files.createTempDirectory("delete-filter-bench").toFile();
+    this.table =
+        tables.create(
+            SCHEMA,
+            PartitionSpec.unpartitioned(),
+            ImmutableMap.of(TableProperties.FORMAT_VERSION, "2"),
+            tableDir.toURI().toString());
+
+    List<Record> rows = RandomGenericData.generate(SCHEMA, NUM_ROWS, SEED);
+    // make ids sequential so position == id, giving deterministic deletes
+    for (int i = 0; i < rows.size(); i++) {
+      rows.get(i).setField("id", (long) i);
+    }
+
+    DataFile dataFile =
+        FileHelpers.writeDataFile(
+            table, Files.localOutput(new File(tableDir, "data.parquet")), rows);
+    table.newAppend().appendFile(dataFile).commit();
+
+    switch (deleteType) {
+      case "POSITION":
+        List<Pair<CharSequence, Long>> posDeletes = Lists.newArrayList();
+        for (long pos = 0; pos < NUM_ROWS; pos += DELETE_EVERY) {
+          posDeletes.add(Pair.of(dataFile.location(), pos));
+        }
+        DeleteFile posDeleteFile =
+            FileHelpers.writeDeleteFile(
+                    table, Files.localOutput(new File(tableDir, "pos-deletes.parquet")), posDeletes)
+                .first();
+        table.newRowDelta().addDeletes(posDeleteFile).commit();
+        break;
+
+      case "EQUALITY":
+        Schema deleteRowSchema = table.schema().select("id");
+        Record keyTemplate = GenericRecord.create(deleteRowSchema);
+        List<Record> eqDeletes = Lists.newArrayList();
+        for (long id = 0; id < NUM_ROWS; id += DELETE_EVERY) {
+          eqDeletes.add(keyTemplate.copy(ImmutableMap.of("id", id)));
+        }
+        DeleteFile eqDeleteFile =
+            FileHelpers.writeDeleteFile(
+                table,
+                Files.localOutput(new File(tableDir, "eq-deletes.parquet")),
+                eqDeletes,
+                deleteRowSchema);
+        table.newRowDelta().addDeletes(eqDeleteFile).commit();
+        break;
+
+      default:
+        // NONE: no delete files
+        break;
+    }
+
+    // build the reusable filter and its pre-materialized input for the filterOnly benchmark
+    List<FileScanTask> tasks;
+    try (CloseableIterable<FileScanTask> planned = table.newScan().planFiles()) {
+      tasks = Lists.newArrayList(planned);
+    }
+    FileScanTask task = Iterables.getOnlyElement(tasks);
+    this.filter = new GenericDeleteFilter(table.io(), task, table.schema(), table.schema());
+    this.filterInput = materializeFilterInput(rows, filter.requiredSchema());
+  }
+
+  // The filter reads the _pos metadata column for position deletes, so when the required schema
+  // includes it we augment each record with its row position. For NONE/EQUALITY the required schema
+  // equals the table schema and the generated rows are used directly.
+  private List<Record> materializeFilterInput(List<Record> rows, Schema requiredSchema) {
+    boolean hasPositionColumn =
+        requiredSchema.findField(MetadataColumns.ROW_POSITION.fieldId()) != null;
+    if (!hasPositionColumn) {
+      return rows;
+    }
+
+    List<Record> withPositions = Lists.newArrayListWithCapacity(rows.size());
+    for (int i = 0; i < rows.size(); i++) {
+      Record base = rows.get(i);
+      GenericRecord withPos = GenericRecord.create(requiredSchema);
+      for (Types.NestedField field : table.schema().columns()) {
+        withPos.setField(field.name(), base.getField(field.name()));
+      }
+      withPos.setField(MetadataColumns.ROW_POSITION.name(), (long) i);
+      withPositions.add(withPos);
+    }
+    return withPositions;
+  }
+
+  @TearDown
+  public void tearDownBenchmark() {
+    if (table != null) {
+      tables.dropTable(tableDir.toURI().toString());
+    }
+    deleteRecursively(tableDir);
+  }
+
+  @Benchmark
+  @Threads(1)
+  public void scan(Blackhole blackhole) throws IOException {
+    try (CloseableIterable<Record> reader = IcebergGenerics.read(table).reuseContainers().build()) {
+      for (Record record : reader) {
+        blackhole.consume(record);
+      }
+    }
+  }
+
+  @Benchmark
+  @Threads(1)
+  public void filterOnly(Blackhole blackhole) throws IOException {
+    CloseableIterable<Record> source = CloseableIterable.withNoopClose(filterInput);
+    try (CloseableIterable<Record> filtered = filter.filter(source)) {
+      for (Record record : filtered) {
+        blackhole.consume(record);
+      }
+    }
+  }
+
+  private static void deleteRecursively(File file) {
+    if (file == null) {
+      return;
+    }
+    File[] children = file.listFiles();
+    if (children != null) {
+      for (File child : children) {
+        deleteRecursively(child);
+      }
+    }
+    file.delete();
+  }
+}
diff --git a/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java b/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java
index 69236cee8f..d7d66452e8 100644
--- a/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java
+++ b/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java
@@ -232,6 +232,10 @@ public abstract class DeleteFilter<T> {
   }
 
   private CloseableIterable<T> applyEqDeletes(CloseableIterable<T> records) {
+    if (eqDeletes.isEmpty()) {
+      return records;
+    }
+
     Predicate<T> isEqDeleted = applyEqDeletes().stream().reduce(Predicate::or).orElse(t -> false);
 
     return createDeleteIterable(records, isEqDeleted);