You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ao...@apache.org on 2023/11/21 20:20:48 UTC
(iceberg) branch main updated: Data: Always use delete index for position deletes (#9117)
This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new c61c3ca017 Data: Always use delete index for position deletes (#9117)
c61c3ca017 is described below
commit c61c3ca017aaec145f57948f0da9cb307bf45cde
Author: Anton Okolnychyi <ao...@apple.com>
AuthorDate: Tue Nov 21 12:20:41 2023 -0800
Data: Always use delete index for position deletes (#9117)
---
.../java/org/apache/iceberg/data/DeleteFilter.java | 18 +++---------------
1 file changed, 3 insertions(+), 15 deletions(-)
diff --git a/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java b/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java
index a7979fd2ed..55acd32008 100644
--- a/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java
+++ b/data/src/main/java/org/apache/iceberg/data/DeleteFilter.java
@@ -58,11 +58,9 @@ import org.slf4j.LoggerFactory;
public abstract class DeleteFilter<T> {
private static final Logger LOG = LoggerFactory.getLogger(DeleteFilter.class);
- private static final long DEFAULT_SET_FILTER_THRESHOLD = 100_000L;
private static final Schema POS_DELETE_SCHEMA =
new Schema(MetadataColumns.DELETE_FILE_PATH, MetadataColumns.DELETE_FILE_POS);
- private final long setFilterThreshold;
private final String filePath;
private final List<DeleteFile> posDeletes;
private final List<DeleteFile> eqDeletes;
@@ -82,7 +80,6 @@ public abstract class DeleteFilter<T> {
Schema tableSchema,
Schema requestedSchema,
DeleteCounter counter) {
- this.setFilterThreshold = DEFAULT_SET_FILTER_THRESHOLD;
this.filePath = filePath;
this.counter = counter;
@@ -245,18 +242,9 @@ public abstract class DeleteFilter<T> {
List<CloseableIterable<Record>> deletes = Lists.transform(posDeletes, this::openPosDeletes);
- // if there are fewer deletes than a reasonable number to keep in memory, use a set
- if (posDeletes.stream().mapToLong(DeleteFile::recordCount).sum() < setFilterThreshold) {
- PositionDeleteIndex positionIndex = Deletes.toPositionIndex(filePath, deletes);
- Predicate<T> isDeleted = record -> positionIndex.isDeleted(pos(record));
- return createDeleteIterable(records, isDeleted);
- }
-
- return hasIsDeletedColumn
- ? Deletes.streamingMarker(
- records, this::pos, Deletes.deletePositions(filePath, deletes), this::markRowDeleted)
- : Deletes.streamingFilter(
- records, this::pos, Deletes.deletePositions(filePath, deletes), counter);
+ PositionDeleteIndex positionIndex = Deletes.toPositionIndex(filePath, deletes);
+ Predicate<T> isDeleted = record -> positionIndex.isDeleted(pos(record));
+ return createDeleteIterable(records, isDeleted);
}
private CloseableIterable<T> createDeleteIterable(