You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by as...@apache.org on 2023/12/19 22:42:42 UTC
(impala) branch master updated (f93bd9862 -> 012996a06)
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from f93bd9862 IMPALA-11805: Use llvm ObjectCache for codegen caching
new 8c3cb56f6 IMPALA-11921: Give more time to test_large_sql in slow builds
new 012996a06 IMPALA-12597: Basic Equality delete read support for Iceberg tables
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/exec/partitioned-hash-join-builder.h | 4 +-
be/src/exec/partitioned-hash-join-node.h | 4 +-
common/thrift/CatalogObjects.thrift | 10 +-
common/thrift/PlanNodes.thrift | 8 +-
.../apache/impala/analysis/BinaryPredicate.java | 5 +-
.../org/apache/impala/catalog/FeIcebergTable.java | 18 +-
.../impala/catalog/IcebergContentFileStore.java | 31 ++-
.../apache/impala/catalog/IcebergDeleteTable.java | 169 +++++++++++++
.../impala/catalog/IcebergEqualityDeleteTable.java | 68 ++++++
.../impala/catalog/IcebergPositionDeleteTable.java | 146 +-----------
.../catalog/iceberg/GroupedContentFiles.java | 24 +-
.../apache/impala/planner/IcebergScanPlanner.java | 264 ++++++++++++++++-----
.../apache/impala/planner/SingleNodePlanner.java | 6 +-
testdata/data/README | 78 ++++++
...471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet | Bin 0 -> 898 bytes
...471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet | Bin 0 -> 697 bytes
...09aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet | Bin 0 -> 885 bytes
...09aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet | Bin 0 -> 657 bytes
...e61b766e-652a169e00000001_800513971_data.0.parq | Bin 0 -> 1606 bytes
.../0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m0.avro | Bin 0 -> 3732 bytes
.../0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m1.avro | Bin 0 -> 3710 bytes
.../3d36bf90-2625-4625-b09b-d4359b979df9-m0.avro | Bin 0 -> 3724 bytes
.../3d36bf90-2625-4625-b09b-d4359b979df9-m1.avro | Bin 0 -> 3710 bytes
.../bb4b8c07-84e1-421a-bb6c-594f297d118e-m0.avro | Bin 0 -> 4037 bytes
...895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro | Bin 0 -> 2326 bytes
...888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro | Bin 0 -> 2648 bytes
...581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro | Bin 0 -> 2808 bytes
.../metadata/v1.metadata.json | 51 ++++
.../metadata/v2.metadata.json | 87 +++++++
.../metadata/v3.metadata.json | 119 ++++++++++
.../metadata/v4.metadata.json | 146 ++++++++++++
.../metadata/version-hint.text | 1 +
...3256830-d9bd9e2f00000000_1372039299_data.0.parq | Bin 0 -> 616 bytes
...f44b347b-e035009600000001_138281890_data.0.parq | Bin 0 -> 1630 bytes
...836ebfcc-95c9ce7a00000000_909175610_data.0.parq | Bin 0 -> 1630 bytes
.../2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro | Bin 0 -> 6716 bytes
.../57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro | Bin 0 -> 6696 bytes
.../8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro | Bin 0 -> 6691 bytes
...726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro | Bin 0 -> 4303 bytes
...078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro | Bin 0 -> 4430 bytes
...958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro | Bin 0 -> 4383 bytes
.../metadata/v1.metadata.json | 53 +++++
.../metadata/v2.metadata.json | 83 +++++++
.../metadata/v3.metadata.json | 110 +++++++++
.../metadata/v4.metadata.json | 137 +++++++++++
.../metadata/version-hint.text | 1 +
...a56dbd8-ddb31c6400000000_1397530881_data.0.parq | Bin 0 -> 611 bytes
...792bdd2-aabb8e2000000000_1239775374_data.0.parq | Bin 0 -> 351 bytes
...0a2b4e6-d242a84000000000_1980761347_data.0.parq | Bin 0 -> 598 bytes
.../25ce5480-23b6-4c70-a724-63931f8d84c6-m0.avro | Bin 0 -> 3698 bytes
.../87d3b6df-f00d-40a4-aafa-5d7f20e3299b-m0.avro | Bin 0 -> 3699 bytes
.../c8b17188-94bf-4496-9069-3eda900cd71d-m0.avro | Bin 0 -> 3663 bytes
...636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro | Bin 0 -> 2484 bytes
...976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro | Bin 0 -> 2165 bytes
...676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro | Bin 0 -> 2325 bytes
.../metadata/v1.metadata.json | 53 +++++
.../metadata/v2.metadata.json | 83 +++++++
.../metadata/v3.metadata.json | 110 +++++++++
.../metadata/v4.metadata.json | 136 +++++++++++
.../metadata/version-hint.text | 1 +
...0c103-bfaf-49b4-935a-16a951e55b1c-00001.parquet | Bin 0 -> 898 bytes
...0c103-bfaf-49b4-935a-16a951e55b1c-00002.parquet | Bin 0 -> 702 bytes
...0d7bb-47e7-4221-8e06-f63922d3fa23-00001.parquet | Bin 0 -> 857 bytes
...0d7bb-47e7-4221-8e06-f63922d3fa23-00002.parquet | Bin 0 -> 662 bytes
.../807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro | Bin 0 -> 3848 bytes
.../807d7d2a-0557-4bbe-9f07-9467de72598a-m1.avro | Bin 0 -> 3834 bytes
.../c9c459ff-9747-4dab-b65f-cace0f31e669-m0.avro | Bin 0 -> 3849 bytes
.../c9c459ff-9747-4dab-b65f-cace0f31e669-m1.avro | Bin 0 -> 3835 bytes
...878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro | Bin 0 -> 2384 bytes
...837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro | Bin 0 -> 2762 bytes
.../metadata/v1.metadata.json | 56 +++++
.../metadata/v2.metadata.json | 92 +++++++
.../metadata/v3.metadata.json | 124 ++++++++++
.../metadata/v4.metadata.json | 142 +++++++++++
.../metadata/version-hint.text | 1 +
...800d3-c638-4591-b40c-158dcd5ebe25-00001.parquet | Bin 0 -> 904 bytes
...800d3-c638-4591-b40c-158dcd5ebe25-00002.parquet | Bin 0 -> 704 bytes
...2da66-fe58-44b5-81bd-575da62c7a91-00001.parquet | Bin 0 -> 898 bytes
...2da66-fe58-44b5-81bd-575da62c7a91-00002.parquet | Bin 0 -> 702 bytes
...289e0-d713-41a1-bdaf-f9feab643720-00001.parquet | Bin 0 -> 898 bytes
...289e0-d713-41a1-bdaf-f9feab643720-00002.parquet | Bin 0 -> 702 bytes
...67ae8-d9c3-4071-b671-8bbbe79d36d1-00001.parquet | Bin 0 -> 857 bytes
...67ae8-d9c3-4071-b671-8bbbe79d36d1-00002.parquet | Bin 0 -> 662 bytes
.../09ca0acc-c19e-4073-80f7-b476a6e568c7-m0.avro | Bin 0 -> 3841 bytes
.../09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro | Bin 0 -> 3827 bytes
.../57f8cd32-619c-46fc-a683-1dee7473c990-m0.avro | Bin 0 -> 3842 bytes
.../57f8cd32-619c-46fc-a683-1dee7473c990-m1.avro | Bin 0 -> 3828 bytes
.../db5041df-259d-48b9-ade1-1bf382a93d5a-m0.avro | Bin 0 -> 3841 bytes
.../db5041df-259d-48b9-ade1-1bf382a93d5a-m1.avro | Bin 0 -> 3827 bytes
.../e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m0.avro | Bin 0 -> 3841 bytes
.../e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m1.avro | Bin 0 -> 3827 bytes
...560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro | Bin 0 -> 2368 bytes
...114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro | Bin 0 -> 3094 bytes
...313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro | Bin 0 -> 3458 bytes
...060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro | Bin 0 -> 2732 bytes
.../metadata/v1.metadata.json | 56 +++++
.../metadata/v2.metadata.json | 92 +++++++
.../metadata/v3.metadata.json | 124 ++++++++++
.../metadata/v4.metadata.json | 156 ++++++++++++
.../metadata/v5.metadata.json | 188 +++++++++++++++
.../metadata/version-hint.text | 1 +
.../functional/functional_schema_template.sql | 60 +++++
.../datasets/functional/schema_constraints.csv | 4 +
.../queries/PlannerTest/iceberg-v2-tables.test | 228 ++++++++++++++++++
.../queries/QueryTest/iceberg-negative.test | 32 +--
.../iceberg-v2-read-equality-deletes.test | 73 ++++++
tests/query_test/test_iceberg.py | 45 ++++
tests/shell/test_shell_commandline.py | 4 +-
108 files changed, 3232 insertions(+), 252 deletions(-)
create mode 100644 fe/src/main/java/org/apache/impala/catalog/IcebergDeleteTable.java
create mode 100644 fe/src/main/java/org/apache/impala/catalog/IcebergEqualityDeleteTable.java
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/delete-074a9e19e61b766e-652a169e00000001_800513971_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/bb4b8c07-84e1-421a-bb6c-594f297d118e-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-8985205515767142888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-911559291487642581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v4.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/version-hint.text
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/af4e128ee3256830-d9bd9e2f00000000_1372039299_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-41417e7df44b347b-e035009600000001_138281890_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-61438487836ebfcc-95c9ce7a00000000_909175610_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-6117850509763739078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v4.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/version-hint.text
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/a94b351bfa56dbd8-ddb31c6400000000_1397530881_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/delete-494fbbd4b792bdd2-aabb8e2000000000_1239775374_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/e6484a6fb0a2b4e6-d242a84000000000_1980761347_data.0.parq
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/25ce5480-23b6-4c70-a724-63931f8d84c6-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/87d3b6df-f00d-40a4-aafa-5d7f20e3299b-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/c8b17188-94bf-4496-9069-3eda900cd71d-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4301391241829251636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-9091814429631192676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v4.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/version-hint.text
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-586812101365618837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v4.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/version-hint.text
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00001.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00002.parquet
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m0.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m1.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3979814664665937114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-4821964189199835313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v4.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v5.metadata.json
create mode 100644 testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/version-hint.text
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-equality-deletes.test
(impala) 01/02: IMPALA-11921: Give more time to test_large_sql in slow builds
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 8c3cb56f64cff688dadcf93fe5d5bd93f21b6206
Author: Csaba Ringhofer <cs...@cloudera.com>
AuthorDate: Tue Dec 19 15:40:09 2023 +0100
IMPALA-11921: Give more time to test_large_sql in slow builds
The test was flaky on ASAN builds, hopefully giving more
time will solve this.
Change-Id: Id1177bee2a0ad2a0d02f8923369645af02063be8
Reviewed-on: http://gerrit.cloudera.org:8080/20817
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/shell/test_shell_commandline.py | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
diff --git a/tests/shell/test_shell_commandline.py b/tests/shell/test_shell_commandline.py
index 380441fd7..32ed405da 100644
--- a/tests/shell/test_shell_commandline.py
+++ b/tests/shell/test_shell_commandline.py
@@ -1057,7 +1057,9 @@ class TestImpalaShell(ImpalaTestSuite):
assert "Could not resolve table reference: 'non_existence_large_table'" \
in result.stderr
end_time = time()
- time_limit_s = 20
+ # Use higher timeout in ASAN/UBSAN to avoid flakiness (IMPALA-11921).
+ build_runs_slowly = ImpalaTestClusterProperties.get_instance().runs_slowly()
+ time_limit_s = 60 if build_runs_slowly else 20
actual_time_s = end_time - start_time
assert actual_time_s <= time_limit_s, (
"It took {0} seconds to execute the query. Time limit is {1} seconds.".format(
(impala) 02/02: IMPALA-12597: Basic Equality delete read support for Iceberg tables
Posted by as...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asherman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 012996a06b9978200bf061a9d26a3b248bcb605a
Author: Gabor Kaszab <ga...@cloudera.com>
AuthorDate: Mon Oct 30 14:44:12 2023 +0100
IMPALA-12597: Basic Equality delete read support for Iceberg tables
In general, applying equality deletes is similar to how position
deletes are applied to data files: using a LEFT ANTI JOIN where the
SCAN for the data rows is on the left side while the SCAN for the
delete rows is on the right side of the JOIN. The difference is the
virtual columns and the conjuncts being used.
For equality deletes the data sequence number of a delete file has to
be greater than the data sequence number of the data file being
investigated. This information is added as a virtual column to the
scans and a conjunct is created in the JOIN node to check the relation.
The equality delete fields from the delete files are checked agains the
respective columns of the data SCANS.
This patch makes it possible for Impala to read Iceberg tables with
basic equality delete files. The Iceberg spec gives great flexibility
for engines for writing equality deletes, however in practice Flink,
one of the engines that write EQ-deletes supports only a subset of the
use cases. This patch focuses on reading the EQ-deletes written by
Flink.
The restrictions are the following:
- All equality delete files in a table should have the same equality
field ID list.
- For partitioned Iceberg tables it is expected that the partition
values are also written into the equality delete files.
- Tables with equality deletes shouldn't have partition or schema
evolution.
- Floating point equality columns aren't supported.
- If a malformed equality delete file doesn't have some of the equality
field IDs then Parquet reader will fill those missing fields with
NULLs. As a side effect this will drop the rows from the result where
the corresponding data columns have a null value.
See IMPALA-11388 epic Jira for more details.
Testing:
- Checked if the existing functional_parquet.iceberg_v2_delete_equality
table can be read successfully.
- Added new test tables so that E2E tests can validate correctness.
Change-Id: I2053e6f321c69f1c82059a84a5d99aeaa9814cad
Reviewed-on: http://gerrit.cloudera.org:8080/20753
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/partitioned-hash-join-builder.h | 4 +-
be/src/exec/partitioned-hash-join-node.h | 4 +-
common/thrift/CatalogObjects.thrift | 10 +-
common/thrift/PlanNodes.thrift | 8 +-
.../apache/impala/analysis/BinaryPredicate.java | 5 +-
.../org/apache/impala/catalog/FeIcebergTable.java | 18 +-
.../impala/catalog/IcebergContentFileStore.java | 31 ++-
.../apache/impala/catalog/IcebergDeleteTable.java | 169 +++++++++++++
.../impala/catalog/IcebergEqualityDeleteTable.java | 68 ++++++
.../impala/catalog/IcebergPositionDeleteTable.java | 146 +-----------
.../catalog/iceberg/GroupedContentFiles.java | 24 +-
.../apache/impala/planner/IcebergScanPlanner.java | 264 ++++++++++++++++-----
.../apache/impala/planner/SingleNodePlanner.java | 6 +-
testdata/data/README | 78 ++++++
...471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet | Bin 0 -> 898 bytes
...471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet | Bin 0 -> 697 bytes
...09aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet | Bin 0 -> 885 bytes
...09aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet | Bin 0 -> 657 bytes
...e61b766e-652a169e00000001_800513971_data.0.parq | Bin 0 -> 1606 bytes
.../0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m0.avro | Bin 0 -> 3732 bytes
.../0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m1.avro | Bin 0 -> 3710 bytes
.../3d36bf90-2625-4625-b09b-d4359b979df9-m0.avro | Bin 0 -> 3724 bytes
.../3d36bf90-2625-4625-b09b-d4359b979df9-m1.avro | Bin 0 -> 3710 bytes
.../bb4b8c07-84e1-421a-bb6c-594f297d118e-m0.avro | Bin 0 -> 4037 bytes
...895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro | Bin 0 -> 2326 bytes
...888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro | Bin 0 -> 2648 bytes
...581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro | Bin 0 -> 2808 bytes
.../metadata/v1.metadata.json | 51 ++++
.../metadata/v2.metadata.json | 87 +++++++
.../metadata/v3.metadata.json | 119 ++++++++++
.../metadata/v4.metadata.json | 146 ++++++++++++
.../metadata/version-hint.text | 1 +
...3256830-d9bd9e2f00000000_1372039299_data.0.parq | Bin 0 -> 616 bytes
...f44b347b-e035009600000001_138281890_data.0.parq | Bin 0 -> 1630 bytes
...836ebfcc-95c9ce7a00000000_909175610_data.0.parq | Bin 0 -> 1630 bytes
.../2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro | Bin 0 -> 6716 bytes
.../57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro | Bin 0 -> 6696 bytes
.../8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro | Bin 0 -> 6691 bytes
...726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro | Bin 0 -> 4303 bytes
...078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro | Bin 0 -> 4430 bytes
...958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro | Bin 0 -> 4383 bytes
.../metadata/v1.metadata.json | 53 +++++
.../metadata/v2.metadata.json | 83 +++++++
.../metadata/v3.metadata.json | 110 +++++++++
.../metadata/v4.metadata.json | 137 +++++++++++
.../metadata/version-hint.text | 1 +
...a56dbd8-ddb31c6400000000_1397530881_data.0.parq | Bin 0 -> 611 bytes
...792bdd2-aabb8e2000000000_1239775374_data.0.parq | Bin 0 -> 351 bytes
...0a2b4e6-d242a84000000000_1980761347_data.0.parq | Bin 0 -> 598 bytes
.../25ce5480-23b6-4c70-a724-63931f8d84c6-m0.avro | Bin 0 -> 3698 bytes
.../87d3b6df-f00d-40a4-aafa-5d7f20e3299b-m0.avro | Bin 0 -> 3699 bytes
.../c8b17188-94bf-4496-9069-3eda900cd71d-m0.avro | Bin 0 -> 3663 bytes
...636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro | Bin 0 -> 2484 bytes
...976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro | Bin 0 -> 2165 bytes
...676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro | Bin 0 -> 2325 bytes
.../metadata/v1.metadata.json | 53 +++++
.../metadata/v2.metadata.json | 83 +++++++
.../metadata/v3.metadata.json | 110 +++++++++
.../metadata/v4.metadata.json | 136 +++++++++++
.../metadata/version-hint.text | 1 +
...0c103-bfaf-49b4-935a-16a951e55b1c-00001.parquet | Bin 0 -> 898 bytes
...0c103-bfaf-49b4-935a-16a951e55b1c-00002.parquet | Bin 0 -> 702 bytes
...0d7bb-47e7-4221-8e06-f63922d3fa23-00001.parquet | Bin 0 -> 857 bytes
...0d7bb-47e7-4221-8e06-f63922d3fa23-00002.parquet | Bin 0 -> 662 bytes
.../807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro | Bin 0 -> 3848 bytes
.../807d7d2a-0557-4bbe-9f07-9467de72598a-m1.avro | Bin 0 -> 3834 bytes
.../c9c459ff-9747-4dab-b65f-cace0f31e669-m0.avro | Bin 0 -> 3849 bytes
.../c9c459ff-9747-4dab-b65f-cace0f31e669-m1.avro | Bin 0 -> 3835 bytes
...878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro | Bin 0 -> 2384 bytes
...837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro | Bin 0 -> 2762 bytes
.../metadata/v1.metadata.json | 56 +++++
.../metadata/v2.metadata.json | 92 +++++++
.../metadata/v3.metadata.json | 124 ++++++++++
.../metadata/v4.metadata.json | 142 +++++++++++
.../metadata/version-hint.text | 1 +
...800d3-c638-4591-b40c-158dcd5ebe25-00001.parquet | Bin 0 -> 904 bytes
...800d3-c638-4591-b40c-158dcd5ebe25-00002.parquet | Bin 0 -> 704 bytes
...2da66-fe58-44b5-81bd-575da62c7a91-00001.parquet | Bin 0 -> 898 bytes
...2da66-fe58-44b5-81bd-575da62c7a91-00002.parquet | Bin 0 -> 702 bytes
...289e0-d713-41a1-bdaf-f9feab643720-00001.parquet | Bin 0 -> 898 bytes
...289e0-d713-41a1-bdaf-f9feab643720-00002.parquet | Bin 0 -> 702 bytes
...67ae8-d9c3-4071-b671-8bbbe79d36d1-00001.parquet | Bin 0 -> 857 bytes
...67ae8-d9c3-4071-b671-8bbbe79d36d1-00002.parquet | Bin 0 -> 662 bytes
.../09ca0acc-c19e-4073-80f7-b476a6e568c7-m0.avro | Bin 0 -> 3841 bytes
.../09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro | Bin 0 -> 3827 bytes
.../57f8cd32-619c-46fc-a683-1dee7473c990-m0.avro | Bin 0 -> 3842 bytes
.../57f8cd32-619c-46fc-a683-1dee7473c990-m1.avro | Bin 0 -> 3828 bytes
.../db5041df-259d-48b9-ade1-1bf382a93d5a-m0.avro | Bin 0 -> 3841 bytes
.../db5041df-259d-48b9-ade1-1bf382a93d5a-m1.avro | Bin 0 -> 3827 bytes
.../e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m0.avro | Bin 0 -> 3841 bytes
.../e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m1.avro | Bin 0 -> 3827 bytes
...560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro | Bin 0 -> 2368 bytes
...114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro | Bin 0 -> 3094 bytes
...313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro | Bin 0 -> 3458 bytes
...060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro | Bin 0 -> 2732 bytes
.../metadata/v1.metadata.json | 56 +++++
.../metadata/v2.metadata.json | 92 +++++++
.../metadata/v3.metadata.json | 124 ++++++++++
.../metadata/v4.metadata.json | 156 ++++++++++++
.../metadata/v5.metadata.json | 188 +++++++++++++++
.../metadata/version-hint.text | 1 +
.../functional/functional_schema_template.sql | 60 +++++
.../datasets/functional/schema_constraints.csv | 4 +
.../queries/PlannerTest/iceberg-v2-tables.test | 228 ++++++++++++++++++
.../queries/QueryTest/iceberg-negative.test | 32 +--
.../iceberg-v2-read-equality-deletes.test | 73 ++++++
tests/query_test/test_iceberg.py | 45 ++++
107 files changed, 3229 insertions(+), 251 deletions(-)
diff --git a/be/src/exec/partitioned-hash-join-builder.h b/be/src/exec/partitioned-hash-join-builder.h
index dd578e318..1e271ca19 100644
--- a/be/src/exec/partitioned-hash-join-builder.h
+++ b/be/src/exec/partitioned-hash-join-builder.h
@@ -82,7 +82,9 @@ class PhjBuilderConfig : public JoinBuilderConfig {
std::vector<ScalarExpr*> build_exprs_;
/// is_not_distinct_from_[i] is true if and only if the ith equi-join predicate is IS
- /// NOT DISTINCT FROM, rather than equality.
+ /// NOT DISTINCT FROM, rather than equality. This is the case when IS NOT DISTINCT FROM
+ /// is explicitly used as a join predicate or when joining Iceberg equality delete
+ /// files to data files.
/// Set in InitExprsAndFilters() and constant thereafter.
std::vector<bool> is_not_distinct_from_;
diff --git a/be/src/exec/partitioned-hash-join-node.h b/be/src/exec/partitioned-hash-join-node.h
index 0a89384c2..c7e7b8901 100644
--- a/be/src/exec/partitioned-hash-join-node.h
+++ b/be/src/exec/partitioned-hash-join-node.h
@@ -53,7 +53,9 @@ class PartitionedHashJoinPlanNode : public BlockingJoinPlanNode {
std::vector<ScalarExpr*> probe_exprs_;
/// is_not_distinct_from_[i] is true if and only if the ith equi-join predicate is IS
- /// NOT DISTINCT FROM, rather than equality.
+ /// NOT DISTINCT FROM, rather than equality. This is the case when IS NOT DISTINCT FROM
+ /// is explicitly used as a join predicate or when joining Iceberg equality delete
+ /// files to data files.
std::vector<bool> is_not_distinct_from_;
/// Non-equi-join conjuncts from the ON clause.
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index 824c9fe17..4fc84476e 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -627,11 +627,13 @@ struct TIcebergContentFileStore {
1: optional map<string, THdfsFileDesc> path_hash_to_data_file_without_deletes
2: optional map<string, THdfsFileDesc> path_hash_to_data_file_with_deletes
3: optional map<string, THdfsFileDesc> path_hash_to_position_delete_file
- 7: optional map<string, THdfsFileDesc> path_hash_to_equality_delete_file
- 4: optional bool has_avro
- 5: optional bool has_orc
- 6: optional bool has_parquet
+ 4: optional map<string, THdfsFileDesc> path_hash_to_equality_delete_file
+ 5: optional list<i32> equality_ids
+ 6: optional bool has_avro
+ 7: optional bool has_orc
+ 8: optional bool has_parquet
}
+
// Represents a drop partition request for Iceberg tables
struct TIcebergDropPartitionRequest {
// List of affected file paths (could be empty if the drop partition
diff --git a/common/thrift/PlanNodes.thrift b/common/thrift/PlanNodes.thrift
index ed95cb7b6..c957a5af7 100644
--- a/common/thrift/PlanNodes.thrift
+++ b/common/thrift/PlanNodes.thrift
@@ -398,7 +398,13 @@ struct TEqJoinCondition {
1: required Exprs.TExpr left;
// right-hand side of "<a> = <b>"
2: required Exprs.TExpr right;
- // true if and only if operator is "<=>", also known as "IS NOT DISTINCT FROM"
+ // In SQL NULL values aren't equal to each other, in other words NULL == NULL is false.
+ // However, there are some cases when joining tables where we'd like to have the
+ // NULL == NULL comparison to return true. This flag is true in this case.
+ // One example is when we join Iceberg equality delete files to the data files where we
+ // want the NULLs in the delete files to match with the NULLs in the data files.
+ // Another example is when this operator is a "<=>", also known as "IS NOT DISTINCT
+ // FROM".
3: required bool is_not_distinct_from;
}
diff --git a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
index 41cbc0cba..21b600055 100644
--- a/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
+++ b/fe/src/main/java/org/apache/impala/analysis/BinaryPredicate.java
@@ -55,9 +55,8 @@ public class BinaryPredicate extends Predicate {
GT(">", "gt", TComparisonOp.GT),
DISTINCT_FROM("IS DISTINCT FROM", "distinctfrom", TComparisonOp.DISTINCT_FROM),
NOT_DISTINCT("IS NOT DISTINCT FROM", "notdistinct", TComparisonOp.NOT_DISTINCT),
- // Same as EQ, except it returns True if the rhs is NULL. There is no backend
- // function for this. The functionality is embedded in the hash-join
- // implementation.
+ // Same as EQ, except it returns True if both sides are NULL. There is no backend
+ // function for this. The functionality is embedded in the hash-join implementation.
NULL_MATCHING_EQ("=", "null_matching_eq", TComparisonOp.EQ);
private final String description_;
diff --git a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
index 553428617..16c68a382 100644
--- a/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/FeIcebergTable.java
@@ -761,7 +761,7 @@ public interface FeIcebergTable extends FeFsTable {
*/
public static IcebergContentFileStore loadAllPartition(
IcebergTable table, GroupedContentFiles icebergFiles)
- throws IOException {
+ throws IOException, ImpalaRuntimeException {
Map<String, HdfsPartition.FileDescriptor> hdfsFileDescMap = new HashMap<>();
Collection<HdfsPartition> partitions =
((HdfsTable)table.getFeFsTable()).partitionMap_.values();
@@ -781,16 +781,14 @@ public interface FeIcebergTable extends FeFsTable {
pathHashAndFd = getPathHashAndFd(dataFile, table, hdfsFileDescMap);
fileStore.addDataFileWithDeletes(pathHashAndFd.first, pathHashAndFd.second);
}
- for (DeleteFile deleteFile : icebergFiles.deleteFiles) {
+ for (DeleteFile deleteFile : icebergFiles.positionDeleteFiles) {
pathHashAndFd = getPathHashAndFd(deleteFile, table, hdfsFileDescMap);
- if (deleteFile.content().equals(FileContent.POSITION_DELETES)) {
- fileStore.addPositionDeleteFile(pathHashAndFd.first, pathHashAndFd.second);
- } else if (deleteFile.content().equals(FileContent.EQUALITY_DELETES)) {
- fileStore.addEqualityDeleteFile(pathHashAndFd.first, pathHashAndFd.second);
- } else {
- Preconditions.checkState(false,
- "Delete file with unknown kind: " + deleteFile.path().toString());
- }
+ fileStore.addPositionDeleteFile(pathHashAndFd.first, pathHashAndFd.second);
+ }
+ for (DeleteFile deleteFile : icebergFiles.equalityDeleteFiles) {
+ pathHashAndFd = getPathHashAndFd(deleteFile, table, hdfsFileDescMap);
+ fileStore.addEqualityDeleteFile(pathHashAndFd.first, pathHashAndFd.second,
+ deleteFile.equalityFieldIds());
}
return fileStore;
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java b/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
index 8f169ae74..20c2e4ea8 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergContentFileStore.java
@@ -21,13 +21,17 @@ import com.google.common.collect.Iterables;
import java.util.ArrayList;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
+import hiveexec.com.google.common.collect.Lists;
import org.apache.curator.shaded.com.google.common.base.Preconditions;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
+import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.fb.FbIcebergDataFileFormat;
import org.apache.impala.thrift.THdfsFileDesc;
import org.apache.impala.thrift.TIcebergContentFileStore;
@@ -98,6 +102,12 @@ public class IcebergContentFileStore {
private MapListContainer positionDeleteFiles_ = new MapListContainer();
private MapListContainer equalityDeleteFiles_ = new MapListContainer();
+ // The equality field IDs associated with the equality delete files.
+ // TODO IMPALA-12598: currently it's not supported to have equality delete files with
+ // different equality field ID lists, or in other words the equality delete files have
+ // to delete by the same columns.
+ private Set<Integer> equalityIds_ = new HashSet<>();
+
// Caches file descriptors loaded during time-travel queries.
private final ConcurrentMap<String, FileDescriptor> oldFileDescMap_ =
new ConcurrentHashMap<>();
@@ -127,10 +137,21 @@ public class IcebergContentFileStore {
}
}
- public void addEqualityDeleteFile(String pathHash, FileDescriptor desc) {
- if (equalityDeleteFiles_.add(pathHash, desc)) {
- updateFileFormats(desc);
+ public void addEqualityDeleteFile(String pathHash, FileDescriptor desc,
+ List<Integer> equalityIds) throws ImpalaRuntimeException {
+ if (equalityIds.isEmpty()) {
+ throw new ImpalaRuntimeException("No equality IDs for equality delete file " +
+ desc.getPath());
}
+ if (equalityIds_.isEmpty()) {
+ equalityIds_.addAll(equalityIds);
+ } else if (equalityIds_.size() != equalityIds.size() ||
+ !equalityIds_.containsAll(equalityIds)) {
+ throw new ImpalaRuntimeException(String.format("Equality delete files with " +
+ "different equality field ID lists aren't supported. %s vs %s", equalityIds,
+ equalityIds_));
+ }
+ if (equalityDeleteFiles_.add(pathHash, desc)) updateFileFormats(desc);
}
// This is only invoked during time travel, when we are querying a snapshot that has
@@ -171,6 +192,8 @@ public class IcebergContentFileStore {
return equalityDeleteFiles_.getList();
}
+ public Set<Integer> getEqualityIds() { return equalityIds_; }
+
public long getNumFiles() {
return dataFilesWithoutDeletes_.getNumFiles() +
dataFilesWithDeletes_.getNumFiles() +
@@ -213,6 +236,7 @@ public class IcebergContentFileStore {
ret.setPath_hash_to_data_file_with_deletes(dataFilesWithDeletes_.toThrift());
ret.setPath_hash_to_position_delete_file(positionDeleteFiles_.toThrift());
ret.setPath_hash_to_equality_delete_file(equalityDeleteFiles_.toThrift());
+ ret.setEquality_ids(Lists.newArrayList(equalityIds_));
ret.setHas_avro(hasAvro_);
ret.setHas_orc(hasOrc_);
ret.setHas_parquet(hasParquet_);
@@ -243,6 +267,7 @@ public class IcebergContentFileStore {
tFileStore.getPath_hash_to_equality_delete_file(),
networkAddresses, hostIndex);
}
+ ret.equalityIds_.addAll(tFileStore.getEquality_ids());
ret.hasAvro_ = tFileStore.isSetHas_avro() ? tFileStore.isHas_avro() : false;
ret.hasOrc_ = tFileStore.isSetHas_orc() ? tFileStore.isHas_orc() : false;
ret.hasParquet_ = tFileStore.isSetHas_parquet() ? tFileStore.isHas_parquet() : false;
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergDeleteTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergDeleteTable.java
new file mode 100644
index 000000000..ed47c7690
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergDeleteTable.java
@@ -0,0 +1,169 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.lang.NotImplementedException;
+import org.apache.iceberg.Table;
+import org.apache.impala.analysis.IcebergPartitionSpec;
+import org.apache.impala.catalog.CatalogObject.ThriftObjectType;
+import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
+import org.apache.impala.thrift.TCompressionCodec;
+import org.apache.impala.thrift.THdfsTable;
+import org.apache.impala.thrift.TIcebergCatalog;
+import org.apache.impala.thrift.TIcebergFileFormat;
+import org.apache.impala.thrift.TIcebergPartitionStats;
+import org.apache.impala.thrift.TTableDescriptor;
+import org.apache.impala.thrift.TTableStats;
+
+/**
+ * Base class for the virtual table implementations for Iceberg deletes, like position or
+ * equality deletes.
+ */
+public abstract class IcebergDeleteTable extends VirtualTable implements FeIcebergTable {
+ protected final static int INVALID_MAP_KEY_ID = -1;
+ protected final static int INVALID_MAP_VALUE_ID = -1;
+
+ protected FeIcebergTable baseTable_;
+ protected Set<FileDescriptor> deleteFiles_;
+ protected long deleteRecordsCount_;
+
+ public IcebergDeleteTable(FeIcebergTable baseTable, String name,
+ Set<FileDescriptor> deleteFiles, long deleteRecordsCount) {
+ super(baseTable.getMetaStoreTable(), baseTable.getDb(), name,
+ baseTable.getOwnerUser());
+ baseTable_ = baseTable;
+ deleteFiles_ = deleteFiles;
+ deleteRecordsCount_ = deleteRecordsCount;
+ }
+
+ public FeIcebergTable getBaseTable() { return baseTable_; }
+
+ @Override
+ public long getNumRows() {
+ return deleteRecordsCount_;
+ }
+
+ @Override
+ public TTableStats getTTableStats() {
+ long totalBytes = 0;
+ for (FileDescriptor df : deleteFiles_) {
+ totalBytes += df.getFileLength();
+ }
+ TTableStats ret = new TTableStats(getNumRows());
+ ret.setTotal_file_bytes(totalBytes);
+ return ret;
+ }
+
+ /**
+ * Return same descriptor as the base table, but with a schema that corresponds to
+ * the schema of this delete table (including virtual columns).
+ */
+ @Override
+ public TTableDescriptor toThriftDescriptor(int tableId,
+ Set<Long> referencedPartitions) {
+ TTableDescriptor desc =
+ baseTable_.toThriftDescriptor(tableId, referencedPartitions);
+ desc.setColumnDescriptors(FeCatalogUtils.getTColumnDescriptors(this));
+ return desc;
+ }
+
+ @Override
+ public IcebergContentFileStore getContentFileStore() {
+ throw new NotImplementedException("This should never be called.");
+ }
+
+ @Override
+ public Map<String, TIcebergPartitionStats> getIcebergPartitionStats(){
+ return null;
+ }
+
+ @Override
+ public FeFsTable getFeFsTable() {
+ return baseTable_.getFeFsTable();
+ }
+
+ @Override
+ public TIcebergCatalog getIcebergCatalog() {
+ return null;
+ }
+
+ @Override
+ public Table getIcebergApiTable() {
+ return baseTable_.getIcebergApiTable();
+ }
+
+ @Override
+ public String getIcebergCatalogLocation() {
+ return null;
+ }
+
+ @Override
+ public TIcebergFileFormat getIcebergFileFormat() {
+ return baseTable_.getIcebergFileFormat();
+ }
+
+ @Override
+ public TCompressionCodec getIcebergParquetCompressionCodec() {
+ return null;
+ }
+
+ @Override
+ public long getIcebergParquetRowGroupSize() {
+ return baseTable_.getIcebergParquetRowGroupSize();
+ }
+
+ @Override
+ public long getIcebergParquetPlainPageSize() {
+ return baseTable_.getIcebergParquetPlainPageSize();
+ }
+
+ @Override
+ public long getIcebergParquetDictPageSize() {
+ return baseTable_.getIcebergParquetDictPageSize();
+ }
+
+ @Override
+ public String getIcebergTableLocation() {
+ return null;
+ }
+
+ @Override
+ public List<IcebergPartitionSpec> getPartitionSpecs() {
+ return baseTable_.getPartitionSpecs();
+ }
+
+ @Override
+ public IcebergPartitionSpec getDefaultPartitionSpec() {
+ return null;
+ }
+
+ @Override
+ public int getDefaultPartitionSpecId() {
+ return -1;
+ }
+
+ @Override
+ public THdfsTable transformToTHdfsTable(boolean updatePartitionFlag,
+ ThriftObjectType type) {
+ throw new IllegalStateException("not implemented here");
+ }
+}
\ No newline at end of file
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergEqualityDeleteTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergEqualityDeleteTable.java
new file mode 100644
index 000000000..5d53ae095
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergEqualityDeleteTable.java
@@ -0,0 +1,68 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
+import org.apache.impala.common.ImpalaRuntimeException;
+import org.apache.impala.util.IcebergSchemaConverter;
+
+import org.apache.iceberg.types.Types.NestedField;
+
+/**
+ * Iceberg equality delete table is created on the fly during planning. It belongs to an
+ * actual Iceberg table (referred to as 'baseTable_'), but has a schema that corresponds
+ * to the file schema of equality delete files. Therefore with the help of it we can
+ * do an ANTI JOIN between data files and equality delete files.
+ */
+public class IcebergEqualityDeleteTable extends IcebergDeleteTable {
+
+ public IcebergEqualityDeleteTable(FeIcebergTable baseTable, String name,
+ Set<FileDescriptor> deleteFiles, Set<Integer> equalityIds, long deleteRecordsCount)
+ throws ImpalaRuntimeException {
+ super(baseTable, name, deleteFiles, deleteRecordsCount);
+
+ int columnPos = 0;
+ for (Integer eqId : equalityIds) {
+ ++columnPos;
+ NestedField field = baseTable.getIcebergSchema().findField(eqId);
+ Type colType = IcebergSchemaConverter.toImpalaType(field.type());
+ if (colType.isComplexType()) {
+ throw new ImpalaRuntimeException(
+ "Equality ID for nested types isn't supported: '" + field.name() + "'");
+ } else if (colType.isFloatingPointType()) {
+ throw new ImpalaRuntimeException(
+ "Equality ID for floating point types isn't supported: '" +
+ field.name() + "'");
+ }
+
+ Column equalityCol = new IcebergColumn(field.name(), colType, field.doc(),
+ columnPos, field.fieldId(), INVALID_MAP_KEY_ID, INVALID_MAP_VALUE_ID,
+ field.isOptional());
+ addColumn(equalityCol);
+ }
+ }
+
+ @Override
+ public List<VirtualColumn> getVirtualColumns() {
+ return Arrays.asList(VirtualColumn.ICEBERG_DATA_SEQUENCE_NUMBER);
+ }
+}
\ No newline at end of file
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergPositionDeleteTable.java b/fe/src/main/java/org/apache/impala/catalog/IcebergPositionDeleteTable.java
index 7aad988cc..fb1667638 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergPositionDeleteTable.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergPositionDeleteTable.java
@@ -18,35 +18,18 @@
package org.apache.impala.catalog;
import java.util.Collections;
-import java.util.List;
-import java.util.Map;
import java.util.Set;
-import org.apache.commons.lang.NotImplementedException;
-import org.apache.iceberg.Table;
-import org.apache.impala.catalog.CatalogObject.ThriftObjectType;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
-import org.apache.impala.analysis.IcebergPartitionSpec;
import org.apache.impala.thrift.TColumnStats;
-import org.apache.impala.thrift.TCompressionCodec;
-import org.apache.impala.thrift.THdfsTable;
-import org.apache.impala.thrift.TIcebergCatalog;
-import org.apache.impala.thrift.TIcebergFileFormat;
-import org.apache.impala.thrift.TIcebergPartitionStats;
-import org.apache.impala.thrift.TTableDescriptor;
-import org.apache.impala.thrift.TTableStats;
/**
* Iceberg position delete table is created on the fly during planning. It belongs to an
* actual Iceberg table (referred to as 'baseTable_'), but has a schema that corresponds
* to the file schema of position delete files. Therefore with the help of it we can
- * do an ANTI JOIN between data files and delete files.
+ * do an ANTI JOIN between data files and position delete files.
*/
-public class IcebergPositionDeleteTable extends VirtualTable implements FeIcebergTable {
- private FeIcebergTable baseTable_;
- private Set<FileDescriptor> deleteFiles_;
- private long deleteRecordsCount_;
-
+public class IcebergPositionDeleteTable extends IcebergDeleteTable {
public static String FILE_PATH_COLUMN = "file_path";
public static String POS_COLUMN = "pos";
@@ -58,24 +41,19 @@ public class IcebergPositionDeleteTable extends VirtualTable implements FeIceber
public IcebergPositionDeleteTable(FeIcebergTable baseTable, String name,
Set<FileDescriptor> deleteFiles,
long deleteRecordsCount, TColumnStats filePathsStats) {
- super(baseTable.getMetaStoreTable(), baseTable.getDb(), name,
- baseTable.getOwnerUser());
- baseTable_ = baseTable;
- deleteFiles_ = deleteFiles;
- deleteRecordsCount_ = deleteRecordsCount;
+ super(baseTable, name, deleteFiles, deleteRecordsCount);
Column filePath = new IcebergColumn(FILE_PATH_COLUMN, Type.STRING, /*comment=*/"",
- colsByPos_.size(), IcebergTable.V2_FILE_PATH_FIELD_ID, -1, -1,
- /*nullable=*/false);
+ colsByPos_.size(), IcebergTable.V2_FILE_PATH_FIELD_ID, INVALID_MAP_KEY_ID,
+ INVALID_MAP_VALUE_ID, /*nullable=*/false);
Column pos = new IcebergColumn(POS_COLUMN, Type.BIGINT, /*comment=*/"",
- colsByPos_.size(), IcebergTable.V2_POS_FIELD_ID, -1, -1, /*nullable=*/false);
+ colsByPos_.size(), IcebergTable.V2_POS_FIELD_ID, INVALID_MAP_KEY_ID,
+ INVALID_MAP_VALUE_ID, /*nullable=*/false);
filePath.updateStats(filePathsStats);
pos.updateStats(getPosStats(pos));
addColumn(filePath);
addColumn(pos);
}
- public FeIcebergTable getBaseTable() { return baseTable_; }
-
private TColumnStats getPosStats(Column pos) {
TColumnStats colStats = new TColumnStats();
colStats.num_distinct_values = deleteRecordsCount_;
@@ -83,114 +61,4 @@ public class IcebergPositionDeleteTable extends VirtualTable implements FeIceber
colStats.max_size = pos.getType().getSlotSize();
return colStats;
}
-
- @Override
- public long getNumRows() {
- return deleteRecordsCount_;
- }
-
- @Override
- public TTableStats getTTableStats() {
- long totalBytes = 0;
- for (FileDescriptor df : deleteFiles_) {
- totalBytes += df.getFileLength();
- }
- TTableStats ret = new TTableStats(getNumRows());
- ret.setTotal_file_bytes(totalBytes);
- return ret;
- }
-
- /**
- * Return same descriptor as the base table, but with a schema that corresponds to
- * the position delete file schema ('file_path', 'pos').
- */
- @Override
- public TTableDescriptor toThriftDescriptor(int tableId,
- Set<Long> referencedPartitions) {
- TTableDescriptor desc = baseTable_.toThriftDescriptor(tableId, referencedPartitions);
- desc.setColumnDescriptors(FeCatalogUtils.getTColumnDescriptors(this));
- return desc;
- }
-
- @Override
- public IcebergContentFileStore getContentFileStore() {
- throw new NotImplementedException("This should never be called.");
- }
-
- @Override
- public Map<String, TIcebergPartitionStats> getIcebergPartitionStats() {
- return null;
- }
-
- @Override
- public FeFsTable getFeFsTable() {
- return baseTable_.getFeFsTable();
- }
-
- @Override
- public TIcebergCatalog getIcebergCatalog() {
- return null;
- }
-
- @Override
- public Table getIcebergApiTable() {
- return baseTable_.getIcebergApiTable();
- }
-
- @Override
- public String getIcebergCatalogLocation() {
- return null;
- }
-
- @Override
- public TIcebergFileFormat getIcebergFileFormat() {
- return baseTable_.getIcebergFileFormat();
- }
-
- @Override
- public TCompressionCodec getIcebergParquetCompressionCodec() {
- return null;
- }
-
- @Override
- public long getIcebergParquetRowGroupSize() {
- return baseTable_.getIcebergParquetRowGroupSize();
- }
-
- @Override
- public long getIcebergParquetPlainPageSize() {
- return baseTable_.getIcebergParquetPlainPageSize();
- }
-
- @Override
- public long getIcebergParquetDictPageSize() {
- return baseTable_.getIcebergParquetDictPageSize();
- }
-
- @Override
- public String getIcebergTableLocation() {
- return null;
- }
-
- @Override
- public List<IcebergPartitionSpec> getPartitionSpecs() {
- return baseTable_.getPartitionSpecs();
- }
-
- @Override
- public IcebergPartitionSpec getDefaultPartitionSpec() {
- return null;
- }
-
- @Override
- public int getDefaultPartitionSpecId() {
- return -1;
- }
-
- @Override
- public THdfsTable transformToTHdfsTable(boolean updatePartitionFlag,
- ThriftObjectType type) {
- throw new IllegalStateException("not implemented here");
- }
-
}
diff --git a/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java b/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
index a01f43c10..dbd170734 100644
--- a/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
+++ b/fe/src/main/java/org/apache/impala/catalog/iceberg/GroupedContentFiles.java
@@ -22,23 +22,27 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import com.google.common.collect.Iterables;
import org.apache.iceberg.ContentFile;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.DeleteFile;
+import org.apache.iceberg.FileContent;
import org.apache.iceberg.FileScanTask;
import org.apache.iceberg.io.CloseableIterable;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Iterables;
+
/**
* Struct-like object to group different Iceberg content files:
* - data files without deleted rows
* - data files with deleted rows
- * - delete files
+ * - delete files (position and equality)
*/
public class GroupedContentFiles {
public List<DataFile> dataFilesWithoutDeletes = new ArrayList<>();
public List<DataFile> dataFilesWithDeletes = new ArrayList<>();
- public Set<DeleteFile> deleteFiles = new HashSet<>();
+ public Set<DeleteFile> positionDeleteFiles = new HashSet<>();
+ public Set<DeleteFile> equalityDeleteFiles = new HashSet<>();
public GroupedContentFiles() { }
@@ -48,18 +52,26 @@ public class GroupedContentFiles {
dataFilesWithoutDeletes.add(scanTask.file());
} else {
dataFilesWithDeletes.add(scanTask.file());
- deleteFiles.addAll(scanTask.deletes());
+ for (DeleteFile delFile : scanTask.deletes()) {
+ if (delFile.content() == FileContent.POSITION_DELETES) {
+ positionDeleteFiles.add(delFile);
+ } else {
+ Preconditions.checkState(delFile.content() == FileContent.EQUALITY_DELETES);
+ equalityDeleteFiles.add(delFile);
+ }
+ }
}
}
}
public Iterable<ContentFile<?>> getAllContentFiles() {
- return Iterables.concat(dataFilesWithoutDeletes, dataFilesWithDeletes, deleteFiles);
+ return Iterables.concat(dataFilesWithoutDeletes, dataFilesWithDeletes,
+ positionDeleteFiles, equalityDeleteFiles);
}
public int size() {
return dataFilesWithDeletes.size() + dataFilesWithoutDeletes.size() +
- deleteFiles.size();
+ positionDeleteFiles.size() + equalityDeleteFiles.size();
}
public boolean isEmpty() {
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index 7863d8b67..13b0251f4 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -49,8 +49,6 @@ import org.apache.impala.analysis.BinaryPredicate;
import org.apache.impala.analysis.BinaryPredicate.Operator;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.IcebergExpressionCollector;
-import org.apache.impala.analysis.InPredicate;
-import org.apache.impala.analysis.IsNullPredicate;
import org.apache.impala.analysis.JoinOperator;
import org.apache.impala.analysis.MultiAggregateInfo;
import org.apache.impala.analysis.SlotDescriptor;
@@ -66,6 +64,7 @@ import org.apache.impala.catalog.FeIcebergTable;
import org.apache.impala.catalog.HdfsPartition.FileDescriptor;
import org.apache.impala.catalog.IcebergColumn;
import org.apache.impala.catalog.IcebergContentFileStore;
+import org.apache.impala.catalog.IcebergEqualityDeleteTable;
import org.apache.impala.catalog.IcebergPositionDeleteTable;
import org.apache.impala.catalog.IcebergTable;
import org.apache.impala.catalog.TableLoadingException;
@@ -113,16 +112,22 @@ public class IcebergScanPlanner {
private final List<Expr> untranslatedExpressions_ = new ArrayList<>();
// Conjuncts on columns not involved in IDENTITY-partitioning.
private List<Expr> nonIdentityConjuncts_ = new ArrayList<>();
+
+ // Containers for different groupings of file descriptors.
private List<FileDescriptor> dataFilesWithoutDeletes_ = new ArrayList<>();
private List<FileDescriptor> dataFilesWithDeletes_ = new ArrayList<>();
- private Set<FileDescriptor> deleteFiles_ = new HashSet<>();
-
+ private Set<FileDescriptor> positionDeleteFiles_ = new HashSet<>();
+ private Set<FileDescriptor> equalityDeleteFiles_ = new HashSet<>();
+ // The equality field IDs to be used for the equality delete files.
+ private Set<Integer> equalityIds_ = new HashSet<>();
// Statistics about the data and delete files. Useful for memory estimates of the
// ANTI JOIN
- private long deletesRecordCount_ = 0;
+ private long positionDeletesRecordCount_ = 0;
+ private long equalityDeletesRecordCount_ = 0;
private long dataFilesWithDeletesSumPaths_ = 0;
private long dataFilesWithDeletesMaxPath_ = 0;
+ private Set<Long> equalityDeleteSequenceNumbers_ = new HashSet<>();
public IcebergScanPlanner(Analyzer analyzer, PlannerContext ctx,
TableRef iceTblRef, List<Expr> conjuncts, MultiAggregateInfo aggInfo)
@@ -163,23 +168,21 @@ public class IcebergScanPlanner {
private void setFileDescriptorsBasedOnFileStore() throws ImpalaException {
IcebergContentFileStore fileStore = getIceTable().getContentFileStore();
- if (!fileStore.getEqualityDeleteFiles().isEmpty()) {
- // TODO(IMPALA-11388): Add support for equality deletes.
- FileDescriptor firstEqualityDeleteFile = fileStore.getEqualityDeleteFiles().get(0);
- throw new ImpalaRuntimeException(String.format(
- "Iceberg table %s has EQUALITY delete file which is currently " +
- "not supported by Impala, for example: %s",
- getIceTable().getFullName(),
- firstEqualityDeleteFile.getAbsolutePath(getIceTable().getLocation())));
- }
dataFilesWithoutDeletes_ = fileStore.getDataFilesWithoutDeletes();
dataFilesWithDeletes_ = fileStore.getDataFilesWithDeletes();
- deleteFiles_ = new HashSet<>(fileStore.getPositionDeleteFiles());
+ positionDeleteFiles_ = new HashSet<>(fileStore.getPositionDeleteFiles());
+ equalityDeleteFiles_ = new HashSet<>(fileStore.getEqualityDeleteFiles());
+ equalityIds_ = fileStore.getEqualityIds();
+
updateDeleteStatistics();
}
+ private boolean noDeleteFiles() {
+ return positionDeleteFiles_.isEmpty() && equalityDeleteFiles_.isEmpty();
+ }
+
private PlanNode createIcebergScanPlanImpl() throws ImpalaException {
- if (deleteFiles_.isEmpty()) {
+ if (noDeleteFiles()) {
// If there are no delete files we can just create a single SCAN node.
Preconditions.checkState(dataFilesWithDeletes_.isEmpty());
PlanNode ret = new IcebergScanNode(ctx_.getNextNodeId(), tblRef_, conjuncts_,
@@ -188,7 +191,12 @@ public class IcebergScanPlanner {
ret.init(analyzer_);
return ret;
}
- PlanNode joinNode = createPositionJoinNode();
+
+ PlanNode joinNode = null;
+ if (!positionDeleteFiles_.isEmpty()) joinNode = createPositionJoinNode();
+
+ if (!equalityDeleteFiles_.isEmpty()) joinNode = createEqualityJoinNode(joinNode);
+ Preconditions.checkNotNull(joinNode);
// If the count star query can be optimized for Iceberg V2 table, the number of rows
// of all DataFiles without corresponding DeleteFiles can be calculated by Iceberg
@@ -219,7 +227,7 @@ public class IcebergScanPlanner {
}
private PlanNode createPositionJoinNode() throws ImpalaException {
- Preconditions.checkState(deletesRecordCount_ != 0);
+ Preconditions.checkState(positionDeletesRecordCount_ != 0);
Preconditions.checkState(dataFilesWithDeletesSumPaths_ != 0);
Preconditions.checkState(dataFilesWithDeletesMaxPath_ != 0);
// The followings just create separate scan nodes for data files and position delete
@@ -228,12 +236,13 @@ public class IcebergScanPlanner {
PlanNodeId deleteScanNodeId = ctx_.getNextNodeId();
IcebergPositionDeleteTable deleteTable = new IcebergPositionDeleteTable(getIceTable(),
getIceTable().getName() + "-POSITION-DELETE-" + deleteScanNodeId.toString(),
- deleteFiles_, deletesRecordCount_, getFilePathStats());
+ positionDeleteFiles_, positionDeletesRecordCount_, getFilePathStats());
analyzer_.addVirtualTable(deleteTable);
TableRef deleteDeltaRef = TableRef.newTableRef(analyzer_,
Arrays.asList(deleteTable.getDb().getName(), deleteTable.getName()),
tblRef_.getUniqueAlias() + "-position-delete");
addDataVirtualPositionSlots(tblRef_);
+ if (!equalityDeleteFiles_.isEmpty()) addSlotsForEqualityDelete(equalityIds_, tblRef_);
addDeletePositionSlots(deleteDeltaRef);
IcebergScanNode dataScanNode = new IcebergScanNode(
dataScanNodeId, tblRef_, conjuncts_, aggInfo_, dataFilesWithDeletes_,
@@ -244,14 +253,14 @@ public class IcebergScanPlanner {
deleteDeltaRef,
Collections.emptyList(), /*conjuncts*/
aggInfo_,
- Lists.newArrayList(deleteFiles_),
+ Lists.newArrayList(positionDeleteFiles_),
Collections.emptyList(), /*nonIdentityConjuncts*/
Collections.emptyList()); /*skippedConjuncts*/
deleteScanNode.init(analyzer_);
// Now let's create the JOIN node
List<BinaryPredicate> positionJoinConjuncts = createPositionJoinConjuncts(
- analyzer_, tblRef_.getDesc(), deleteDeltaRef.getDesc());
+ analyzer_, tblRef_.getDesc(), deleteDeltaRef.getDesc());
TQueryOptions queryOpts = analyzer_.getQueryCtx().client_request.query_options;
JoinNode joinNode = null;
@@ -270,38 +279,43 @@ public class IcebergScanPlanner {
}
private void addDataVirtualPositionSlots(TableRef tblRef) throws AnalysisException {
- List<String> rawPath = new ArrayList<>();
- rawPath.add(tblRef.getUniqueAlias());
- // Add slot refs for position delete fields;
- String[] posFields = {VirtualColumn.INPUT_FILE_NAME.getName(),
- VirtualColumn.FILE_POSITION.getName()};
- for (String posField : posFields) {
- rawPath.add(posField);
+ List<String> rawPath = Lists.newArrayList(
+ tblRef.getUniqueAlias(), VirtualColumn.INPUT_FILE_NAME.getName());
+ SlotDescriptor fileNameSlotDesc =
+ SingleNodePlanner.addSlotRefToDesc(analyzer_, rawPath);
+ fileNameSlotDesc.setStats(virtualInputFileNameStats());
+
+ rawPath = Lists.newArrayList(
+ tblRef.getUniqueAlias(), VirtualColumn.FILE_POSITION.getName());
+ SlotDescriptor filePosSlotDesc =
+ SingleNodePlanner.addSlotRefToDesc(analyzer_, rawPath);
+ filePosSlotDesc.setStats(virtualFilePositionStats());
+ }
+
+ private void addSlotsForEqualityDelete(Set<Integer> equalityIds, TableRef tblRef)
+ throws AnalysisException {
+ List<String> rawPath = Lists.newArrayList(
+ tblRef.getUniqueAlias(), VirtualColumn.ICEBERG_DATA_SEQUENCE_NUMBER.getName());
+ SlotDescriptor slotDesc = SingleNodePlanner.addSlotRefToDesc(analyzer_, rawPath);
+ slotDesc.setStats(virtualDataSeqNumStats());
+
+ Preconditions.checkState(!equalityIds.isEmpty());
+ for (Integer eqId : equalityIds) {
+ String eqColName = getIceTable().getIcebergSchema().findColumnName(eqId);
+ Preconditions.checkNotNull(eqColName);
+ rawPath = Lists.newArrayList(tblRef.getUniqueAlias(), eqColName);
SingleNodePlanner.addSlotRefToDesc(analyzer_, rawPath);
- rawPath.remove(rawPath.size() - 1);
- }
- for (SlotDescriptor insertSlotDesc : tblRef.getDesc().getSlots()) {
- TVirtualColumnType virtColType = insertSlotDesc.getVirtualColumnType();
- if (virtColType == TVirtualColumnType.INPUT_FILE_NAME) {
- insertSlotDesc.setStats(virtualInputFileNameStats());
- } else if (virtColType == TVirtualColumnType.FILE_POSITION) {
- insertSlotDesc.setStats(virtualFilePositionStats());
- }
}
}
private void addDeletePositionSlots(TableRef tblRef)
throws AnalysisException {
- List<String> rawPath = new ArrayList<>();
- rawPath.add(tblRef.getUniqueAlias());
- // Add slot refs for position delete fields;
- String[] posFields = {IcebergPositionDeleteTable.FILE_PATH_COLUMN,
- IcebergPositionDeleteTable.POS_COLUMN};
- for (String posField : posFields) {
- rawPath.add(posField);
- SingleNodePlanner.addSlotRefToDesc(analyzer_, rawPath);
- rawPath.remove(rawPath.size() - 1);
- }
+ SingleNodePlanner.addSlotRefToDesc(analyzer_,
+ Lists.newArrayList(
+ tblRef.getUniqueAlias(), IcebergPositionDeleteTable.FILE_PATH_COLUMN));
+ SingleNodePlanner.addSlotRefToDesc(analyzer_,
+ Lists.newArrayList(
+ tblRef.getUniqueAlias(), IcebergPositionDeleteTable.POS_COLUMN));
}
private List<BinaryPredicate> createPositionJoinConjuncts(Analyzer analyzer,
@@ -344,6 +358,48 @@ public class IcebergScanPlanner {
return ret;
}
+ private Pair<List<BinaryPredicate>, List<Expr>> createEqualityJoinConjuncts(
+ Analyzer analyzer, TupleDescriptor dataTupleDesc, TupleDescriptor deleteTupleDesc)
+ throws AnalysisException, ImpalaRuntimeException {
+ // Pre-process the slots for faster lookup by field ID.
+ Map<Integer, SlotDescriptor> fieldIdToIcebergColumn = new HashMap<>();
+ SlotDescriptor dataSeqNumSlot = null;
+ for (SlotDescriptor dataSlotDesc : dataTupleDesc.getSlots()) {
+ if (dataSlotDesc.getVirtualColumnType() ==
+ TVirtualColumnType.ICEBERG_DATA_SEQUENCE_NUMBER) {
+ dataSeqNumSlot = dataSlotDesc;
+ } else if (dataSlotDesc.getColumn() instanceof IcebergColumn) {
+ IcebergColumn icebergCol = (IcebergColumn)dataSlotDesc.getColumn();
+ fieldIdToIcebergColumn.put(icebergCol.getFieldId(), dataSlotDesc);
+ }
+ }
+
+ List<BinaryPredicate> eqPredicates = new ArrayList<>();
+ List<Expr> seqNumPredicate = new ArrayList<>();
+ for (SlotDescriptor deleteSlotDesc : deleteTupleDesc.getSlots()) {
+ if (deleteSlotDesc.getVirtualColumnType() ==
+ TVirtualColumnType.ICEBERG_DATA_SEQUENCE_NUMBER) {
+ BinaryPredicate pred = new BinaryPredicate(Operator.LT,
+ new SlotRef(dataSeqNumSlot), new SlotRef(deleteSlotDesc));
+ pred.analyze(analyzer);
+ seqNumPredicate.add(pred);
+ } else {
+ Preconditions.checkState(deleteSlotDesc.getColumn() instanceof IcebergColumn);
+ int fieldId = ((IcebergColumn)deleteSlotDesc.getColumn()).getFieldId();
+ if (!fieldIdToIcebergColumn.containsKey(fieldId)) {
+ throw new ImpalaRuntimeException("Field ID not found in table: " + fieldId);
+ }
+ SlotRef dataSlotRef = new SlotRef(fieldIdToIcebergColumn.get(fieldId));
+ SlotRef deleteSlotRef = new SlotRef(deleteSlotDesc);
+ BinaryPredicate eqColPred = new BinaryPredicate(
+ Operator.NOT_DISTINCT, dataSlotRef, deleteSlotRef);
+ eqColPred.analyze(analyzer);
+ eqPredicates.add(eqColPred);
+ }
+ }
+ return new Pair<>(eqPredicates, seqNumPredicate);
+ }
+
private ColumnStats virtualInputFileNameStats() {
ColumnStats ret = new ColumnStats(Type.STRING);
ret.setNumDistinctValues(dataFilesWithDeletes_.size());
@@ -352,10 +408,77 @@ public class IcebergScanPlanner {
private ColumnStats virtualFilePositionStats() {
ColumnStats ret = new ColumnStats(Type.BIGINT);
- ret.setNumDistinctValues(deletesRecordCount_ / dataFilesWithDeletes_.size());
+ ret.setNumDistinctValues(positionDeletesRecordCount_ / dataFilesWithDeletes_.size());
return ret;
}
+ private ColumnStats virtualDataSeqNumStats() {
+ ColumnStats ret = new ColumnStats(Type.BIGINT);
+ ret.setNumDistinctValues(equalityDeleteSequenceNumbers_.size());
+ return ret;
+ }
+
+ private PlanNode createEqualityJoinNode(PlanNode positionJoinNode)
+ throws ImpalaException {
+ Preconditions.checkState(!equalityDeleteFiles_.isEmpty());
+ Preconditions.checkState(equalityDeletesRecordCount_ > 0);
+
+ if (getIceTable().getPartitionSpecs().size() > 1) {
+ throw new ImpalaRuntimeException("Equality delete files are not supported for " +
+ "tables with partition evolution");
+ }
+
+ PlanNode leftSideOfJoin = null;
+ if (positionJoinNode != null) {
+ leftSideOfJoin = positionJoinNode;
+ } else {
+ PlanNodeId dataScanNodeId = ctx_.getNextNodeId();
+ IcebergScanNode dataScanNode = new IcebergScanNode(
+ dataScanNodeId, tblRef_, conjuncts_, aggInfo_, dataFilesWithDeletes_,
+ nonIdentityConjuncts_, getSkippedConjuncts());
+ addSlotsForEqualityDelete(equalityIds_, tblRef_);
+ dataScanNode.init(analyzer_);
+
+ leftSideOfJoin = dataScanNode;
+ }
+
+ JoinNode joinNode = null;
+ PlanNodeId deleteScanNodeId = ctx_.getNextNodeId();
+ IcebergEqualityDeleteTable deleteTable =
+ new IcebergEqualityDeleteTable(getIceTable(),
+ getIceTable().getName() + "-EQUALITY-DELETE-" + deleteScanNodeId.toString(),
+ equalityDeleteFiles_, equalityIds_, equalityDeletesRecordCount_);
+ analyzer_.addVirtualTable(deleteTable);
+
+ TableRef deleteTblRef = TableRef.newTableRef(analyzer_,
+ Arrays.asList(deleteTable.getDb().getName(), deleteTable.getName()),
+ tblRef_.getUniqueAlias() + "-equality-delete-" + deleteScanNodeId.toString());
+ addSlotsForEqualityDelete(equalityIds_, deleteTblRef);
+
+ // TODO IMPALA-12608: As an optimization we can populate the conjuncts below that are
+ // relevant for the delete scan node.
+ IcebergScanNode deleteScanNode = new IcebergScanNode(
+ deleteScanNodeId,
+ deleteTblRef,
+ Collections.emptyList(), /*conjuncts*/
+ aggInfo_,
+ Lists.newArrayList(equalityDeleteFiles_),
+ Collections.emptyList(), /*nonIdentityConjuncts*/
+ Collections.emptyList()); /*skippedConjuncts*/
+ deleteScanNode.init(analyzer_);
+
+ Pair<List<BinaryPredicate>, List<Expr>> equalityJoinConjuncts =
+ createEqualityJoinConjuncts(
+ analyzer_, tblRef_.getDesc(), deleteTblRef.getDesc());
+
+ joinNode = new HashJoinNode(leftSideOfJoin, deleteScanNode,
+ /*straight_join=*/true, DistributionMode.NONE, JoinOperator.LEFT_ANTI_JOIN,
+ equalityJoinConjuncts.first, equalityJoinConjuncts.second);
+ joinNode.setId(ctx_.getNextNodeId());
+ joinNode.init(analyzer_);
+ return joinNode;
+ }
+
private void filterFileDescriptors() throws ImpalaException {
TimeTravelSpec timeTravelSpec = tblRef_.getTimeTravelSpec();
@@ -375,16 +498,15 @@ public class IcebergScanPlanner {
} else {
dataFilesWithDeletes_.add(fileDesc.first);
for (DeleteFile delFile : fileScanTask.deletes()) {
- // TODO(IMPALA-11388): Add support for equality deletes.
- if (delFile.content() == FileContent.EQUALITY_DELETES) {
- throw new ImpalaRuntimeException(String.format(
- "Iceberg table %s has EQUALITY delete file which is currently " +
- "not supported by Impala, for example: %s", getIceTable().getFullName(),
- delFile.path()));
- }
Pair<FileDescriptor, Boolean> delFileDesc = getFileDescriptor(delFile);
if (!delFileDesc.second) ++dataFilesCacheMisses;
- deleteFiles_.add(delFileDesc.first);
+ if (delFile.content() == FileContent.EQUALITY_DELETES) {
+ equalityDeleteFiles_.add(delFileDesc.first);
+ addEqualityIds(delFile.equalityFieldIds());
+ } else {
+ Preconditions.checkState(delFile.content() == FileContent.POSITION_DELETES);
+ positionDeleteFiles_.add(delFileDesc.first);
+ }
}
}
}
@@ -401,6 +523,19 @@ public class IcebergScanPlanner {
updateDeleteStatistics();
}
+
+ private void addEqualityIds(List<Integer> equalityFieldIds)
+ throws ImpalaRuntimeException {
+ if (equalityIds_.isEmpty()) {
+ equalityIds_.addAll(equalityFieldIds);
+ } else if (equalityIds_.size() != equalityFieldIds.size() ||
+ !equalityIds_.containsAll(equalityFieldIds)) {
+ throw new ImpalaRuntimeException(String.format("Equality delete files with " +
+ "different equality field ID lists aren't supported. %s vs %s", equalityIds_,
+ equalityFieldIds));
+ }
+ }
+
private void filterConjuncts() {
if (residualExpressions_.isEmpty()) {
conjuncts_.removeAll(impalaIcebergPredicateMapping_.values());
@@ -442,8 +577,11 @@ public class IcebergScanPlanner {
for (FileDescriptor fd : dataFilesWithDeletes_) {
updateDataFilesWithDeletesStatistics(fd);
}
- for (FileDescriptor fd : deleteFiles_) {
- updateDeleteFilesStatistics(fd);
+ for (FileDescriptor fd : positionDeleteFiles_) {
+ updatePositionDeleteFilesStatistics(fd);
+ }
+ for (FileDescriptor fd : equalityDeleteFiles_) {
+ updateEqualityDeleteFilesStatistics(fd);
}
}
@@ -456,8 +594,14 @@ public class IcebergScanPlanner {
}
}
- private void updateDeleteFilesStatistics(FileDescriptor fd) {
- deletesRecordCount_ += getRecordCount(fd);
+ private void updatePositionDeleteFilesStatistics(FileDescriptor fd) {
+ positionDeletesRecordCount_ += getRecordCount(fd);
+ }
+
+ private void updateEqualityDeleteFilesStatistics(FileDescriptor fd) {
+ equalityDeletesRecordCount_ += getRecordCount(fd);
+ equalityDeleteSequenceNumbers_.add(
+ fd.getFbFileMetadata().icebergMetadata().dataSequenceNumber());
}
private long getRecordCount(FileDescriptor fd) {
diff --git a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
index 03d573002..413e4949a 100644
--- a/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/SingleNodePlanner.java
@@ -1716,9 +1716,10 @@ public class SingleNodePlanner {
/**
* Adds a new slot ref with path 'rawPath' to its tuple descriptor. This is a no-op if
- * the tuple descriptor already has a slot ref with the given raw path.
+ * the tuple descriptor already has a slot ref with the given raw path. Returns the slot
+ * descriptor (new or already existing) for 'rawPath'.
*/
- public static void addSlotRefToDesc(Analyzer analyzer, List<String> rawPath)
+ public static SlotDescriptor addSlotRefToDesc(Analyzer analyzer, List<String> rawPath)
throws AnalysisException {
Path resolvedPath = null;
try {
@@ -1730,6 +1731,7 @@ public class SingleNodePlanner {
Preconditions.checkNotNull(resolvedPath);
SlotDescriptor desc = analyzer.registerSlotRef(resolvedPath);
desc.setIsMaterialized(true);
+ return desc;
}
/**
diff --git a/testdata/data/README b/testdata/data/README
index c29d8880b..9f4c05b16 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -770,6 +770,84 @@ statements:
This table was created with HDFS absolute paths, which were replaced with the script
specified in `iceberg_test/hadoop_catalog/ice/iceberg_v2_no_deletes`.
+iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls:
+This table has an equality delete file that contains NULL value. Created by a hacked
+Impala where IcebergCatalogOpExecutor is changed to write equality delete metadata
+instead of position delete metadata when running a DELETE FROM statement. In a second
+step the underlying delete file was replaced by another parquet file with the desired
+content.
+The content:
+1: insert into functional_parquet.iceberg_v2_delete_equality_nulls values (1, "str1"), (null, "str2"), (3, "str3");
+2: EQ-delete file for the first column with values: (null), (3)
+3: insert into functional_parquet.iceberg_v2_delete_equality_nulls values (4, "str4"), (null, "str5");
+As a result 2 values (including the row with null) will be dropped from the first data
+file, while there is going to be another data file containing a null value that has
+greater data sequence number than the delete file.
+
+iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos:
+This table is created by Flink with 2 columns as primary key. Some data and equality
+delete files were added by Flink, and then Impala was used for dropping a row by writing
+a positional delete file.
+Steps:
+1-Flink:
+ create table ice.iceberg_v2_delete_both_eq_and_pos
+ (i int, s string, d date, primary key (i, d) not enforced)
+ with ('format-version'='2', 'write.upsert.enabled'='true');
+2: Flink:
+ insert into ice.iceberg_v2_delete_both_eq_and_pos values
+ (1, 'str1', to_date('2023-12-13')),
+ (2, 'str2', to_date('2023-12-13'));
+3-Flink:
+ insert into ice.iceberg_v2_delete_both_eq_and_pos values
+ (3, 'str3', to_date('2023-12-23')),
+ (2, 'str2_updated', to_date('2023-12-13'));
+4-Impala: delete from functional_parquet.iceberg_v2_delete_both_eq_and_pos where i = 1;
+
+iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids:
+Used a hacked Impala to write Iceberg metadata where one equality delete file deletes by
+field ID [1] while another deletes by field IDs [1,2]. Note, the underlying delete files
+are position delete files written by Impala. It's not possible to add this table to the
+data load because it'd give a TableLoadingException when doing a refresh on the table.
+
+iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned:
+Flink is used for creating this test table. The statements executed are the follow:
+1: Create the table with the partition column part of the primary key. This is enforced
+by Flink:
+ create table ice.iceberg_v2_delete_equality_partitioned
+ (i int, s string, d date, primary key (d, s) not enforced)
+ partitioned by (d)
+ w ith ('format-version'='2', 'write.upsert.enabled'='true');
+2: Populate one partition.
+ insert into ice.iceberg_v2_delete_equality_partitioned partition (d='2023-12-24') values
+ (1, 'str1'), (2, 'str2'), (3, 'str3');
+3: Populate another partition
+ insert into ice.iceberg_v2_delete_equality_partitioned partition (d='2023-12-25') values
+ (1, 'str1'), (2, 'str2');
+4: Update one row in the first partiton and add a new row.
+ insert into ice.iceberg_v2_delete_equality_partitioned partition (d='2023-12-24') values
+ (333333, 'str3'), (4, 'str4');
+5: Update one row in the second partition.
+ insert into ice.iceberg_v2_delete_equality_partitioned partition (d='2023-12-25') values
+ (222, 'str2');
+
+iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution:
+Flink is used to create and populate this simple table to have an equlity delete file.
+Impala is used for doing some partition evolution on this table.
+1-Flink:
+ create table ice.iceberg_v2_delete_equality_partition_evolution
+ (i int, s string, d date, primary key (d, s) not enforced)
+ partitioned by (d)
+ with ('format-version'='2', 'write.upsert.enabled'='true');
+2-Flink:
+ insert into ice.iceberg_v2_delete_equality_partition_evolution
+ partition (d='2023-12-24') values (1, 'str1'), (2, 'str2');
+3-Flink:
+ insert into ice.iceberg_v2_delete_equality_partition_evolution
+ partition (d='2023-12-24') values (111, 'str1');
+4-Impala:
+ alter table functional_parquet.iceberg_v2_delete_equality_partition_evolution
+ set partition spec (d, i);
+
iceberg_test/iceberg_migrated_alter_test
Generated and migrated by Hive
CREATE TABLE iceberg_migrated_alter_test (int_col int, string_col string, double_col double) stored as parquet;
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet
new file mode 100644
index 000000000..16aa098ba
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet
new file mode 100644
index 000000000..8301c05f7
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-38a471ff-46f4-4350-85cc-2e7ba946b34c-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet
new file mode 100644
index 000000000..9ea797232
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet
new file mode 100644
index 000000000..65606e48b
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/00000-0-72709aba-fb15-4bd6-9758-5f39eb9bdcb7-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/delete-074a9e19e61b766e-652a169e00000001_800513971_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/delete-074a9e19e61b766e-652a169e00000001_800513971_data.0.parq
new file mode 100644
index 000000000..cb77503a8
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/data/delete-074a9e19e61b766e-652a169e00000001_800513971_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m0.avro
new file mode 100644
index 000000000..990dcb775
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m1.avro
new file mode 100644
index 000000000..c120341a8
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m0.avro
new file mode 100644
index 000000000..9edba7c42
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m1.avro
new file mode 100644
index 000000000..64947e4ad
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/3d36bf90-2625-4625-b09b-d4359b979df9-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/bb4b8c07-84e1-421a-bb6c-594f297d118e-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/bb4b8c07-84e1-421a-bb6c-594f297d118e-m0.avro
new file mode 100644
index 000000000..b7e011b79
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/bb4b8c07-84e1-421a-bb6c-594f297d118e-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro
new file mode 100644
index 000000000..1194b8086
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-8985205515767142888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-8985205515767142888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro
new file mode 100644
index 000000000..f79e81579
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-8985205515767142888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-911559291487642581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-911559291487642581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro
new file mode 100644
index 000000000..19ca9b6b0
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-911559291487642581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json
new file mode 100644
index 000000000..d041d5f87
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json
@@ -0,0 +1,51 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "352654fb-efac-4192-9944-a3beae0068cf",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1702459661521,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 1, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : true,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v2.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v2.metadata.json
new file mode 100644
index 000000000..3a3c32a47
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v2.metadata.json
@@ -0,0 +1,87 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "352654fb-efac-4192-9944-a3beae0068cf",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1702459703855,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 1, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : true,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 3802179086205335895,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 3802179086205335895,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3802179086205335895,
+ "timestamp-ms" : 1702459703855,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "94ebe46e1c7ca3559289f8bff73d9a4e",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1595",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "2",
+ "total-files-size" : "1595",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702459703855,
+ "snapshot-id" : 3802179086205335895
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702459661521,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v3.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v3.metadata.json
new file mode 100644
index 000000000..6573222d1
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v3.metadata.json
@@ -0,0 +1,119 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "352654fb-efac-4192-9944-a3beae0068cf",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos",
+ "last-sequence-number" : 2,
+ "last-updated-ms" : 1702459737285,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 1, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : true,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 8985205515767142888,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 8985205515767142888,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3802179086205335895,
+ "timestamp-ms" : 1702459703855,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "94ebe46e1c7ca3559289f8bff73d9a4e",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1595",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "2",
+ "total-files-size" : "1595",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 8985205515767142888,
+ "parent-snapshot-id" : 3802179086205335895,
+ "timestamp-ms" : 1702459737285,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "7ab03b339acf084740dccfbcf5f26440",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1542",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "4",
+ "total-files-size" : "3137",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "4"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-8985205515767142888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702459703855,
+ "snapshot-id" : 3802179086205335895
+ }, {
+ "timestamp-ms" : 1702459737285,
+ "snapshot-id" : 8985205515767142888
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702459661521,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702459703855,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v2.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v4.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v4.metadata.json
new file mode 100644
index 000000000..57a74b0c4
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v4.metadata.json
@@ -0,0 +1,146 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "352654fb-efac-4192-9944-a3beae0068cf",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos",
+ "last-sequence-number" : 3,
+ "last-updated-ms" : 1702460042344,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 1, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : true,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 911559291487642581,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 911559291487642581,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3802179086205335895,
+ "timestamp-ms" : 1702459703855,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "94ebe46e1c7ca3559289f8bff73d9a4e",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1595",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "2",
+ "total-files-size" : "1595",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-3802179086205335895-1-3d36bf90-2625-4625-b09b-d4359b979df9.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 8985205515767142888,
+ "parent-snapshot-id" : 3802179086205335895,
+ "timestamp-ms" : 1702459737285,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "7ab03b339acf084740dccfbcf5f26440",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1542",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "4",
+ "total-files-size" : "3137",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "4"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-8985205515767142888-1-0cf1a310-d39c-4c6a-bfef-c3fe33cd0c25.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 3,
+ "snapshot-id" : 911559291487642581,
+ "parent-snapshot-id" : 8985205515767142888,
+ "timestamp-ms" : 1702460042344,
+ "summary" : {
+ "operation" : "overwrite",
+ "added-position-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-files-size" : "1606",
+ "added-position-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "4",
+ "total-files-size" : "4743",
+ "total-data-files" : "2",
+ "total-delete-files" : "3",
+ "total-position-deletes" : "1",
+ "total-equality-deletes" : "4"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/snap-911559291487642581-1-bb4b8c07-84e1-421a-bb6c-594f297d118e.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702459703855,
+ "snapshot-id" : 3802179086205335895
+ }, {
+ "timestamp-ms" : 1702459737285,
+ "snapshot-id" : 8985205515767142888
+ }, {
+ "timestamp-ms" : 1702460042344,
+ "snapshot-id" : 911559291487642581
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702459661521,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702459703855,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v2.metadata.json"
+ }, {
+ "timestamp-ms" : 1702459737285,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/v3.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/version-hint.text b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/version-hint.text
new file mode 100644
index 000000000..bf0d87ab1
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos/metadata/version-hint.text
@@ -0,0 +1 @@
+4
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/af4e128ee3256830-d9bd9e2f00000000_1372039299_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/af4e128ee3256830-d9bd9e2f00000000_1372039299_data.0.parq
new file mode 100644
index 000000000..6d38d4b5c
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/af4e128ee3256830-d9bd9e2f00000000_1372039299_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-41417e7df44b347b-e035009600000001_138281890_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-41417e7df44b347b-e035009600000001_138281890_data.0.parq
new file mode 100644
index 000000000..3b3aa77e6
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-41417e7df44b347b-e035009600000001_138281890_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-61438487836ebfcc-95c9ce7a00000000_909175610_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-61438487836ebfcc-95c9ce7a00000000_909175610_data.0.parq
new file mode 100644
index 000000000..bb2f119e8
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/data/delete-61438487836ebfcc-95c9ce7a00000000_909175610_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro
new file mode 100644
index 000000000..bc8a653fb
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/2d3fafd7-bce6-483f-be82-e0ccce9203fc-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro
new file mode 100644
index 000000000..707c6980c
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/57a963d3-0e4e-4540-8080-a57afd51ba99-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro
new file mode 100644
index 000000000..d6abd01f4
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/8bd425d8-25fb-4603-8cc7-aeb5ad2a3917-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro
new file mode 100644
index 000000000..164b3094c
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-6117850509763739078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-6117850509763739078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro
new file mode 100644
index 000000000..f50e3d07f
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-6117850509763739078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro
new file mode 100644
index 000000000..1783d72c9
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json
new file mode 100644
index 000000000..181d54c02
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json
@@ -0,0 +1,53 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "3d258856-43d4-4715-9ed9-7824750c4652",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1702391192239,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_different_equality_ids"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v2.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v2.metadata.json
new file mode 100644
index 000000000..593010938
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v2.metadata.json
@@ -0,0 +1,83 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "3d258856-43d4-4715-9ed9-7824750c4652",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1702391239615,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_different_equality_ids"
+ },
+ "current-snapshot-id" : 397031335297740726,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 397031335297740726,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 397031335297740726,
+ "timestamp-ms" : 1702391239615,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "616",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "616",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702391239615,
+ "snapshot-id" : 397031335297740726
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702391192239,
+ "metadata-file" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v3.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v3.metadata.json
new file mode 100644
index 000000000..88f38fdf5
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v3.metadata.json
@@ -0,0 +1,110 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "3d258856-43d4-4715-9ed9-7824750c4652",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids",
+ "last-sequence-number" : 2,
+ "last-updated-ms" : 1702391268756,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_different_equality_ids"
+ },
+ "current-snapshot-id" : 8494861454990126958,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 8494861454990126958,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 397031335297740726,
+ "timestamp-ms" : 1702391239615,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "616",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "616",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 8494861454990126958,
+ "parent-snapshot-id" : 397031335297740726,
+ "timestamp-ms" : 1702391268756,
+ "summary" : {
+ "operation" : "overwrite",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-files-size" : "1630",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "2246",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "1"
+ },
+ "manifest-list" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702391239615,
+ "snapshot-id" : 397031335297740726
+ }, {
+ "timestamp-ms" : 1702391268756,
+ "snapshot-id" : 8494861454990126958
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702391192239,
+ "metadata-file" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702391239615,
+ "metadata-file" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v2.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v4.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v4.metadata.json
new file mode 100644
index 000000000..6535f496f
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v4.metadata.json
@@ -0,0 +1,137 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "3d258856-43d4-4715-9ed9-7824750c4652",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids",
+ "last-sequence-number" : 3,
+ "last-updated-ms" : 1702391589205,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_different_equality_ids"
+ },
+ "current-snapshot-id" : 6117850509763739078,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 6117850509763739078,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 397031335297740726,
+ "timestamp-ms" : 1702391239615,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "616",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "616",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-397031335297740726-1-2d3fafd7-bce6-483f-be82-e0ccce9203fc.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 8494861454990126958,
+ "parent-snapshot-id" : 397031335297740726,
+ "timestamp-ms" : 1702391268756,
+ "summary" : {
+ "operation" : "overwrite",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-files-size" : "1630",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "2246",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "1"
+ },
+ "manifest-list" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-8494861454990126958-1-8bd425d8-25fb-4603-8cc7-aeb5ad2a3917.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 3,
+ "snapshot-id" : 6117850509763739078,
+ "parent-snapshot-id" : 8494861454990126958,
+ "timestamp-ms" : 1702391589205,
+ "summary" : {
+ "operation" : "overwrite",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-files-size" : "1630",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "3876",
+ "total-data-files" : "1",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/snap-6117850509763739078-1-57a963d3-0e4e-4540-8080-a57afd51ba99.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702391239615,
+ "snapshot-id" : 397031335297740726
+ }, {
+ "timestamp-ms" : 1702391268756,
+ "snapshot-id" : 8494861454990126958
+ }, {
+ "timestamp-ms" : 1702391589205,
+ "snapshot-id" : 6117850509763739078
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702391192239,
+ "metadata-file" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702391239615,
+ "metadata-file" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v2.metadata.json"
+ }, {
+ "timestamp-ms" : 1702391268756,
+ "metadata-file" : "hdfs://localhost:20500/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/v3.metadata.json"
+ } ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/version-hint.text b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/version-hint.text
new file mode 100644
index 000000000..bf0d87ab1
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_different_equality_ids/metadata/version-hint.text
@@ -0,0 +1 @@
+4
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/a94b351bfa56dbd8-ddb31c6400000000_1397530881_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/a94b351bfa56dbd8-ddb31c6400000000_1397530881_data.0.parq
new file mode 100644
index 000000000..c9919940f
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/a94b351bfa56dbd8-ddb31c6400000000_1397530881_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/delete-494fbbd4b792bdd2-aabb8e2000000000_1239775374_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/delete-494fbbd4b792bdd2-aabb8e2000000000_1239775374_data.0.parq
new file mode 100644
index 000000000..bb160ad38
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/delete-494fbbd4b792bdd2-aabb8e2000000000_1239775374_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/e6484a6fb0a2b4e6-d242a84000000000_1980761347_data.0.parq b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/e6484a6fb0a2b4e6-d242a84000000000_1980761347_data.0.parq
new file mode 100644
index 000000000..0b9ab1000
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/data/e6484a6fb0a2b4e6-d242a84000000000_1980761347_data.0.parq differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/25ce5480-23b6-4c70-a724-63931f8d84c6-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/25ce5480-23b6-4c70-a724-63931f8d84c6-m0.avro
new file mode 100644
index 000000000..f4fb1e4e9
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/25ce5480-23b6-4c70-a724-63931f8d84c6-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/87d3b6df-f00d-40a4-aafa-5d7f20e3299b-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/87d3b6df-f00d-40a4-aafa-5d7f20e3299b-m0.avro
new file mode 100644
index 000000000..ba4c05e34
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/87d3b6df-f00d-40a4-aafa-5d7f20e3299b-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/c8b17188-94bf-4496-9069-3eda900cd71d-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/c8b17188-94bf-4496-9069-3eda900cd71d-m0.avro
new file mode 100644
index 000000000..147ea9893
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/c8b17188-94bf-4496-9069-3eda900cd71d-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4301391241829251636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4301391241829251636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro
new file mode 100644
index 000000000..25196ae82
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4301391241829251636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro
new file mode 100644
index 000000000..bb2244295
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-9091814429631192676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-9091814429631192676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro
new file mode 100644
index 000000000..b8c31453d
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-9091814429631192676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json
new file mode 100644
index 000000000..9121ccf1e
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json
@@ -0,0 +1,53 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "56e56cde-1aac-49d0-810a-15d2998778b2",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1702383834781,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_equality_nulls"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v2.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v2.metadata.json
new file mode 100644
index 000000000..4a909fded
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v2.metadata.json
@@ -0,0 +1,83 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "56e56cde-1aac-49d0-810a-15d2998778b2",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1702383846889,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_equality_nulls"
+ },
+ "current-snapshot-id" : 4346796256488077976,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 4346796256488077976,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 4346796256488077976,
+ "timestamp-ms" : 1702383846889,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "611",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "611",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702383846889,
+ "snapshot-id" : 4346796256488077976
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702383834781,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v3.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v3.metadata.json
new file mode 100644
index 000000000..740fb8621
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v3.metadata.json
@@ -0,0 +1,110 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "56e56cde-1aac-49d0-810a-15d2998778b2",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls",
+ "last-sequence-number" : 2,
+ "last-updated-ms" : 1702383865598,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_equality_nulls"
+ },
+ "current-snapshot-id" : 9091814429631192676,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 9091814429631192676,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 4346796256488077976,
+ "timestamp-ms" : 1702383846889,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "611",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "611",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 9091814429631192676,
+ "parent-snapshot-id" : 4346796256488077976,
+ "timestamp-ms" : 1702383865598,
+ "summary" : {
+ "operation" : "overwrite",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-files-size" : "1589",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "2200",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "1"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-9091814429631192676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702383846889,
+ "snapshot-id" : 4346796256488077976
+ }, {
+ "timestamp-ms" : 1702383865598,
+ "snapshot-id" : 9091814429631192676
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702383834781,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702383846889,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v2.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v4.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v4.metadata.json
new file mode 100644
index 000000000..e8a6bd020
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v4.metadata.json
@@ -0,0 +1,136 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "56e56cde-1aac-49d0-810a-15d2998778b2",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls",
+ "last-sequence-number" : 3,
+ "last-updated-ms" : 1702384074863,
+ "last-column-id" : 2,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : false,
+ "type" : "string"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ ]
+ } ],
+ "last-partition-id" : 999,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "engine.hive.enabled" : "true",
+ "write.merge.mode" : "merge-on-read",
+ "write.format.default" : "parquet",
+ "write.delete.mode" : "merge-on-read",
+ "iceberg.catalog_location" : "/test-warehouse/iceberg_test/hadoop_catalog",
+ "OBJCAPABILITIES" : "EXTREAD,EXTWRITE",
+ "write.update.mode" : "merge-on-read",
+ "storage_handler" : "org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ "iceberg.catalog" : "hadoop.catalog",
+ "iceberg.table_identifier" : "ice.iceberg_v2_delete_equality_nulls"
+ },
+ "current-snapshot-id" : 4301391241829251636,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 4301391241829251636,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 4346796256488077976,
+ "timestamp-ms" : 1702383846889,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "611",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "611",
+ "total-data-files" : "1",
+ "total-delete-files" : "0",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "0"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4346796256488077976-1-87d3b6df-f00d-40a4-aafa-5d7f20e3299b.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 9091814429631192676,
+ "parent-snapshot-id" : 4346796256488077976,
+ "timestamp-ms" : 1702383865598,
+ "summary" : {
+ "operation" : "overwrite",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-files-size" : "1589",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "2200",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "1"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-9091814429631192676-1-c8b17188-94bf-4496-9069-3eda900cd71d.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 3,
+ "snapshot-id" : 4301391241829251636,
+ "parent-snapshot-id" : 9091814429631192676,
+ "timestamp-ms" : 1702384074863,
+ "summary" : {
+ "operation" : "append",
+ "added-data-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "598",
+ "changed-partition-count" : "1",
+ "total-records" : "5",
+ "total-files-size" : "2798",
+ "total-data-files" : "2",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "1"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/snap-4301391241829251636-1-25ce5480-23b6-4c70-a724-63931f8d84c6.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702383846889,
+ "snapshot-id" : 4346796256488077976
+ }, {
+ "timestamp-ms" : 1702383865598,
+ "snapshot-id" : 9091814429631192676
+ }, {
+ "timestamp-ms" : 1702384074863,
+ "snapshot-id" : 4301391241829251636
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702383834781,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702383846889,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v2.metadata.json"
+ }, {
+ "timestamp-ms" : 1702383865598,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/v3.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/version-hint.text b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/version-hint.text
new file mode 100644
index 000000000..bf0d87ab1
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls/metadata/version-hint.text
@@ -0,0 +1 @@
+4
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00001.parquet
new file mode 100644
index 000000000..baaddd908
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00002.parquet
new file mode 100644
index 000000000..5299d3b68
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-bdb0c103-bfaf-49b4-935a-16a951e55b1c-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00001.parquet
new file mode 100644
index 000000000..dad997eca
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00002.parquet
new file mode 100644
index 000000000..1c4b8ed09
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/data/d=2023-12-24/00000-0-fe90d7bb-47e7-4221-8e06-f63922d3fa23-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro
new file mode 100644
index 000000000..656203f82
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m1.avro
new file mode 100644
index 000000000..a9254ddbc
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/807d7d2a-0557-4bbe-9f07-9467de72598a-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m0.avro
new file mode 100644
index 000000000..dbea31096
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m1.avro
new file mode 100644
index 000000000..344623a8f
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/c9c459ff-9747-4dab-b65f-cace0f31e669-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro
new file mode 100644
index 000000000..a2cf14749
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-586812101365618837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-586812101365618837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro
new file mode 100644
index 000000000..d7a56a713
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-586812101365618837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json
new file mode 100644
index 000000000..04f725210
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json
@@ -0,0 +1,56 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "9258fcb2-d821-426d-8a88-304ff9e7364c",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1702498563674,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v2.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v2.metadata.json
new file mode 100644
index 000000000..06c3dbc00
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v2.metadata.json
@@ -0,0 +1,92 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "9258fcb2-d821-426d-8a88-304ff9e7364c",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1702498592419,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 3409033829588781878,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 3409033829588781878,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3409033829588781878,
+ "timestamp-ms" : 1702498592419,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "1bf562ae989b81282dd5d3a188bf905c",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "2",
+ "total-files-size" : "1600",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702498592419,
+ "snapshot-id" : 3409033829588781878
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702498563674,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v3.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v3.metadata.json
new file mode 100644
index 000000000..7a939faf5
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v3.metadata.json
@@ -0,0 +1,124 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "9258fcb2-d821-426d-8a88-304ff9e7364c",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution",
+ "last-sequence-number" : 2,
+ "last-updated-ms" : 1702498647202,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 586812101365618837,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 586812101365618837,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3409033829588781878,
+ "timestamp-ms" : 1702498592419,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "1bf562ae989b81282dd5d3a188bf905c",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "2",
+ "total-files-size" : "1600",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 586812101365618837,
+ "parent-snapshot-id" : 3409033829588781878,
+ "timestamp-ms" : 1702498647202,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "fbb4ef531e002f8fb3a2052db255adf5",
+ "flink.job-id" : "dd939cbb702c58c53221a09712f171c3",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "1519",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "3119",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "3"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-586812101365618837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702498592419,
+ "snapshot-id" : 3409033829588781878
+ }, {
+ "timestamp-ms" : 1702498647202,
+ "snapshot-id" : 586812101365618837
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702498563674,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702498592419,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v2.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v4.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v4.metadata.json
new file mode 100644
index 000000000..84d2a57d5
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v4.metadata.json
@@ -0,0 +1,142 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "9258fcb2-d821-426d-8a88-304ff9e7364c",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution",
+ "last-sequence-number" : 2,
+ "last-updated-ms" : 1702498757874,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 1,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ }, {
+ "spec-id" : 1,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ }, {
+ "name" : "i",
+ "transform" : "identity",
+ "source-id" : 1,
+ "field-id" : 1001
+ } ]
+ } ],
+ "last-partition-id" : 1001,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "impala.events.catalogServiceId" : "82734f9389d94c17:adba9a6a60c75a10",
+ "impala.events.catalogVersion" : "1777",
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 586812101365618837,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 586812101365618837,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3409033829588781878,
+ "timestamp-ms" : 1702498592419,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "1bf562ae989b81282dd5d3a188bf905c",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "2",
+ "total-files-size" : "1600",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "2"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-3409033829588781878-1-c9c459ff-9747-4dab-b65f-cace0f31e669.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 586812101365618837,
+ "parent-snapshot-id" : 3409033829588781878,
+ "timestamp-ms" : 1702498647202,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "fbb4ef531e002f8fb3a2052db255adf5",
+ "flink.job-id" : "dd939cbb702c58c53221a09712f171c3",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "1519",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "3119",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "3"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/snap-586812101365618837-1-807d7d2a-0557-4bbe-9f07-9467de72598a.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702498592419,
+ "snapshot-id" : 3409033829588781878
+ }, {
+ "timestamp-ms" : 1702498647202,
+ "snapshot-id" : 586812101365618837
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702498563674,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702498592419,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v2.metadata.json"
+ }, {
+ "timestamp-ms" : 1702498647202,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/v3.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/version-hint.text b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/version-hint.text
new file mode 100644
index 000000000..bf0d87ab1
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution/metadata/version-hint.text
@@ -0,0 +1 @@
+4
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00001.parquet
new file mode 100644
index 000000000..aaf832f01
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00002.parquet
new file mode 100644
index 000000000..25ac8933d
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-0c3800d3-c638-4591-b40c-158dcd5ebe25-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00001.parquet
new file mode 100644
index 000000000..aeab70978
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00002.parquet
new file mode 100644
index 000000000..48d8944fe
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-24/00000-0-c6e2da66-fe58-44b5-81bd-575da62c7a91-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00001.parquet
new file mode 100644
index 000000000..4afaeaa96
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00002.parquet
new file mode 100644
index 000000000..0b7f0178a
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-759289e0-d713-41a1-bdaf-f9feab643720-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00001.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00001.parquet
new file mode 100644
index 000000000..f43b11df7
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00001.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00002.parquet b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00002.parquet
new file mode 100644
index 000000000..02915e0dd
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/data/d=2023-12-25/00000-0-e1567ae8-d9c3-4071-b671-8bbbe79d36d1-00002.parquet differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m0.avro
new file mode 100644
index 000000000..3cdeb156e
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro
new file mode 100644
index 000000000..1d04453db
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/09ca0acc-c19e-4073-80f7-b476a6e568c7-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m0.avro
new file mode 100644
index 000000000..0a24b9f64
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m1.avro
new file mode 100644
index 000000000..bca5ed16a
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/57f8cd32-619c-46fc-a683-1dee7473c990-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m0.avro
new file mode 100644
index 000000000..12a7a2a57
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m1.avro
new file mode 100644
index 000000000..cd07c56f1
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/db5041df-259d-48b9-ade1-1bf382a93d5a-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m0.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m0.avro
new file mode 100644
index 000000000..67635d7ee
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m0.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m1.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m1.avro
new file mode 100644
index 000000000..6d3f44a66
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/e3dac70e-a8aa-4d15-9d35-20c4f25f36d5-m1.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro
new file mode 100644
index 000000000..e1d3fe81a
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3979814664665937114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3979814664665937114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro
new file mode 100644
index 000000000..21ab294e3
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3979814664665937114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-4821964189199835313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-4821964189199835313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro
new file mode 100644
index 000000000..814e54c7a
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-4821964189199835313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro
new file mode 100644
index 000000000..73d9fbeec
Binary files /dev/null and b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro differ
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json
new file mode 100644
index 000000000..c97139f9f
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json
@@ -0,0 +1,56 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "72376038-8325-453e-a757-20e8f8feed5e",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned",
+ "last-sequence-number" : 0,
+ "last-updated-ms" : 1702476288199,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : -1,
+ "refs" : { },
+ "snapshots" : [ ],
+ "statistics" : [ ],
+ "snapshot-log" : [ ],
+ "metadata-log" : [ ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json
new file mode 100644
index 000000000..e9ff1bd7a
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json
@@ -0,0 +1,92 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "72376038-8325-453e-a757-20e8f8feed5e",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned",
+ "last-sequence-number" : 1,
+ "last-updated-ms" : 1702476300728,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 3217166167862484560,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 3217166167862484560,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476300728,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "751aac623aff5b6900c68eac5be5ffe6",
+ "flink.job-id" : "d790a22a176d9ddd8657533e41628505",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "1608",
+ "added-equality-deletes" : "3",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "1608",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "3"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702476300728,
+ "snapshot-id" : 3217166167862484560
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702476288199,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v3.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v3.metadata.json
new file mode 100644
index 000000000..c0c6e5402
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v3.metadata.json
@@ -0,0 +1,124 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "72376038-8325-453e-a757-20e8f8feed5e",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned",
+ "last-sequence-number" : 2,
+ "last-updated-ms" : 1702476310943,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 7564375228633944060,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 7564375228633944060,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476300728,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "751aac623aff5b6900c68eac5be5ffe6",
+ "flink.job-id" : "d790a22a176d9ddd8657533e41628505",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "1608",
+ "added-equality-deletes" : "3",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "1608",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "3"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 7564375228633944060,
+ "parent-snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476310943,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "2df40cec755b051393a30a9ff5247e4d",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "5",
+ "total-files-size" : "3208",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "5"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702476300728,
+ "snapshot-id" : 3217166167862484560
+ }, {
+ "timestamp-ms" : 1702476310943,
+ "snapshot-id" : 7564375228633944060
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702476288199,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702476300728,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v4.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v4.metadata.json
new file mode 100644
index 000000000..ab25ac32d
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v4.metadata.json
@@ -0,0 +1,156 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "72376038-8325-453e-a757-20e8f8feed5e",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned",
+ "last-sequence-number" : 3,
+ "last-updated-ms" : 1702476332813,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 3979814664665937114,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 3979814664665937114,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476300728,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "751aac623aff5b6900c68eac5be5ffe6",
+ "flink.job-id" : "d790a22a176d9ddd8657533e41628505",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "1608",
+ "added-equality-deletes" : "3",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "1608",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "3"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 7564375228633944060,
+ "parent-snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476310943,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "2df40cec755b051393a30a9ff5247e4d",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "5",
+ "total-files-size" : "3208",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "5"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 3,
+ "snapshot-id" : 3979814664665937114,
+ "parent-snapshot-id" : 7564375228633944060,
+ "timestamp-ms" : 1702476332813,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "0cbe9ac985f1c730a7953b16e5cec31f",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "7",
+ "total-files-size" : "4808",
+ "total-data-files" : "3",
+ "total-delete-files" : "3",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "7"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3979814664665937114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702476300728,
+ "snapshot-id" : 3217166167862484560
+ }, {
+ "timestamp-ms" : 1702476310943,
+ "snapshot-id" : 7564375228633944060
+ }, {
+ "timestamp-ms" : 1702476332813,
+ "snapshot-id" : 3979814664665937114
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702476288199,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702476300728,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json"
+ }, {
+ "timestamp-ms" : 1702476310943,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v3.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v5.metadata.json b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v5.metadata.json
new file mode 100644
index 000000000..1db450cd6
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v5.metadata.json
@@ -0,0 +1,188 @@
+{
+ "format-version" : 2,
+ "table-uuid" : "72376038-8325-453e-a757-20e8f8feed5e",
+ "location" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned",
+ "last-sequence-number" : 4,
+ "last-updated-ms" : 1702476380992,
+ "last-column-id" : 3,
+ "current-schema-id" : 0,
+ "schemas" : [ {
+ "type" : "struct",
+ "schema-id" : 0,
+ "identifier-field-ids" : [ 2, 3 ],
+ "fields" : [ {
+ "id" : 1,
+ "name" : "i",
+ "required" : false,
+ "type" : "int"
+ }, {
+ "id" : 2,
+ "name" : "s",
+ "required" : true,
+ "type" : "string"
+ }, {
+ "id" : 3,
+ "name" : "d",
+ "required" : true,
+ "type" : "date"
+ } ]
+ } ],
+ "default-spec-id" : 0,
+ "partition-specs" : [ {
+ "spec-id" : 0,
+ "fields" : [ {
+ "name" : "d",
+ "transform" : "identity",
+ "source-id" : 3,
+ "field-id" : 1000
+ } ]
+ } ],
+ "last-partition-id" : 1000,
+ "default-sort-order-id" : 0,
+ "sort-orders" : [ {
+ "order-id" : 0,
+ "fields" : [ ]
+ } ],
+ "properties" : {
+ "write.parquet.compression-codec" : "zstd",
+ "write.upsert.enabled" : "true"
+ },
+ "current-snapshot-id" : 4821964189199835313,
+ "refs" : {
+ "main" : {
+ "snapshot-id" : 4821964189199835313,
+ "type" : "branch"
+ }
+ },
+ "snapshots" : [ {
+ "sequence-number" : 1,
+ "snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476300728,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "751aac623aff5b6900c68eac5be5ffe6",
+ "flink.job-id" : "d790a22a176d9ddd8657533e41628505",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "3",
+ "added-files-size" : "1608",
+ "added-equality-deletes" : "3",
+ "changed-partition-count" : "1",
+ "total-records" : "3",
+ "total-files-size" : "1608",
+ "total-data-files" : "1",
+ "total-delete-files" : "1",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "3"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3217166167862484560-1-db5041df-259d-48b9-ade1-1bf382a93d5a.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 2,
+ "snapshot-id" : 7564375228633944060,
+ "parent-snapshot-id" : 3217166167862484560,
+ "timestamp-ms" : 1702476310943,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "2df40cec755b051393a30a9ff5247e4d",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "5",
+ "total-files-size" : "3208",
+ "total-data-files" : "2",
+ "total-delete-files" : "2",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "5"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-7564375228633944060-1-57f8cd32-619c-46fc-a683-1dee7473c990.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 3,
+ "snapshot-id" : 3979814664665937114,
+ "parent-snapshot-id" : 7564375228633944060,
+ "timestamp-ms" : 1702476332813,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "a5df36bf3be49d13acf7c9a290f491bf",
+ "flink.job-id" : "0cbe9ac985f1c730a7953b16e5cec31f",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "2",
+ "added-files-size" : "1600",
+ "added-equality-deletes" : "2",
+ "changed-partition-count" : "1",
+ "total-records" : "7",
+ "total-files-size" : "4808",
+ "total-data-files" : "3",
+ "total-delete-files" : "3",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "7"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-3979814664665937114-1-09ca0acc-c19e-4073-80f7-b476a6e568c7.avro",
+ "schema-id" : 0
+ }, {
+ "sequence-number" : 4,
+ "snapshot-id" : 4821964189199835313,
+ "parent-snapshot-id" : 3979814664665937114,
+ "timestamp-ms" : 1702476380992,
+ "summary" : {
+ "operation" : "overwrite",
+ "flink.operator-id" : "fbb4ef531e002f8fb3a2052db255adf5",
+ "flink.job-id" : "0909766ab9d7637470b746e6f14cbde6",
+ "flink.max-committed-checkpoint-id" : "9223372036854775807",
+ "added-data-files" : "1",
+ "added-equality-delete-files" : "1",
+ "added-delete-files" : "1",
+ "added-records" : "1",
+ "added-files-size" : "1519",
+ "added-equality-deletes" : "1",
+ "changed-partition-count" : "1",
+ "total-records" : "8",
+ "total-files-size" : "6327",
+ "total-data-files" : "4",
+ "total-delete-files" : "4",
+ "total-position-deletes" : "0",
+ "total-equality-deletes" : "8"
+ },
+ "manifest-list" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/snap-4821964189199835313-1-e3dac70e-a8aa-4d15-9d35-20c4f25f36d5.avro",
+ "schema-id" : 0
+ } ],
+ "statistics" : [ ],
+ "snapshot-log" : [ {
+ "timestamp-ms" : 1702476300728,
+ "snapshot-id" : 3217166167862484560
+ }, {
+ "timestamp-ms" : 1702476310943,
+ "snapshot-id" : 7564375228633944060
+ }, {
+ "timestamp-ms" : 1702476332813,
+ "snapshot-id" : 3979814664665937114
+ }, {
+ "timestamp-ms" : 1702476380992,
+ "snapshot-id" : 4821964189199835313
+ } ],
+ "metadata-log" : [ {
+ "timestamp-ms" : 1702476288199,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v1.metadata.json"
+ }, {
+ "timestamp-ms" : 1702476300728,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v2.metadata.json"
+ }, {
+ "timestamp-ms" : 1702476310943,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v3.metadata.json"
+ }, {
+ "timestamp-ms" : 1702476332813,
+ "metadata-file" : "/test-warehouse/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/v4.metadata.json"
+ } ]
+}
diff --git a/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/version-hint.text b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/version-hint.text
new file mode 100644
index 000000000..7813681f5
--- /dev/null
+++ b/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned/metadata/version-hint.text
@@ -0,0 +1 @@
+5
\ No newline at end of file
diff --git a/testdata/datasets/functional/functional_schema_template.sql b/testdata/datasets/functional/functional_schema_template.sql
index 5de22e79d..62662fa56 100644
--- a/testdata/datasets/functional/functional_schema_template.sql
+++ b/testdata/datasets/functional/functional_schema_template.sql
@@ -3573,6 +3573,66 @@ hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/ice/i
---- DATASET
functional
---- BASE_TABLE_NAME
+iceberg_v2_delete_equality_nulls
+---- CREATE
+CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+STORED AS ICEBERG
+TBLPROPERTIES('write.format.default'='parquet', 'iceberg.catalog'='hadoop.catalog',
+ 'iceberg.catalog_location'='/test-warehouse/iceberg_test/hadoop_catalog',
+ 'iceberg.table_identifier'='ice.iceberg_v2_delete_equality_nulls',
+ 'format-version'='2', 'write.update.mode'='merge-on-read');
+---- DEPENDENT_LOAD
+`hadoop fs -mkdir -p /test-warehouse/iceberg_test/hadoop_catalog/ice && \
+hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_nulls /test-warehouse/iceberg_test/hadoop_catalog/ice
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+iceberg_v2_delete_both_eq_and_pos
+---- CREATE
+CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+STORED AS ICEBERG
+TBLPROPERTIES('write.format.default'='parquet', 'iceberg.catalog'='hadoop.catalog',
+ 'iceberg.catalog_location'='/test-warehouse/iceberg_test/hadoop_catalog',
+ 'iceberg.table_identifier'='ice.iceberg_v2_delete_both_eq_and_pos',
+ 'format-version'='2', 'write.update.mode'='merge-on-read');
+---- DEPENDENT_LOAD
+`hadoop fs -mkdir -p /test-warehouse/iceberg_test/hadoop_catalog/ice && \
+hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_both_eq_and_pos /test-warehouse/iceberg_test/hadoop_catalog/ice
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+iceberg_v2_delete_equality_partitioned
+---- CREATE
+CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+STORED AS ICEBERG
+TBLPROPERTIES('write.format.default'='parquet', 'iceberg.catalog'='hadoop.catalog',
+ 'iceberg.catalog_location'='/test-warehouse/iceberg_test/hadoop_catalog',
+ 'iceberg.table_identifier'='ice.iceberg_v2_delete_equality_partitioned',
+ 'format-version'='2', 'write.update.mode'='merge-on-read');
+---- DEPENDENT_LOAD
+`hadoop fs -mkdir -p /test-warehouse/iceberg_test/hadoop_catalog/ice && \
+hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partitioned /test-warehouse/iceberg_test/hadoop_catalog/ice
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
+iceberg_v2_delete_equality_partition_evolution
+---- CREATE
+CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
+STORED AS ICEBERG
+TBLPROPERTIES('write.format.default'='parquet', 'iceberg.catalog'='hadoop.catalog',
+ 'iceberg.catalog_location'='/test-warehouse/iceberg_test/hadoop_catalog',
+ 'iceberg.table_identifier'='ice.iceberg_v2_delete_equality_partition_evolution',
+ 'format-version'='2', 'write.update.mode'='merge-on-read');
+---- DEPENDENT_LOAD
+`hadoop fs -mkdir -p /test-warehouse/iceberg_test/hadoop_catalog/ice && \
+hadoop fs -put -f ${IMPALA_HOME}/testdata/data/iceberg_test/hadoop_catalog/ice/iceberg_v2_delete_equality_partition_evolution /test-warehouse/iceberg_test/hadoop_catalog/ice
+====
+---- DATASET
+functional
+---- BASE_TABLE_NAME
iceberg_multiple_storage_locations
---- CREATE
CREATE EXTERNAL TABLE IF NOT EXISTS {db_name}{db_suffix}.{table_name}
diff --git a/testdata/datasets/functional/schema_constraints.csv b/testdata/datasets/functional/schema_constraints.csv
index b067580c3..5e478992f 100644
--- a/testdata/datasets/functional/schema_constraints.csv
+++ b/testdata/datasets/functional/schema_constraints.csv
@@ -84,6 +84,10 @@ table_name:iceberg_timestamptz_part, constraint:restrict_to, table_format:parque
table_name:iceberg_uppercase_col, constraint:restrict_to, table_format:parquet/none/none
table_name:iceberg_v2_delete_positional, constraint:restrict_to, table_format:parquet/none/none
table_name:iceberg_v2_delete_equality, constraint:restrict_to, table_format:parquet/none/none
+table_name:iceberg_v2_delete_equality_nulls, constraint:restrict_to, table_format:parquet/none/none
+table_name:iceberg_v2_delete_both_eq_and_pos, constraint:restrict_to, table_format:parquet/none/none
+table_name:iceberg_v2_delete_equality_partitioned, constraint:restrict_to, table_format:parquet/none/none
+table_name:iceberg_v2_delete_equality_partition_evolution, constraint:restrict_to, table_format:parquet/none/none
table_name:iceberg_v2_no_deletes, constraint:restrict_to, table_format:parquet/none/none
table_name:iceberg_v2_no_deletes_orc, constraint:restrict_to, table_format:parquet/none/none
table_name:iceberg_v2_positional_update_all_rows, constraint:restrict_to, table_format:parquet/none/none
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test
index f5a3c9d87..29578ffe2 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/iceberg-v2-tables.test
@@ -1148,3 +1148,231 @@ PLAN-ROOT SINK
HDFS partitions=1/1 files=2 size=1.22KB
row-size=20B cardinality=6
====
+select * from functional_parquet.iceberg_v2_delete_equality;
+---- PLAN
+PLAN-ROOT SINK
+|
+04:UNION
+| pass-through-operands: all
+| row-size=28B cardinality=3
+|
+|--02:HASH JOIN [LEFT ANTI JOIN]
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id
+| | other join predicates: functional_parquet.iceberg_v2_delete_equality.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality-equality-delete-01.iceberg__data__sequence__number
+| | row-size=28B cardinality=1
+| |
+| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
+| | HDFS partitions=1/1 files=1 size=457B
+| | row-size=16B cardinality=1
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+| HDFS partitions=1/1 files=1 size=709B
+| row-size=28B cardinality=1
+|
+03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+ HDFS partitions=1/1 files=2 size=1.41KB
+ row-size=28B cardinality=2
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+06:EXCHANGE [UNPARTITIONED]
+|
+04:UNION
+| pass-through-operands: all
+| row-size=28B cardinality=3
+|
+|--02:HASH JOIN [LEFT ANTI JOIN, BROADCAST]
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id
+| | other join predicates: functional_parquet.iceberg_v2_delete_equality.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality-equality-delete-01.iceberg__data__sequence__number
+| | row-size=28B cardinality=1
+| |
+| |--05:EXCHANGE [BROADCAST]
+| | |
+| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
+| | HDFS partitions=1/1 files=1 size=457B
+| | row-size=16B cardinality=1
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+| HDFS partitions=1/1 files=1 size=709B
+| row-size=28B cardinality=1
+|
+03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+ HDFS partitions=1/1 files=2 size=1.41KB
+ row-size=28B cardinality=2
+====
+select * from functional_parquet.iceberg_v2_delete_equality for system_version as of 5763349507283783091;
+---- PLAN
+PLAN-ROOT SINK
+|
+04:UNION
+| pass-through-operands: all
+| row-size=28B cardinality=3
+|
+|--02:HASH JOIN [LEFT ANTI JOIN]
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id
+| | other join predicates: functional_parquet.iceberg_v2_delete_equality.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality-equality-delete-01.iceberg__data__sequence__number
+| | row-size=28B cardinality=1
+| |
+| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
+| | HDFS partitions=1/1 files=1 size=457B
+| | row-size=16B cardinality=1
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+| HDFS partitions=1/1 files=1 size=709B
+| row-size=28B cardinality=1
+|
+03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+ HDFS partitions=1/1 files=2 size=1.41KB
+ row-size=28B cardinality=2
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+06:EXCHANGE [UNPARTITIONED]
+|
+04:UNION
+| pass-through-operands: all
+| row-size=28B cardinality=3
+|
+|--02:HASH JOIN [LEFT ANTI JOIN, BROADCAST]
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality.id IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality-equality-delete-01.id
+| | other join predicates: functional_parquet.iceberg_v2_delete_equality.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality-equality-delete-01.iceberg__data__sequence__number
+| | row-size=28B cardinality=1
+| |
+| |--05:EXCHANGE [BROADCAST]
+| | |
+| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality-equality-delete-01]
+| | HDFS partitions=1/1 files=1 size=457B
+| | row-size=16B cardinality=1
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+| HDFS partitions=1/1 files=1 size=709B
+| row-size=28B cardinality=1
+|
+03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality]
+ HDFS partitions=1/1 files=2 size=1.41KB
+ row-size=28B cardinality=2
+====
+# Table has both equality and positional delete files. Equality fields are (i int, d date).
+select * from functional_parquet.iceberg_v2_delete_both_eq_and_pos;
+---- PLAN
+PLAN-ROOT SINK
+|
+06:UNION
+| pass-through-operands: all
+| row-size=48B cardinality=3
+|
+|--04:HASH JOIN [LEFT ANTI JOIN]
+| | hash predicates: functional_parquet.iceberg_v2_delete_both_eq_and_pos.d IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03.d, functional_parquet.iceberg_v2_delete_both_eq_and_pos.i IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03.i
+| | other join predicates: functional_parquet.iceberg_v2_delete_both_eq_and_pos.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03.iceberg__data__sequence__number
+| | row-size=48B cardinality=1
+| |
+| |--03:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-EQUALITY-DELETE-03 functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03]
+| | HDFS partitions=1/1 files=1 size=657B
+| | row-size=16B cardinality=2
+| |
+| 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN]
+| | row-size=48B cardinality=1
+| |
+| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_both_eq_and_pos-position-delete]
+| | HDFS partitions=1/1 files=1 size=1.57KB
+| | row-size=187B cardinality=1
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
+| HDFS partitions=1/1 files=1 size=898B
+| row-size=48B cardinality=2
+|
+05:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
+ HDFS partitions=1/1 files=1 size=885B
+ row-size=48B cardinality=2
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+09:EXCHANGE [UNPARTITIONED]
+|
+06:UNION
+| pass-through-operands: all
+| row-size=48B cardinality=3
+|
+|--04:HASH JOIN [LEFT ANTI JOIN, BROADCAST]
+| | hash predicates: functional_parquet.iceberg_v2_delete_both_eq_and_pos.d IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03.d, functional_parquet.iceberg_v2_delete_both_eq_and_pos.i IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03.i
+| | other join predicates: functional_parquet.iceberg_v2_delete_both_eq_and_pos.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03.iceberg__data__sequence__number
+| | row-size=48B cardinality=1
+| |
+| |--08:EXCHANGE [BROADCAST]
+| | |
+| | 03:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-EQUALITY-DELETE-03 functional_parquet.iceberg_v2_delete_both_eq_and_pos-equality-delete-03]
+| | HDFS partitions=1/1 files=1 size=657B
+| | row-size=16B cardinality=2
+| |
+| 02:DELETE EVENTS ICEBERG DELETE [ICEBERG DELETE JOIN, DIRECTED]
+| | row-size=48B cardinality=1
+| |
+| |--07:EXCHANGE [DIRECTED]
+| | |
+| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos-POSITION-DELETE-01 functional_parquet.iceberg_v2_delete_both_eq_and_pos-position-delete]
+| | HDFS partitions=1/1 files=1 size=1.57KB
+| | row-size=187B cardinality=1
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
+| HDFS partitions=1/1 files=1 size=898B
+| row-size=48B cardinality=2
+|
+05:SCAN HDFS [functional_parquet.iceberg_v2_delete_both_eq_and_pos]
+ HDFS partitions=1/1 files=1 size=885B
+ row-size=48B cardinality=2
+====
+# Partitioned table with equality deletes.
+select * from functional_parquet.iceberg_v2_delete_equality_partitioned
+---- PLAN
+PLAN-ROOT SINK
+|
+04:UNION
+| pass-through-operands: all
+| row-size=28B cardinality=8
+|
+|--02:HASH JOIN [LEFT ANTI JOIN]
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality_partitioned.d IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.d, functional_parquet.iceberg_v2_delete_equality_partitioned.s IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.s
+| | other join predicates: functional_parquet.iceberg_v2_delete_equality_partitioned.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.iceberg__data__sequence__number
+| | row-size=28B cardinality=5
+| |
+| |--01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01]
+| | HDFS partitions=1/1 files=2 size=1.33KB
+| | row-size=24B cardinality=3
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
+| HDFS partitions=1/1 files=2 size=1.76KB
+| row-size=28B cardinality=5
+|
+03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
+ HDFS partitions=1/1 files=2 size=1.71KB
+ row-size=28B cardinality=3
+---- DISTRIBUTEDPLAN
+PLAN-ROOT SINK
+|
+07:EXCHANGE [UNPARTITIONED]
+|
+04:UNION
+| pass-through-operands: all
+| row-size=28B cardinality=8
+|
+|--02:HASH JOIN [LEFT ANTI JOIN, PARTITIONED]
+| | hash predicates: functional_parquet.iceberg_v2_delete_equality_partitioned.d IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.d, functional_parquet.iceberg_v2_delete_equality_partitioned.s IS NOT DISTINCT FROM functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.s
+| | other join predicates: functional_parquet.iceberg_v2_delete_equality_partitioned.iceberg__data__sequence__number < functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.iceberg__data__sequence__number
+| | row-size=28B cardinality=5
+| |
+| |--06:EXCHANGE [HASH(functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.d,functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01.s)]
+| | |
+| | 01:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned-EQUALITY-DELETE-01 functional_parquet.iceberg_v2_delete_equality_partitioned-equality-delete-01]
+| | HDFS partitions=1/1 files=2 size=1.33KB
+| | row-size=24B cardinality=3
+| |
+| 05:EXCHANGE [HASH(functional_parquet.iceberg_v2_delete_equality_partitioned.d,functional_parquet.iceberg_v2_delete_equality_partitioned.s)]
+| |
+| 00:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
+| HDFS partitions=1/1 files=2 size=1.76KB
+| row-size=28B cardinality=5
+|
+03:SCAN HDFS [functional_parquet.iceberg_v2_delete_equality_partitioned]
+ HDFS partitions=1/1 files=2 size=1.71KB
+ row-size=28B cardinality=3
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-negative.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-negative.test
index e0946c828..ba26d193a 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-negative.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-negative.test
@@ -716,27 +716,6 @@ select * from functional_parquet.iceberg_alltypes_part for system_time as of '20
IllegalArgumentException: Cannot find a snapshot older than 2000-01-01 01:02:03
====
---- QUERY
-# Querying a table with equality deletes is not allowed.
-# We don't use time-travel, so we plan the query from cached metadata.
-select * from functional_parquet.iceberg_v2_delete_equality;
----- CATCH
-ImpalaRuntimeException: Iceberg table functional_parquet.iceberg_v2_delete_equality has EQUALITY delete file which is currently not supported by Impala
-====
----- QUERY
-# Querying a table with equality deletes is not allowed.
-# Use time-travel based on snapshot id.
-select * from functional_parquet.iceberg_v2_delete_equality for system_version as of 5763349507283783091;
----- CATCH
-ImpalaRuntimeException: Iceberg table functional_parquet.iceberg_v2_delete_equality has EQUALITY delete file which is currently not supported by Impala
-====
----- QUERY
-# Querying a table with equality deletes is not allowed.
-# Use time-travel based on timestamp.
-select * from functional_parquet.iceberg_v2_delete_equality for system_time as of now();
----- CATCH
-ImpalaRuntimeException: Iceberg table functional_parquet.iceberg_v2_delete_equality has EQUALITY delete file which is currently not supported by Impala
-====
----- QUERY
create table ice_delete (i int, s string)
stored by iceberg
tblproperties ('format-version'='1');
@@ -863,3 +842,14 @@ update update_orc set i = 3;
---- CATCH
AnalysisException: Impala can only write delete files in PARQUET, but the given table uses a different file format: $DATABASE.update_orc
====
+---- QUERY
+select * from functional_parquet.iceberg_v2_delete_equality_partition_evolution;
+---- CATCH
+ImpalaRuntimeException: Equality delete files are not supported for tables with partition evolution
+====
+---- QUERY
+# Cannot drop Flink primary key column.
+alter table functional_parquet.iceberg_v2_delete_equality_partitioned drop column s;
+---- CATCH
+Failed to ALTER table 'iceberg_v2_delete_equality_partitioned': Cannot delete identifier field 2: s: required string.
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-equality-deletes.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-equality-deletes.test
new file mode 100644
index 000000000..4d16e2de5
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v2-read-equality-deletes.test
@@ -0,0 +1,73 @@
+====
+---- QUERY
+# We don't use time-travel, so we plan the query from cached metadata.
+select * from functional_parquet.iceberg_v2_delete_equality;
+---- RESULTS
+1,'test_1_base'
+2,'test_2_updated'
+---- TYPES
+BIGINT,STRING
+====
+---- QUERY
+# Use time-travel based on snapshot id.
+select * from functional_parquet.iceberg_v2_delete_equality for system_version as of 5763349507283783091;
+---- RESULTS
+1,'test_1_base'
+2,'test_2_updated'
+---- TYPES
+BIGINT,STRING
+====
+---- QUERY
+# Use time-travel based on timestamp.
+select * from functional_parquet.iceberg_v2_delete_equality for system_time as of now();
+---- RESULTS
+1,'test_1_base'
+2,'test_2_updated'
+---- TYPES
+BIGINT,STRING
+====
+---- QUERY
+# Equality delete file with data sequance number 2 contains (NULL), (3). Drops the
+# matching values with lower data sequence number, while keeps the one with higher seq
+# nums.
+select *, ICEBERG__DATA__SEQUENCE__NUMBER from functional_parquet.iceberg_v2_delete_equality_nulls;
+---- RESULTS
+1,'str1',1
+4,'str4',3
+NULL,'str5',3
+---- TYPES
+INT,STRING,BIGINT
+====
+---- QUERY
+# Check that the NULL value was present in the first snapshot of the table.
+select *, ICEBERG__DATA__SEQUENCE__NUMBER from functional_parquet.iceberg_v2_delete_equality_nulls
+for system_version as of 4346796256488077976;
+---- RESULTS
+1,'str1',1
+NULL,'str2',1
+3,'str3',1
+---- TYPES
+INT,STRING,BIGINT
+====
+---- QUERY
+# Query a table that has both equality and positional delete files.
+select * from functional_parquet.iceberg_v2_delete_both_eq_and_pos;
+---- RESULTS
+2,'str2_updated',2023-12-13
+3,'str3',2023-12-23
+---- TYPES
+INT,STRING,DATE
+====
+---- QUERY
+# Query a partitioned table that has eq-delete files in both partitions.
+select * from functional_parquet.iceberg_v2_delete_equality_partitioned order by d, s;
+---- RESULTS
+1,'str1',2023-12-24
+2,'str2',2023-12-24
+333333,'str3',2023-12-24
+4,'str4',2023-12-24
+1,'str1',2023-12-25
+222,'str2',2023-12-25
+---- TYPES
+INT,STRING,DATE
+====
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index acdee3946..557334544 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -1234,6 +1234,51 @@ class TestIcebergV2Table(IcebergTestSuite):
def test_read_position_deletes(self, vector):
self.run_test_case('QueryTest/iceberg-v2-read-position-deletes', vector)
+ @SkipIfDockerizedCluster.internal_hostname
+ @SkipIf.hardcoded_uris
+ def test_read_equality_deletes(self, vector):
+ self.run_test_case('QueryTest/iceberg-v2-read-equality-deletes', vector)
+
+ @SkipIfDockerizedCluster.internal_hostname
+ @SkipIf.hardcoded_uris
+ def test_multiple_equality_ids(self, unique_database):
+ """This test loads an Iceberg table that has 2 equality delete files with different
+ equality ID lists. A query on such a table fails due to lack of support."""
+ SRC_DIR = os.path.join(os.environ['IMPALA_HOME'],
+ "testdata/data/iceberg_test/hadoop_catalog/ice/"
+ "iceberg_v2_delete_different_equality_ids")
+ DST_DIR = "/test-warehouse/iceberg_test/hadoop_catalog/ice/" \
+ "iceberg_v2_delete_different_equality_ids"
+ TBL_NAME = "iceberg_v2_delete_different_equality_ids"
+ FULL_TBL_NAME = unique_database + "." + TBL_NAME
+
+ try:
+ self.filesystem_client.make_dir(DST_DIR, permission=777)
+
+ self.filesystem_client.copy_from_local(os.path.join(SRC_DIR, "data"), DST_DIR)
+ self.filesystem_client.copy_from_local(os.path.join(SRC_DIR, "metadata"), DST_DIR)
+
+ self.execute_query_expect_success(self.client,
+ """create external table {0} stored as iceberg
+ tblproperties('format-version'='2', 'iceberg.catalog'='hadoop.catalog',
+ 'iceberg.catalog_location'='/test-warehouse/iceberg_test/hadoop_catalog',
+ 'iceberg.table_identifier'=
+ 'ice.iceberg_v2_delete_different_equality_ids')""".format(FULL_TBL_NAME))
+
+ err = self.execute_query_expect_failure(self.client,
+ "select * from " + FULL_TBL_NAME)
+ assert "Equality delete files with different equality field ID lists aren't " \
+ "supported." in str(err)
+ # The error message also contains the mismatching equality ID lists but the order
+ # of the lists can change so we assert them separately. The end of the original
+ # message looks like: "[1] vs [1, 2]" or "[1, 2] vs [1]"
+ assert "[1, 2]" in str(err)
+ assert "[1]" in str(err)
+
+ finally:
+ # Clean up the test directory
+ self.filesystem_client.delete_file_dir(DST_DIR, True)
+
@SkipIfDockerizedCluster.internal_hostname
@SkipIf.hardcoded_uris
def test_read_position_deletes_orc(self, vector):