You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ja...@apache.org on 2026/04/23 15:29:48 UTC
(impala) branch master updated (d8b81c51e -> c9f8b5c0d)
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from d8b81c51e IMPALA-14895: Added tests for newly supported Calcite queries
new 417ba2d3e IMPALA-14897: [Patch 1 of 2] Configure TLS v1.3 Cipher Suites on Thrift Servers
new 1a73414d7 IMPALA-14763: Prevent admissiond OOM during request decompression
new c9f8b5c0d IMPALA-14589: [Part 1] Add support for Iceberg V3 default values
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/catalog/catalogd-main.cc | 4 +-
be/src/exec/avro/hdfs-avro-scanner.cc | 7 +-
be/src/exec/file-metadata-utils.cc | 72 ++++-
be/src/exec/file-metadata-utils.h | 22 ++
be/src/exec/orc/hdfs-orc-scanner.cc | 5 +-
be/src/exec/parquet/hdfs-parquet-scanner.cc | 7 +-
be/src/runtime/descriptors.cc | 3 +
be/src/runtime/descriptors.h | 2 +
be/src/scheduling/admission-controller.cc | 141 +++++---
be/src/scheduling/admission-controller.h | 28 ++
be/src/service/impala-server.cc | 9 +-
be/src/statestore/statestore-subscriber.cc | 4 +-
be/src/statestore/statestore.cc | 7 +-
be/src/testutil/certificates-info.txt | 29 ++
be/src/testutil/localhost-ecdsa.key | 5 +
be/src/testutil/localhost-ecdsa.pem | 16 +
be/src/testutil/localhost.pem | 33 +-
be/src/testutil/wildcardCA-ecdsa.key | 5 +
be/src/testutil/wildcardCA-ecdsa.pem | 16 +
bin/start-impala-cluster.py | 43 +++
common/thrift/CatalogObjects.thrift | 3 +
common/thrift/Descriptors.thrift | 2 +
.../org/apache/impala/analysis/InsertStmt.java | 9 +
.../java/org/apache/impala/catalog/Column.java | 6 +-
.../org/apache/impala/catalog/IcebergColumn.java | 33 +-
.../apache/impala/planner/IcebergScanPlanner.java | 130 ++++++--
.../apache/impala/util/IcebergSchemaConverter.java | 7 +-
testdata/data/README | 26 ++
.../metadata/v4.metadata.json | 2 +-
...b0ad3b-49a1-4da4-a830-8e39b42a2731-0-00001.avro | Bin 0 -> 266 bytes
...9b106d-737e-4ec3-a49d-814716a2b794-0-00001.avro | Bin 0 -> 355 bytes
.../90885e58-806c-4b63-b088-39f06e1fd90c-m0.avro} | Bin 7848 -> 7821 bytes
.../da4c8917-801b-42db-913a-c9f3a006d771-m0.avro} | Bin 7784 -> 7769 bytes
...56-1-90885e58-806c-4b63-b088-39f06e1fd90c.avro} | Bin 4825 -> 4837 bytes
...16-1-da4c8917-801b-42db-913a-c9f3a006d771.avro} | Bin 4754 -> 4761 bytes
.../metadata/v1.metadata.json | 1 +
.../metadata/v2.metadata.json | 1 +
.../metadata/v3.metadata.json | 1 +
.../metadata/v4.metadata.json | 1 +
.../metadata/version-hint.text | 0
...2ce281-9c77-4045-aec4-8338f184d93e-0-00001.orc} | Bin
...1c07679-b344-4993-aaa7-1c0aa90555ed-0-00001.orc | Bin 0 -> 289 bytes
.../16d77afa-0e7f-4de8-945a-a3df191625f3-m0.avro} | Bin 7848 -> 7843 bytes
.../2e7e1961-217f-424b-a485-75827f5741a9-m0.avro} | Bin 7784 -> 7785 bytes
...56-1-2e7e1961-217f-424b-a485-75827f5741a9.avro} | Bin 4754 -> 4756 bytes
...11-1-16d77afa-0e7f-4de8-945a-a3df191625f3.avro} | Bin 4825 -> 4832 bytes
.../metadata/v1.metadata.json | 1 +
.../metadata/v2.metadata.json | 1 +
.../metadata/v3.metadata.json | 1 +
.../metadata/v4.metadata.json | 1 +
...28-7b42-4efb-a3f1-77ca256afa44-0-00001.parquet} | Bin
...769-5fc4-461d-a35b-f15a3779c122-0-00001.parquet | Bin 0 -> 930 bytes
.../45121802-1f9f-49ed-b088-fbbe0c63a810-m0.avro} | Bin 7848 -> 7987 bytes
.../4efe63ec-63be-461b-a4fc-2b1347bf40dc-m0.avro} | Bin 7786 -> 7786 bytes
...23-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro} | Bin 4825 -> 4829 bytes
...22-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro} | Bin 4754 -> 4751 bytes
.../test_complex_default/metadata/v1.metadata.json | 1 +
.../test_complex_default/metadata/v2.metadata.json | 1 +
.../test_complex_default/metadata/v3.metadata.json | 132 ++++++++
.../test_complex_default/metadata/v4.metadata.json | 168 ++++++++++
.../metadata/version-hint.txt | 0
...97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet} | Bin
...77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet} | Bin
...9a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet} | Bin
...97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet} | Bin
...77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet} | Bin
...9a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet} | Bin
...db4-fe9f-433c-ad9b-e6556ca6c828-0-00002.parquet | Bin 0 -> 603 bytes
...53a-a5bf-4cce-801c-217e50e5c478-0-00002.parquet | Bin 0 -> 603 bytes
...b4-fe9f-433c-ad9b-e6556ca6c828-0-00001.parquet} | Bin 603 -> 603 bytes
...3a-a5bf-4cce-801c-217e50e5c478-0-00001.parquet} | Bin 603 -> 603 bytes
...949-4a73-4e64-980c-9139ea31874e-0-00001.parquet | Bin 0 -> 575 bytes
...b7d-9859-4e03-855b-f79080ff9dea-0-00001.parquet | Bin 0 -> 575 bytes
.../255d8a57-3727-4f9a-85bc-77501dbd160c-m0.avro | Bin 0 -> 6984 bytes
.../3017ab79-5cf7-4b0f-82da-71da3f213a7e-m0.avro | Bin 0 -> 7249 bytes
.../4f406ba3-5a68-49c4-9052-02efa68e4e88-m0.avro | Bin 0 -> 6982 bytes
.../72aaa146-43b8-4166-afa1-03669e13753a-m0.avro | Bin 0 -> 8030 bytes
.../75c8c665-8b6e-44a6-8e5d-74d9e339833c-m0.avro | Bin 0 -> 8063 bytes
.../80cb93fa-c2c3-40ae-b434-820828b5e52a-m0.avro} | Bin 7797 -> 7802 bytes
.../cfb04a5f-58ed-458d-85ef-fb6ffb227a89-m0.avro | Bin 0 -> 7218 bytes
...454-1-255d8a57-3727-4f9a-85bc-77501dbd160c.avro | Bin 0 -> 4460 bytes
...79-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro} | Bin 4754 -> 4750 bytes
...700-1-3017ab79-5cf7-4b0f-82da-71da3f213a7e.avro | Bin 0 -> 4546 bytes
...953-1-cfb04a5f-58ed-458d-85ef-fb6ffb227a89.avro | Bin 0 -> 4603 bytes
...204-1-4f406ba3-5a68-49c4-9052-02efa68e4e88.avro | Bin 0 -> 4463 bytes
...70-1-72aaa146-43b8-4166-afa1-03669e13753a.avro} | Bin 4825 -> 4894 bytes
...62-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro} | Bin 4825 -> 4839 bytes
.../test_default_part/metadata/v1.metadata.json | 1 +
.../test_default_part/metadata/v2.metadata.json | 1 +
.../test_default_part/metadata/v3.metadata.json | 116 +++++++
.../test_default_part/metadata/v4.metadata.json | 131 ++++++++
.../test_default_part/metadata/v5.metadata.json | 167 ++++++++++
.../test_default_part/metadata/v6.metadata.json | 203 ++++++++++++
.../test_default_part/metadata/version-hint.text | 1 +
.../QueryTest/iceberg-v3-default-values.test | 280 ++++++++++++++++
.../queries/QueryTest/iceberg-v3-negative.test | 62 +---
tests/common/custom_cluster_test_suite.py | 8 +-
tests/common/file_utils.py | 4 +-
tests/common/impala_cluster.py | 4 +
tests/common/network.py | 67 +++-
tests/custom_cluster/test_admission_controller.py | 13 +
tests/custom_cluster/test_server_tls.py | 353 +++++++++++++++++++++
tests/query_test/test_iceberg.py | 10 +
103 files changed, 2244 insertions(+), 163 deletions(-)
create mode 100644 be/src/testutil/localhost-ecdsa.key
create mode 100644 be/src/testutil/localhost-ecdsa.pem
create mode 100644 be/src/testutil/wildcardCA-ecdsa.key
create mode 100644 be/src/testutil/wildcardCA-ecdsa.pem
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-3-e9b0ad3b-49a1-4da4-a830-8e39b42a2731-0-00001.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-4-809b106d-737e-4ec3-a49d-814716a2b794-0-00001.avro
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro => iceberg_v3_default_value_avro/metadata/90885e58-806c-4b63-b088-39f06e1fd90c-m0.avro} (97%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage_orc/metadata/e5f99ba8-b804-434f-aa9e-d51e86cc0180-m0.avro => iceberg_v3_default_value_avro/metadata/da4c8917-801b-42db-913a-c9f3a006d771-m0.avro} (97%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro => iceberg_v3_default_value_avro/metadata/snap-1138771512771688056-1-90885e58-806c-4b63-b088-39f06e1fd90c.avro} (91%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/metadata/snap-8059325670730066324-1-8ea2cf61-8fe7-4599-923a-d64b424cae3f.avro => iceberg_v3_default_value_avro/metadata/snap-2819462914350205316-1-da4c8917-801b-42db-913a-c9f3a006d771.avro} (92%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v4.metadata.json
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value => iceberg_v3_default_value_avro}/metadata/version-hint.text (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage_orc/data/00000-0-e69cb204-0c90-4255-8b0b-7af3aec3f75d-0-00001.orc => iceberg_v3_default_value_orc/data/00000-0-ab2ce281-9c77-4045-aec4-8338f184d93e-0-00001.orc} (100%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-1-01c07679-b344-4993-aaa7-1c0aa90555ed-0-00001.orc
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro => iceberg_v3_default_value_orc/metadata/16d77afa-0e7f-4de8-945a-a3df191625f3-m0.avro} (97%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage_orc/metadata/e5f99ba8-b804-434f-aa9e-d51e86cc0180-m0.avro => iceberg_v3_default_value_orc/metadata/2e7e1961-217f-424b-a485-75827f5741a9-m0.avro} (97%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/metadata/snap-8059325670730066324-1-8ea2cf61-8fe7-4599-923a-d64b424cae3f.avro => iceberg_v3_default_value_orc/metadata/snap-1536392834658079256-1-2e7e1961-217f-424b-a485-75827f5741a9.avro} (92%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro => iceberg_v3_default_value_orc/metadata/snap-4321940050961723411-1-16d77afa-0e7f-4de8-945a-a3df191625f3.avro} (91%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v4.metadata.json
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/data/00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet => test_complex_default/data/00000-0-70c34928-7b42-4efb-a3f1-77ca256afa44-0-00001.parquet} (100%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-1-e5d7e769-5fc4-461d-a35b-f15a3779c122-0-00001.parquet
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/ed3d9f93-7a24-464c-9e74-f3d910f45451-m0.avro => test_complex_default/metadata/45121802-1f9f-49ed-b088-fbbe0c63a810-m0.avro} (93%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage_orc/metadata/8542d294-4d10-4efc-9e9d-69d3dce88108-m1.avro => test_complex_default/metadata/4efe63ec-63be-461b-a4fc-2b1347bf40dc-m0.avro} (96%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro => test_complex_default/metadata/snap-2556729165577948823-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro} (91%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/metadata/snap-8059325670730066324-1-8ea2cf61-8fe7-4599-923a-d64b424cae3f.avro => test_complex_default/metadata/snap-5009345500136574322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro} (92%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v4.metadata.json
copy testdata/data/iceberg_test/{hadoop_catalog/ice/iceberg_v2_delete_equality => iceberg_v3/test_complex_default}/metadata/version-hint.txt (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/data/00000-0-153001a8-dc43-4e8b-ad61-b691a1754e16-0-00001.parquet => test_default_part/data/00000-0-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet} (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/data/00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet => test_default_part/data/00000-0-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet} (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/data/00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet => test_default_part/data/00000-9-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet} (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/data/00000-1-e55b64a3-1aa3-4a3c-87a1-cd3d2988c499-0-00001.parquet => test_default_part/data/00001-1-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet} (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_deletion_vectors/data/00001-1-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet => test_default_part/data/00001-1-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet} (100%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_deletion_vectors/data/00001-1-ec047627-1122-495a-9b07-87e0c47aebbb-0-00001.parquet => test_default_part/data/00001-10-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet} (100%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00002.parquet
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00002.parquet
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/data/00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet => test_default_part/data/j=5/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00001.parquet} (53%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/data/00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet => test_default_part/data/j=5/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00001.parquet} (53%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-15-160d8949-4a73-4e64-980c-9139ea31874e-0-00001.parquet
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-6-51f5db7d-9859-4e03-855b-f79080ff9dea-0-00001.parquet
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/255d8a57-3727-4f9a-85bc-77501dbd160c-m0.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/3017ab79-5cf7-4b0f-82da-71da3f213a7e-m0.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/4f406ba3-5a68-49c4-9052-02efa68e4e88-m0.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/72aaa146-43b8-4166-afa1-03669e13753a-m0.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/75c8c665-8b6e-44a6-8e5d-74d9e339833c-m0.avro
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/metadata/7411e291-ddc0-4c54-9e25-75ef7878df0d-m2.avro => test_default_part/metadata/80cb93fa-c2c3-40ae-b434-820828b5e52a-m0.avro} (96%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/cfb04a5f-58ed-458d-85ef-fb6ffb227a89-m0.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-1848864042488207454-1-255d8a57-3727-4f9a-85bc-77501dbd160c.avro
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_row_lineage/metadata/snap-8059325670730066324-1-8ea2cf61-8fe7-4599-923a-d64b424cae3f.avro => test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro} (92%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5064620223589928700-1-3017ab79-5cf7-4b0f-82da-71da3f213a7e.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5334867362062016953-1-cfb04a5f-58ed-458d-85ef-fb6ffb227a89.avro
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5370916276675601204-1-4f406ba3-5a68-49c4-9052-02efa68e4e88.avro
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro => test_default_part/metadata/snap-5788798547140819570-1-72aaa146-43b8-4166-afa1-03669e13753a.avro} (90%)
copy testdata/data/iceberg_test/iceberg_v3/{iceberg_v3_default_value/metadata/snap-5466875730470528452-1-ed3d9f93-7a24-464c-9e74-f3d910f45451.avro => test_default_part/metadata/snap-7704689856474798162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro} (91%)
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v1.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v2.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v3.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v4.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v5.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json
create mode 100644 testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/version-hint.text
create mode 100644 testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test
create mode 100644 tests/custom_cluster/test_server_tls.py
(impala) 03/03: IMPALA-14589: [Part 1] Add support for Iceberg V3 default values
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c9f8b5c0db8a2e9c35f9e59032ea0dbbcd26534f
Author: Arnab Karmakar <ar...@gmail.com>
AuthorDate: Mon Apr 13 10:51:38 2026 -0700
IMPALA-14589: [Part 1] Add support for Iceberg V3 default values
Iceberg V3 introduces two types of default values to enable non-blocking
schema evolution, this patch adds support for them in Impala.
1. **initial-default**: Applied when READING old data files that were
written before a column was added to the schema. This allows adding
columns with defaults without rewriting existing data files. The
default value is materialized at read time for missing columns.
2. **write-default**: Applied when WRITING new rows that don't specify a
value for a column. This ensures consistent defaults across all
engines writing to the table, regardless of whether they're inserting
via Spark, Flink, or Impala.
How Default Values Work:
- Default values are stored in the Iceberg table's schema metadata
(not in files)
- When scanning a file missing a column, Impala checks the schema for
initial-default
- If present, the value is materialized in the template tuple before
scanning
- For writes, InsertStmt checks for write-default on unmentioned columns
- Both default types are represented as JSON-serialized literals in the schema
This patch adds support for ONLY initial-default values.
Testing:
- Updated iceberg-v3-negative.test with schema evolution expectations
- Added negative tests for complex types with default values
- Extended iceberg-v3-default-values.test with comprehensive coverage:
* Read path: SELECT, WHERE, scanner and predicate pushdown with compound
predicates
* Tests validating partition key values override initial defaults
* Time travel: Schema evolution across snapshots
Change-Id: I9f1be994a336b30b17b17819091417d777a39be9
Reviewed-on: http://gerrit.cloudera.org:8080/24088
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/exec/avro/hdfs-avro-scanner.cc | 7 +-
be/src/exec/file-metadata-utils.cc | 72 +++++-
be/src/exec/file-metadata-utils.h | 22 ++
be/src/exec/orc/hdfs-orc-scanner.cc | 5 +-
be/src/exec/parquet/hdfs-parquet-scanner.cc | 7 +-
be/src/runtime/descriptors.cc | 3 +
be/src/runtime/descriptors.h | 2 +
common/thrift/CatalogObjects.thrift | 3 +
common/thrift/Descriptors.thrift | 2 +
.../org/apache/impala/analysis/InsertStmt.java | 9 +
.../java/org/apache/impala/catalog/Column.java | 6 +-
.../org/apache/impala/catalog/IcebergColumn.java | 33 ++-
.../apache/impala/planner/IcebergScanPlanner.java | 130 ++++++++--
.../apache/impala/util/IcebergSchemaConverter.java | 7 +-
testdata/data/README | 26 ++
.../metadata/v4.metadata.json | 2 +-
...b0ad3b-49a1-4da4-a830-8e39b42a2731-0-00001.avro | Bin 0 -> 266 bytes
...9b106d-737e-4ec3-a49d-814716a2b794-0-00001.avro | Bin 0 -> 355 bytes
.../90885e58-806c-4b63-b088-39f06e1fd90c-m0.avro | Bin 0 -> 7821 bytes
.../da4c8917-801b-42db-913a-c9f3a006d771-m0.avro | Bin 0 -> 7769 bytes
...056-1-90885e58-806c-4b63-b088-39f06e1fd90c.avro | Bin 0 -> 4837 bytes
...316-1-da4c8917-801b-42db-913a-c9f3a006d771.avro | Bin 0 -> 4761 bytes
.../metadata/v1.metadata.json | 1 +
.../metadata/v2.metadata.json | 1 +
.../metadata/v3.metadata.json | 1 +
.../metadata/v4.metadata.json | 1 +
.../metadata/version-hint.text | 1 +
...b2ce281-9c77-4045-aec4-8338f184d93e-0-00001.orc | Bin 0 -> 235 bytes
...1c07679-b344-4993-aaa7-1c0aa90555ed-0-00001.orc | Bin 0 -> 289 bytes
.../16d77afa-0e7f-4de8-945a-a3df191625f3-m0.avro | Bin 0 -> 7843 bytes
.../2e7e1961-217f-424b-a485-75827f5741a9-m0.avro | Bin 0 -> 7785 bytes
...256-1-2e7e1961-217f-424b-a485-75827f5741a9.avro | Bin 0 -> 4756 bytes
...411-1-16d77afa-0e7f-4de8-945a-a3df191625f3.avro | Bin 0 -> 4832 bytes
.../metadata/v1.metadata.json | 1 +
.../metadata/v2.metadata.json | 1 +
.../metadata/v3.metadata.json | 1 +
.../metadata/v4.metadata.json | 1 +
...928-7b42-4efb-a3f1-77ca256afa44-0-00001.parquet | Bin 0 -> 392 bytes
...769-5fc4-461d-a35b-f15a3779c122-0-00001.parquet | Bin 0 -> 930 bytes
.../45121802-1f9f-49ed-b088-fbbe0c63a810-m0.avro | Bin 0 -> 7987 bytes
.../4efe63ec-63be-461b-a4fc-2b1347bf40dc-m0.avro | Bin 0 -> 7786 bytes
...823-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro | Bin 0 -> 4829 bytes
...322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro | Bin 0 -> 4751 bytes
.../test_complex_default/metadata/v1.metadata.json | 1 +
.../test_complex_default/metadata/v2.metadata.json | 1 +
.../test_complex_default/metadata/v3.metadata.json | 132 ++++++++++
.../test_complex_default/metadata/v4.metadata.json | 168 +++++++++++++
.../test_complex_default/metadata/version-hint.txt | 1 +
...a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet | Bin 0 -> 392 bytes
...f77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet | Bin 0 -> 392 bytes
...99a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet | Bin 0 -> 392 bytes
...a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet | Bin 0 -> 392 bytes
...f77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet | Bin 0 -> 392 bytes
...99a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet | Bin 0 -> 392 bytes
...db4-fe9f-433c-ad9b-e6556ca6c828-0-00002.parquet | Bin 0 -> 603 bytes
...53a-a5bf-4cce-801c-217e50e5c478-0-00002.parquet | Bin 0 -> 603 bytes
...db4-fe9f-433c-ad9b-e6556ca6c828-0-00001.parquet | Bin 0 -> 603 bytes
...53a-a5bf-4cce-801c-217e50e5c478-0-00001.parquet | Bin 0 -> 603 bytes
...949-4a73-4e64-980c-9139ea31874e-0-00001.parquet | Bin 0 -> 575 bytes
...b7d-9859-4e03-855b-f79080ff9dea-0-00001.parquet | Bin 0 -> 575 bytes
.../255d8a57-3727-4f9a-85bc-77501dbd160c-m0.avro | Bin 0 -> 6984 bytes
.../3017ab79-5cf7-4b0f-82da-71da3f213a7e-m0.avro | Bin 0 -> 7249 bytes
.../4f406ba3-5a68-49c4-9052-02efa68e4e88-m0.avro | Bin 0 -> 6982 bytes
.../72aaa146-43b8-4166-afa1-03669e13753a-m0.avro | Bin 0 -> 8030 bytes
.../75c8c665-8b6e-44a6-8e5d-74d9e339833c-m0.avro | Bin 0 -> 8063 bytes
.../80cb93fa-c2c3-40ae-b434-820828b5e52a-m0.avro | Bin 0 -> 7802 bytes
.../cfb04a5f-58ed-458d-85ef-fb6ffb227a89-m0.avro | Bin 0 -> 7218 bytes
...454-1-255d8a57-3727-4f9a-85bc-77501dbd160c.avro | Bin 0 -> 4460 bytes
...979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro | Bin 0 -> 4750 bytes
...700-1-3017ab79-5cf7-4b0f-82da-71da3f213a7e.avro | Bin 0 -> 4546 bytes
...953-1-cfb04a5f-58ed-458d-85ef-fb6ffb227a89.avro | Bin 0 -> 4603 bytes
...204-1-4f406ba3-5a68-49c4-9052-02efa68e4e88.avro | Bin 0 -> 4463 bytes
...570-1-72aaa146-43b8-4166-afa1-03669e13753a.avro | Bin 0 -> 4894 bytes
...162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro | Bin 0 -> 4839 bytes
.../test_default_part/metadata/v1.metadata.json | 1 +
.../test_default_part/metadata/v2.metadata.json | 1 +
.../test_default_part/metadata/v3.metadata.json | 116 +++++++++
.../test_default_part/metadata/v4.metadata.json | 131 ++++++++++
.../test_default_part/metadata/v5.metadata.json | 167 ++++++++++++
.../test_default_part/metadata/v6.metadata.json | 203 +++++++++++++++
.../test_default_part/metadata/version-hint.text | 1 +
.../QueryTest/iceberg-v3-default-values.test | 280 +++++++++++++++++++++
.../queries/QueryTest/iceberg-v3-negative.test | 62 +----
tests/common/file_utils.py | 4 +-
tests/query_test/test_iceberg.py | 10 +
85 files changed, 1540 insertions(+), 84 deletions(-)
diff --git a/be/src/exec/avro/hdfs-avro-scanner.cc b/be/src/exec/avro/hdfs-avro-scanner.cc
index 9b5ec3bd6..1ab958252 100644
--- a/be/src/exec/avro/hdfs-avro-scanner.cc
+++ b/be/src/exec/avro/hdfs-avro-scanner.cc
@@ -251,7 +251,12 @@ Status HdfsAvroScanner::ResolveSchemas(const AvroSchemaElement& table_root,
avro_schema_record_field_get_index(file_record->schema, field_name);
if (file_field_idx < 0) {
- // This field doesn't exist in the file schema. Check if there is a default value.
+ // This field doesn't exist in the file schema.
+ // Skip if we don't need data from file (partition/virtual/default columns).
+ if (!file_metadata_utils_.NeedDataInFile(slot_desc)) {
+ continue;
+ }
+ // Check if there is a default value in the Avro schema.
avro_datum_t default_value =
avro_schema_record_field_default(table_record->schema, table_field_idx);
if (default_value == nullptr) {
diff --git a/be/src/exec/file-metadata-utils.cc b/be/src/exec/file-metadata-utils.cc
index 10a030792..9dff7570e 100644
--- a/be/src/exec/file-metadata-utils.cc
+++ b/be/src/exec/file-metadata-utils.cc
@@ -50,10 +50,55 @@ Tuple* FileMetadataUtils::CreateTemplateTuple(int64_t partition_id, MemPool* mem
}
if (scan_node_->hdfs_table()->IsIcebergTable()) {
AddIcebergColumns(mem_pool, &template_tuple, slot_descs_written);
+
+ PopulateIcebergDefaults(template_tuple, mem_pool);
}
return template_tuple;
}
+TextConverter FileMetadataUtils::CreateTextConverter() const {
+ return TextConverter(/* escape_char */ '\\',
+ scan_node_->hdfs_table()->null_partition_key_value(),
+ /* check_null */ true, /* strict_mode */ true);
+}
+
+void FileMetadataUtils::PopulateIcebergDefaults(Tuple* template_tuple, MemPool* pool) {
+ DCHECK(scan_node_->hdfs_table()->IsIcebergTable());
+ if (template_tuple == nullptr) return;
+
+ TextConverter text_converter = CreateTextConverter();
+
+ for (int i = 0; i < scan_node_->tuple_desc()->slots().size(); ++i) {
+ const SlotDescriptor* slot_desc = scan_node_->tuple_desc()->slots()[i];
+
+ // Skip if not a top-level column (complex types not supported for defaults)
+ if (slot_desc->col_path().size() != 1) continue;
+
+ int col_idx = slot_desc->col_path()[0];
+ DCHECK_LT(col_idx, scan_node_->hdfs_table()->col_descs().size());
+ const ColumnDescriptor& col_desc = scan_node_->hdfs_table()->col_descs()[col_idx];
+
+ if (col_desc.initial_default().empty()) continue;
+
+ // Don't overwrite partition columns
+ if (IsValuePartitionCol(slot_desc)) continue;
+
+ if (!text_converter.WriteSlot(slot_desc, template_tuple,
+ col_desc.initial_default().c_str(),
+ col_desc.initial_default().size(),
+ /* copy_string = */ true, /* need_escape = */ false,
+ pool)) {
+ ErrorMsg error_msg(TErrorCode::GENERAL,
+ Substitute("Could not parse Iceberg initial-default value for "
+ "column '$0' in file '$1'. Default string is '$2'",
+ col_desc.name(), file_desc_->filename, col_desc.initial_default()));
+ state_->LogError(error_msg);
+ continue;
+ }
+ template_tuple->SetNotNull(slot_desc->null_indicator_offset());
+ }
+}
+
void FileMetadataUtils::AddFileLevelVirtualColumns(MemPool* mem_pool,
Tuple* template_tuple) {
if (template_tuple == nullptr) return;
@@ -113,12 +158,22 @@ auto FileMetadataUtils::IcebergPartitionTransforms() const {
return ice_metadata->partition_keys();
}
+bool FileMetadataUtils::HasIcebergDefaultValue(const SlotDescriptor* slot_desc) {
+ if (!scan_node_->hdfs_table()->IsIcebergTable()) return false;
+
+ const SchemaPath& path = slot_desc->col_path();
+ if (path.size() != 1) return false;
+
+ DCHECK_LT(path.front(), scan_node_->hdfs_table()->col_descs().size());
+ const ColumnDescriptor& col_desc =
+ scan_node_->hdfs_table()->col_descs()[path.front()];
+ return !col_desc.initial_default().empty();
+}
+
void FileMetadataUtils::AddIcebergColumns(MemPool* mem_pool, Tuple** template_tuple,
std::map<const SlotId, const SlotDescriptor*>* slot_descs_written) {
using namespace org::apache::impala::fb;
- TextConverter text_converter(/* escape_char = */ '\\',
- scan_node_->hdfs_table()->null_partition_key_value(),
- /* check_null = */ true, /* strict_mode = */ true);
+ TextConverter text_converter = CreateTextConverter();
const FbFileMetadata* file_metadata = file_desc_->file_metadata;
const FbIcebergMetadata* ice_metadata = file_metadata->iceberg_metadata();
auto transforms = ice_metadata->partition_keys();
@@ -302,7 +357,18 @@ Status FileMetadataUtils::AdjustFieldIdForMigratedPartitionedTables(int *fieldID
bool FileMetadataUtils::NeedDataInFile(const SlotDescriptor* slot_desc) {
if (IsValuePartitionCol(slot_desc)) return false;
if (slot_desc->IsVirtual()) return false;
+ if (HasIcebergDefaultValue(slot_desc)) return false;
return true;
}
+bool FileMetadataUtils::ShouldSkipReadFromFile(const SlotDescriptor* slot_desc) {
+ // Skip partition columns as they're populated from partition values
+ if (IsValuePartitionCol(slot_desc)) return true;
+ // Skip virtual columns
+ if (slot_desc->IsVirtual()) return true;
+ // For default values: don't skip - we want to read data if it exists in the file
+ // The default is only used when data is missing from the file
+ return false;
+}
+
} // namespace impala
diff --git a/be/src/exec/file-metadata-utils.h b/be/src/exec/file-metadata-utils.h
index bf929e166..41c5b235d 100644
--- a/be/src/exec/file-metadata-utils.h
+++ b/be/src/exec/file-metadata-utils.h
@@ -18,6 +18,7 @@
#pragma once
#include "common/global-types.h"
+#include "exec/text-converter.h"
#include <cstdint>
#include <map>
@@ -64,6 +65,23 @@ public:
/// partition columns, virtual columns.
bool NeedDataInFile(const SlotDescriptor* slot_desc);
+ /// Returns true if the column has an Iceberg initial-default value.
+ /// This is used by scanners to determine if a missing column should use the default
+ /// value instead of being set to NULL.
+ bool HasIcebergDefaultValue(const SlotDescriptor* slot_desc);
+
+ /// Returns true if we should skip reading this column from the file.
+ /// This returns true for partition columns, but false for columns with default values
+ /// when we have both defaults and data in files - in such cases we want the column
+ /// reader to be created and read data from file.
+ bool ShouldSkipReadFromFile(const SlotDescriptor* slot_desc);
+
+ /// Populates Iceberg initial-default values into the given template tuple.
+ /// The tuple must already exist (typically copied from the partition template tuple
+ /// built in HdfsScanPlanNode::InitTemplateTuple). This applies initial-default values
+ /// to top-level columns for Iceberg tables. Complex type defaults are not supported.
+ void PopulateIcebergDefaults(Tuple* template_tuple, MemPool* pool);
+
/// Adjusts the file-level fieldID for data files residing in migrated partitioned
/// Iceberg tables. It is OK to have none of the partition columns in the data
/// file, or to have all of the partition columns, in any other case this method
@@ -88,6 +106,10 @@ private:
/// for Iceberg tables.
auto IcebergPartitionTransforms() const;
+ /// Creates a TextConverter with the standard parameters used for parsing partition
+ /// values and default values.
+ TextConverter CreateTextConverter() const;
+
HdfsScanNodeBase* const scan_node_;
// Members below are set in Open()
diff --git a/be/src/exec/orc/hdfs-orc-scanner.cc b/be/src/exec/orc/hdfs-orc-scanner.cc
index bea638901..9f3871407 100644
--- a/be/src/exec/orc/hdfs-orc-scanner.cc
+++ b/be/src/exec/orc/hdfs-orc-scanner.cc
@@ -607,7 +607,7 @@ Status HdfsOrcScanner::ResolveColumns(const TupleDescriptor& tuple_desc,
SlotDescriptor* pos_slot_desc = nullptr;
for (SlotDescriptor* slot_desc : tuple_desc.slots()) {
// Skip columns not (necessarily) stored in the data files.
- if (!file_metadata_utils_.NeedDataInFile(slot_desc)) {
+ if (file_metadata_utils_.ShouldSkipReadFromFile(slot_desc)) {
if (slot_desc->virtual_column_type() == TVirtualColumnType::FILE_POSITION) {
DCHECK(pos_slot_desc == nullptr)
<< "There should only be one position slot per tuple";
@@ -641,7 +641,8 @@ Status HdfsOrcScanner::ResolveColumns(const TupleDescriptor& tuple_desc,
}
if (acid_original_file_ && schema_resolver_->IsAcidColumn(slot_desc->col_path())) {
SetSyntheticAcidFieldForOriginalFile(slot_desc, *template_tuple);
- } else {
+ } else if (file_metadata_utils_.NeedDataInFile(slot_desc)) {
+ // Set NULL if the field is missing and we need data from the file.
(*template_tuple)->SetNull(slot_desc->null_indicator_offset());
}
missing_field_slots_.insert(slot_desc);
diff --git a/be/src/exec/parquet/hdfs-parquet-scanner.cc b/be/src/exec/parquet/hdfs-parquet-scanner.cc
index d56935821..afe621256 100644
--- a/be/src/exec/parquet/hdfs-parquet-scanner.cc
+++ b/be/src/exec/parquet/hdfs-parquet-scanner.cc
@@ -2915,7 +2915,7 @@ Status HdfsParquetScanner::CreateColumnReaders(const TupleDescriptor& tuple_desc
for (SlotDescriptor* slot_desc: tuple_desc.slots()) {
// Skip partition columns
- if (!file_metadata_utils_.NeedDataInFile(slot_desc)) {
+ if (file_metadata_utils_.ShouldSkipReadFromFile(slot_desc)) {
if (UNLIKELY(slot_desc->virtual_column_type() ==
TVirtualColumnType::FILE_POSITION)) {
DCHECK(file_pos_slot_desc == nullptr)
@@ -2939,7 +2939,10 @@ Status HdfsParquetScanner::CreateColumnReaders(const TupleDescriptor& tuple_desc
*template_tuple =
Tuple::Create(tuple_desc.byte_size(), template_tuple_pool_.get());
}
- (*template_tuple)->SetNull(slot_desc->null_indicator_offset());
+ // Set NULL if the field is missing and we need data from the file.
+ if (file_metadata_utils_.NeedDataInFile(slot_desc)) {
+ (*template_tuple)->SetNull(slot_desc->null_indicator_offset());
+ }
continue;
}
diff --git a/be/src/runtime/descriptors.cc b/be/src/runtime/descriptors.cc
index 552001cb0..477a74ae0 100644
--- a/be/src/runtime/descriptors.cc
+++ b/be/src/runtime/descriptors.cc
@@ -177,6 +177,9 @@ ColumnDescriptor::ColumnDescriptor(const TColumnDescriptor& tdesc)
// Get key and value field_id for Iceberg table column with Map type.
field_map_key_id_ = tdesc.icebergFieldMapKeyId;
field_map_value_id_ = tdesc.icebergFieldMapValueId;
+ if (tdesc.__isset.icebergInitialDefault) {
+ initial_default_ = tdesc.icebergInitialDefault;
+ }
}
}
diff --git a/be/src/runtime/descriptors.h b/be/src/runtime/descriptors.h
index ba6a36bee..bbe8b9c28 100644
--- a/be/src/runtime/descriptors.h
+++ b/be/src/runtime/descriptors.h
@@ -342,6 +342,7 @@ class ColumnDescriptor {
int field_id() const { return field_id_; }
int field_map_key_id() const { return field_map_key_id_; }
int field_map_value_id() const { return field_map_value_id_; }
+ const std::string& initial_default() const { return initial_default_; }
std::string DebugString() const;
@@ -352,6 +353,7 @@ class ColumnDescriptor {
int field_id_ = -1;
int field_map_key_id_ = -1;
int field_map_value_id_ = -1;
+ std::string initial_default_;
};
/// Base class for table descriptors.
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index b8e51f562..291f1750e 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -342,6 +342,9 @@ struct TColumn {
// will reuse the iceberg_field_id, is_key, is_nullable
// for Paimon table.
26: optional bool is_paimon_column
+ // Iceberg V3 default value fields
+ 27: optional string iceberg_initial_default
+ 28: optional string iceberg_write_default
}
diff --git a/common/thrift/Descriptors.thrift b/common/thrift/Descriptors.thrift
index a4066ac7e..eb7e8f212 100644
--- a/common/thrift/Descriptors.thrift
+++ b/common/thrift/Descriptors.thrift
@@ -74,6 +74,8 @@ struct TColumnDescriptor {
// Key and value field id for Iceberg column with Map type.
4: optional i32 icebergFieldMapKeyId
5: optional i32 icebergFieldMapValueId
+ // Iceberg V3 default value fields
+ 6: optional string icebergInitialDefault
}
// "Union" of all table types.
diff --git a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
index 0c37d8d9b..0233c7033 100644
--- a/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/InsertStmt.java
@@ -999,6 +999,15 @@ public class InsertStmt extends DmlStatementBase {
} else {
// Unmentioned non-clustering columns get NULL literals with the appropriate
// target type because Parquet cannot handle NULL_TYPE (IMPALA-617).
+ // TODO: Add write-default support for Iceberg tables
+ if (isIcebergTarget()) {
+ IcebergColumn iceCol = (IcebergColumn) tblColumn;
+ if (iceCol.getWriteDefault() != null) {
+ throw new AnalysisException("Write-default values are not yet " +
+ "supported. Column '" + iceCol.getName() + "' has a write-default " +
+ "value defined.");
+ }
+ }
NullLiteral nullExpr = NullLiteral.create(tblColumn.getType());
resultExprs_.add(nullExpr);
// In the case of INSERT INTO iceberg_tbl (col_a, col_b, ...), if the
diff --git a/fe/src/main/java/org/apache/impala/catalog/Column.java b/fe/src/main/java/org/apache/impala/catalog/Column.java
index 716f757ec..1571c824b 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Column.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Column.java
@@ -115,10 +115,14 @@ public class Column {
if (columnDesc.isIs_iceberg_column()) {
Preconditions.checkState(columnDesc.isSetIceberg_field_id());
boolean isHidden = columnDesc.isSetIs_hidden() && columnDesc.isIs_hidden();
+ String initialDefault = columnDesc.isSetIceberg_initial_default() ?
+ columnDesc.getIceberg_initial_default() : null;
+ String writeDefault = columnDesc.isSetIceberg_write_default() ?
+ columnDesc.getIceberg_write_default() : null;
col = new IcebergColumn(columnDesc.getColumnName(), type, comment, position,
columnDesc.getIceberg_field_id(), columnDesc.getIceberg_field_map_key_id(),
columnDesc.getIceberg_field_map_value_id(), columnDesc.isIs_nullable(),
- isHidden);
+ isHidden, initialDefault, writeDefault);
} else if (columnDesc.isIs_paimon_column()) {
Preconditions.checkState(columnDesc.isSetIceberg_field_id());
col = new PaimonColumn(columnDesc.getColumnName(), type, comment, position,
diff --git a/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java b/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
index 3d7f3aa98..a8b989729 100644
--- a/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
+++ b/fe/src/main/java/org/apache/impala/catalog/IcebergColumn.java
@@ -42,6 +42,9 @@ public class IcebergColumn extends Column {
private final boolean isNullable_;
// Until only Iceberg columns can be hidden, we can keep this flag here.
private final boolean isHidden_;
+ // Iceberg V3 default values stored as JSON string representation
+ private final String initialDefault_;
+ private final String writeDefault_;
public IcebergColumn(String name, Type type, String comment, int position,
int fieldId, int fieldMapKeyId, int fieldMapValueId, boolean isNullable) {
@@ -52,19 +55,28 @@ public class IcebergColumn extends Column {
public IcebergColumn(String name, Type type, String comment, int position,
int fieldId, int fieldMapKeyId, int fieldMapValueId, boolean isNullable,
boolean isHidden) {
+ this(name, type, comment, position, fieldId, fieldMapKeyId, fieldMapValueId,
+ isNullable, isHidden, null, null);
+ }
+
+ public IcebergColumn(String name, Type type, String comment, int position,
+ int fieldId, int fieldMapKeyId, int fieldMapValueId, boolean isNullable,
+ boolean isHidden, String initialDefault, String writeDefault) {
super(name.toLowerCase(), type, comment, position);
fieldId_ = fieldId;
fieldMapKeyId_ = fieldMapKeyId;
fieldMapValueId_ = fieldMapValueId;
isNullable_ = isNullable;
isHidden_ = isHidden;
+ initialDefault_ = initialDefault;
+ writeDefault_ = writeDefault;
}
public static IcebergColumn cloneWithNullability(IcebergColumn source,
boolean isNullable) {
return new IcebergColumn(source.name_, source.type_, source.comment_,
- source.position_, source.fieldId_, source.fieldMapKeyId_, source.fieldMapKeyId_,
- isNullable);
+ source.position_, source.fieldId_, source.fieldMapKeyId_, source.fieldMapValueId_,
+ isNullable, source.isHidden_, source.initialDefault_, source.writeDefault_);
}
public int getFieldId() {
@@ -76,6 +88,10 @@ public class IcebergColumn extends Column {
@Override
public boolean isHidden() { return isHidden_; }
+ public String getInitialDefault() { return initialDefault_; }
+
+ public String getWriteDefault() { return writeDefault_; }
+
@Override
public TColumn toThrift() {
TColumn tcol = super.toThrift();
@@ -84,6 +100,12 @@ public class IcebergColumn extends Column {
tcol.setIceberg_field_map_key_id(fieldMapKeyId_);
tcol.setIceberg_field_map_value_id(fieldMapValueId_);
tcol.setIs_nullable(isNullable_);
+ if (initialDefault_ != null) {
+ tcol.setIceberg_initial_default(initialDefault_);
+ }
+ if (writeDefault_ != null) {
+ tcol.setIceberg_write_default(writeDefault_);
+ }
return tcol;
}
@@ -93,6 +115,9 @@ public class IcebergColumn extends Column {
desc.setIcebergFieldId(fieldId_);
desc.setIcebergFieldMapKeyId(fieldMapKeyId_);
desc.setIcebergFieldMapValueId(fieldMapValueId_);
+ if (initialDefault_ != null) {
+ desc.setIcebergInitialDefault(initialDefault_);
+ }
return desc;
}
@@ -102,6 +127,8 @@ public class IcebergColumn extends Column {
.add("fieldId_", fieldId_)
.add("fieldMapKeyId_", fieldMapKeyId_)
.add("fieldMapValueId_", fieldMapValueId_)
- .add("isNullable_", isNullable_);
+ .add("isNullable_", isNullable_)
+ .add("initialDefault_", initialDefault_)
+ .add("writeDefault_", writeDefault_);
}
}
diff --git a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
index efb167c09..aa6aad43e 100644
--- a/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
+++ b/fe/src/main/java/org/apache/impala/planner/IcebergScanPlanner.java
@@ -17,9 +17,11 @@
package org.apache.impala.planner;
+import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import java.io.IOException;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -48,6 +50,7 @@ import org.apache.iceberg.io.CloseableIterable;
import org.apache.iceberg.metrics.InMemoryMetricsReporter;
import org.apache.iceberg.metrics.ScanMetricsResult;
import org.apache.iceberg.metrics.ScanReport;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.types.Types.NestedField;
import org.apache.impala.analysis.Analyzer;
import org.apache.impala.analysis.BinaryPredicate;
@@ -57,6 +60,7 @@ import org.apache.impala.analysis.IcebergExpressionCollector;
import org.apache.impala.analysis.IcebergPartitionSpec;
import org.apache.impala.analysis.JoinOperator;
import org.apache.impala.analysis.MultiAggregateInfo;
+import org.apache.impala.analysis.Path;
import org.apache.impala.analysis.SlotDescriptor;
import org.apache.impala.analysis.SlotId;
import org.apache.impala.analysis.SlotRef;
@@ -93,6 +97,7 @@ import org.apache.impala.thrift.TIcebergPartitionTransformType;
import org.apache.impala.thrift.TQueryOptions;
import org.apache.impala.thrift.TVirtualColumnType;
import org.apache.impala.util.Hash128;
+import org.apache.impala.util.IcebergSchemaConverter;
import org.apache.impala.util.IcebergUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -250,26 +255,10 @@ public class IcebergScanPlanner {
dataFileToDV_.isEmpty();
}
- private void validateSlotRefs() throws ImpalaException {
- if (getIceTable().getFormatVersion() < 3) return;
- for (SlotDescriptor slotDesc : tblRef_.getDesc().getSlots()) {
- if (!slotDesc.isMaterialized()) continue;
- Column column = slotDesc.getColumn();
- if (column == null) continue;
- int fieldId = ((IcebergColumn) column).getFieldId();
- NestedField field = getIceTable().getIcebergApiTable().schema().findField(fieldId);
- if (field != null && field.initialDefaultLiteral() != null) {
- throw new ImpalaRuntimeException(String.format(
- "Iceberg columns with default values not supported yet. " +
- "Table: %s Column: %s Default value: %s",
- getIceTable().getFullName(), column.getName(),
- field.initialDefaultLiteral().toString()));
- }
- }
- }
-
private PlanNode createIcebergScanPlanImpl() throws ImpalaException {
- validateSlotRefs();
+ // Check for nested fields as they are not supported for Iceberg default values
+ checkForNestedDefaultValues();
+
boolean isPartitionKeyScan = IsPartitionKeyScan();
if (noDeleteFiles()) {
Preconditions.checkState(!tblRef_.optimizeCountStarForIcebergV2());
@@ -1071,6 +1060,109 @@ public class IcebergScanPlanner {
transformType -> transformType == TIcebergPartitionTransformType.IDENTITY);
}
+ /**
+ * Rejects scans when a default value appears on a nested field or on a top-level
+ * complex column, but only for Iceberg fields that are part of this scan (including
+ * all nested field ids under a materialized struct, array, or map).
+ *
+ * The catalog only attaches defaults to top-level columns, so this walks the
+ * Iceberg schema tree. Slots that are not referenced by the query are skipped.
+ */
+ private void checkForNestedDefaultValues() throws AnalysisException {
+ Set<Integer> materializedFieldIds = computeMaterializedIcebergFieldIds();
+ for (NestedField field : getIceTable().getIcebergSchema().columns()) {
+ visitIcebergFieldForUnsupportedDefaults(field,
+ Collections.singletonList(field.name()), materializedFieldIds);
+ }
+ }
+
+ /**
+ * Resolves materialized scan slots to Iceberg field ids, then adds every descendant
+ * field id so that selecting a struct column still covers inner fields for this check.
+ */
+ private Set<Integer> computeMaterializedIcebergFieldIds() {
+ Schema schema = getIceTable().getIcebergSchema();
+ Set<Integer> seedIds = new HashSet<>();
+ for (SlotDescriptor slot : tblRef_.getDesc().getSlots()) {
+ if (!slot.isMaterialized() || !slot.isScanSlot()) continue;
+ Path path = slot.getPath();
+ if (path == null || path.getRawPath().isEmpty()) continue;
+ String qualified = Joiner.on(".").join(path.getRawPath());
+ NestedField nf = schema.findField(qualified);
+ if (nf == null) nf = schema.caseInsensitiveFindField(qualified);
+ if (nf != null) seedIds.add(nf.fieldId());
+ }
+ return expandMaterializedFieldIds(seedIds, schema);
+ }
+
+ private static Set<Integer> expandMaterializedFieldIds(
+ Set<Integer> seedIds, Schema schema) {
+ Set<Integer> expanded = new HashSet<>(seedIds);
+ ArrayDeque<Integer> queue = new ArrayDeque<>(seedIds);
+ while (!queue.isEmpty()) {
+ int id = queue.remove();
+ NestedField nf = schema.findField(id);
+ if (nf == null) continue;
+ for (NestedField child : getIcebergChildFieldsToVisit(nf.type())) {
+ if (expanded.add(child.fieldId())) queue.add(child.fieldId());
+ }
+ }
+ return expanded;
+ }
+
+ /**
+ * If this field has a default, checks that Impala would allow it for fields that
+ * appear in this scan. Then recurses into struct, list, or map children.
+ */
+ private void visitIcebergFieldForUnsupportedDefaults(NestedField field,
+ List<String> pathFromRoot, Set<Integer> materializedFieldIds)
+ throws AnalysisException {
+ boolean hasDefault =
+ field.initialDefault() != null || field.writeDefault() != null;
+ if (hasDefault && materializedFieldIds.contains(field.fieldId())) {
+ boolean isNested = pathFromRoot.size() > 1;
+ Type impalaType;
+ try {
+ impalaType = IcebergSchemaConverter.toImpalaType(field.type());
+ } catch (ImpalaRuntimeException e) {
+ throw new AnalysisException(e.getMessage(), e);
+ }
+ if (isNested) {
+ throw new AnalysisException(String.format(
+ "Default values are not supported for nested fields. " +
+ "Nested field '%s' in column '%s' cannot have default values.",
+ String.join(".", pathFromRoot.subList(1, pathFromRoot.size())),
+ pathFromRoot.get(0)));
+ }
+ if (impalaType.isComplexType()) {
+ throw new AnalysisException(String.format(
+ "Default values are not supported for complex types. " +
+ "Column '%s' has type %s which cannot have default values.",
+ field.name(), impalaType.toSql()));
+ }
+ }
+
+ for (NestedField child : getIcebergChildFieldsToVisit(field.type())) {
+ List<String> childPath = new ArrayList<>(pathFromRoot);
+ childPath.add(child.name());
+ visitIcebergFieldForUnsupportedDefaults(child, childPath, materializedFieldIds);
+ }
+ }
+
+ private static List<NestedField> getIcebergChildFieldsToVisit(
+ org.apache.iceberg.types.Type type) {
+ switch (type.typeId()) {
+ case STRUCT:
+ return type.asStructType().fields();
+ case LIST:
+ return type.asListType().fields();
+ case MAP:
+ return type.asMapType().fields();
+ default:
+ return Collections.emptyList();
+ }
+ }
+
private boolean hasPartitionTransformType(List<IcebergColumn> cols,
Map<SlotId, SlotDescriptor> idToSlotDesc,
Predicate<TIcebergPartitionTransformType> pred) {
diff --git a/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java b/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
index b60319e67..ad03ccec4 100644
--- a/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
+++ b/fe/src/main/java/org/apache/impala/util/IcebergSchemaConverter.java
@@ -145,8 +145,13 @@ public class IcebergSchemaConverter {
keyId = mapType.keyId();
valueId = mapType.valueId();
}
+ String initialDefault = column.initialDefaultLiteral() != null ?
+ column.initialDefaultLiteral().toString() : null;
+ String writeDefault = column.writeDefaultLiteral() != null ?
+ column.writeDefaultLiteral().toString() : null;
ret.add(new IcebergColumn(column.name(), colType, column.doc(), pos++,
- column.fieldId(), keyId, valueId, column.isOptional()));
+ column.fieldId(), keyId, valueId, column.isOptional(), false,
+ initialDefault, writeDefault));
}
return ret;
}
diff --git a/testdata/data/README b/testdata/data/README
index 804b22c27..5c6e91208 100644
--- a/testdata/data/README
+++ b/testdata/data/README
@@ -1219,6 +1219,32 @@ Since Spark still doesn't support default values, the metadata.json file was edi
manually to set the default value for column 'j':
{"id":2,"name":"j","required":false,"type":"int","initial-default":-1,"write-default":-1}
+testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc
+Same as iceberg_v3_default_value, but with ORC data files (tblproperty write.format.default=orc).
+
+testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro
+Same as iceberg_v3_default_value, but with Avro data files (tblproperty write.format.default=avro).
+
+testdata/data/iceberg_test/iceberg_v3/test_default_part
+Iceberg V3 table with an identity-partitioned column that uses initial-default
+in metadata (for partition-default behavior in tests). Created with
+Spark/Iceberg similarly to iceberg_v3_deletion_vectors, then modified:
+ 1. Initial table: column i (int), inserted (1), (2)
+ 2. Added column j (int), added partition by spec (j)
+ 3. Inserted (3, 5), (4, 10), (5, NULL)
+ 4. Manually set initial-default=-1 for column j in metadata JSON files
+The table exercises initial-default on partition columns where older files (from before
+the schema evolution) lack values for j, which should read as -1.
+
+testdata/data/iceberg_test/iceberg_v3/test_complex_default
+Iceberg V3 table for negative tests: struct column with an initial-default on a nested
+primitive field. Created with Spark/Iceberg, then copied from HDFS:
+ 1. Initial table: column i (int), inserted (1)
+ 2. ALTER TABLE ADD COLUMN j STRUCT<f1: INT, f2: STRING>
+ 3. Inserted (2, named_struct('f1', 100, 'f2', 'hello'))
+ 4. Manually added initial-default=-1 to nested field j.f1 (field id 3) in metadata
+Impala should reject scans that materialize any nested field with a default.
+
testdata/data/iceberg_test/iceberg_v3/iceberg_v3_row_lineage(_orc):
Iceberg V3 table created by Spark 4.0.2, see above (iceberg_v3_deletion_vectors).
sql("create table iceberg_v3_row_lineage (i int) using iceberg options('format-version'='3')").collect()
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json
index 690b20bc7..eefebe61f 100644
--- a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value/metadata/v4.metadata.json
@@ -1 +1 @@
-{"format-version":3,"table-uuid":"a0ffb3b6-f99a-4df8-8ddc-ffb49ecb56e2","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value","last-sequence-number":2,"last-updated-ms":1770928072044,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"int","i [...]
+{"format-version":3,"table-uuid":"a0ffb3b6-f99a-4df8-8ddc-ffb49ecb56e2","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value","last-sequence-number":2,"last-updated-ms":1770928072044,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"int","i [...]
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-3-e9b0ad3b-49a1-4da4-a830-8e39b42a2731-0-00001.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-3-e9b0ad3b-49a1-4da4-a830-8e39b42a2731-0-00001.avro
new file mode 100644
index 000000000..f54e11d27
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-3-e9b0ad3b-49a1-4da4-a830-8e39b42a2731-0-00001.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-4-809b106d-737e-4ec3-a49d-814716a2b794-0-00001.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-4-809b106d-737e-4ec3-a49d-814716a2b794-0-00001.avro
new file mode 100644
index 000000000..79ea2d1f1
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/data/00000-4-809b106d-737e-4ec3-a49d-814716a2b794-0-00001.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/90885e58-806c-4b63-b088-39f06e1fd90c-m0.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/90885e58-806c-4b63-b088-39f06e1fd90c-m0.avro
new file mode 100644
index 000000000..33414a3e0
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/90885e58-806c-4b63-b088-39f06e1fd90c-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/da4c8917-801b-42db-913a-c9f3a006d771-m0.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/da4c8917-801b-42db-913a-c9f3a006d771-m0.avro
new file mode 100644
index 000000000..b33a4578d
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/da4c8917-801b-42db-913a-c9f3a006d771-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/snap-1138771512771688056-1-90885e58-806c-4b63-b088-39f06e1fd90c.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/snap-1138771512771688056-1-90885e58-806c-4b63-b088-39f06e1fd90c.avro
new file mode 100644
index 000000000..7032b4720
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/snap-1138771512771688056-1-90885e58-806c-4b63-b088-39f06e1fd90c.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/snap-2819462914350205316-1-da4c8917-801b-42db-913a-c9f3a006d771.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/snap-2819462914350205316-1-da4c8917-801b-42db-913a-c9f3a006d771.avro
new file mode 100644
index 000000000..9be5dad03
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/snap-2819462914350205316-1-da4c8917-801b-42db-913a-c9f3a006d771.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v1.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v1.metadata.json
new file mode 100644
index 000000000..c4e23e9c8
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"e2c6f9b0-d329-41f0-92ce-e15520f3ddd9","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_avro","last-sequence-number":0,"last-updated-ms":1774461987698,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"ord [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v2.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v2.metadata.json
new file mode 100644
index 000000000..73d2f8acb
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"e2c6f9b0-d329-41f0-92ce-e15520f3ddd9","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_avro","last-sequence-number":1,"last-updated-ms":1774461998250,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"ord [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v3.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v3.metadata.json
new file mode 100644
index 000000000..76026c99f
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v3.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"e2c6f9b0-d329-41f0-92ce-e15520f3ddd9","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_avro","last-sequence-number":1,"last-updated-ms":1774462004151,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"in [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v4.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v4.metadata.json
new file mode 100644
index 000000000..5d9537503
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/v4.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"e2c6f9b0-d329-41f0-92ce-e15520f3ddd9","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_avro","last-sequence-number":2,"last-updated-ms":1774462011457,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"in [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/version-hint.text b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/version-hint.text
new file mode 100644
index 000000000..b8626c4cf
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_avro/metadata/version-hint.text
@@ -0,0 +1 @@
+4
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-0-ab2ce281-9c77-4045-aec4-8338f184d93e-0-00001.orc b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-0-ab2ce281-9c77-4045-aec4-8338f184d93e-0-00001.orc
new file mode 100644
index 000000000..45eec9326
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-0-ab2ce281-9c77-4045-aec4-8338f184d93e-0-00001.orc differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-1-01c07679-b344-4993-aaa7-1c0aa90555ed-0-00001.orc b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-1-01c07679-b344-4993-aaa7-1c0aa90555ed-0-00001.orc
new file mode 100644
index 000000000..8980cc9f5
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/data/00000-1-01c07679-b344-4993-aaa7-1c0aa90555ed-0-00001.orc differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/16d77afa-0e7f-4de8-945a-a3df191625f3-m0.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/16d77afa-0e7f-4de8-945a-a3df191625f3-m0.avro
new file mode 100644
index 000000000..5cb011655
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/16d77afa-0e7f-4de8-945a-a3df191625f3-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/2e7e1961-217f-424b-a485-75827f5741a9-m0.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/2e7e1961-217f-424b-a485-75827f5741a9-m0.avro
new file mode 100644
index 000000000..19af73d31
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/2e7e1961-217f-424b-a485-75827f5741a9-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/snap-1536392834658079256-1-2e7e1961-217f-424b-a485-75827f5741a9.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/snap-1536392834658079256-1-2e7e1961-217f-424b-a485-75827f5741a9.avro
new file mode 100644
index 000000000..0ca41b2d9
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/snap-1536392834658079256-1-2e7e1961-217f-424b-a485-75827f5741a9.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/snap-4321940050961723411-1-16d77afa-0e7f-4de8-945a-a3df191625f3.avro b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/snap-4321940050961723411-1-16d77afa-0e7f-4de8-945a-a3df191625f3.avro
new file mode 100644
index 000000000..1958c3612
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/snap-4321940050961723411-1-16d77afa-0e7f-4de8-945a-a3df191625f3.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v1.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v1.metadata.json
new file mode 100644
index 000000000..3c6bd3aea
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"0a619311-43be-4f3f-909a-c5317dbef1e3","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_orc","last-sequence-number":0,"last-updated-ms":1774461750540,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"orde [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v2.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v2.metadata.json
new file mode 100644
index 000000000..e8a35ea6f
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"0a619311-43be-4f3f-909a-c5317dbef1e3","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_orc","last-sequence-number":1,"last-updated-ms":1774461759001,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"orde [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v3.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v3.metadata.json
new file mode 100644
index 000000000..8d99504cc
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v3.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"0a619311-43be-4f3f-909a-c5317dbef1e3","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_orc","last-sequence-number":1,"last-updated-ms":1774461765394,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"int [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v4.metadata.json b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v4.metadata.json
new file mode 100644
index 000000000..7e0a095d4
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/iceberg_v3_default_value_orc/metadata/v4.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"0a619311-43be-4f3f-909a-c5317dbef1e3","location":"hdfs://localhost:20500/test-warehouse/iceberg_v3_default_value_orc","last-sequence-number":2,"last-updated-ms":1774461772100,"last-column-id":2,"current-schema-id":1,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]},{"type":"struct","schema-id":1,"fields":[{"id":1,"name":"i","required":false,"type":"int"},{"id":2,"name":"j","required":false,"type":"int [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-0-70c34928-7b42-4efb-a3f1-77ca256afa44-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-0-70c34928-7b42-4efb-a3f1-77ca256afa44-0-00001.parquet
new file mode 100644
index 000000000..c1a3e60cc
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-0-70c34928-7b42-4efb-a3f1-77ca256afa44-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-1-e5d7e769-5fc4-461d-a35b-f15a3779c122-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-1-e5d7e769-5fc4-461d-a35b-f15a3779c122-0-00001.parquet
new file mode 100644
index 000000000..271c74296
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/data/00000-1-e5d7e769-5fc4-461d-a35b-f15a3779c122-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/45121802-1f9f-49ed-b088-fbbe0c63a810-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/45121802-1f9f-49ed-b088-fbbe0c63a810-m0.avro
new file mode 100644
index 000000000..ef6ea093a
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/45121802-1f9f-49ed-b088-fbbe0c63a810-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/4efe63ec-63be-461b-a4fc-2b1347bf40dc-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/4efe63ec-63be-461b-a4fc-2b1347bf40dc-m0.avro
new file mode 100644
index 000000000..7b24c7cda
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/4efe63ec-63be-461b-a4fc-2b1347bf40dc-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/snap-2556729165577948823-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/snap-2556729165577948823-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro
new file mode 100644
index 000000000..b1e39bd49
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/snap-2556729165577948823-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/snap-5009345500136574322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/snap-5009345500136574322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro
new file mode 100644
index 000000000..e332e8041
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/snap-5009345500136574322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v1.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v1.metadata.json
new file mode 100644
index 000000000..59ebefe3d
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"3d465034-bbde-4a45-8631-bd21a039fd5f","location":"hdfs://localhost:20500/test-warehouse/test_complex_default","last-sequence-number":0,"last-updated-ms":1776515247241,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0, [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v2.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v2.metadata.json
new file mode 100644
index 000000000..81b3395d1
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"3d465034-bbde-4a45-8631-bd21a039fd5f","location":"hdfs://localhost:20500/test-warehouse/test_complex_default","last-sequence-number":1,"last-updated-ms":1776515249839,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0, [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v3.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v3.metadata.json
new file mode 100644
index 000000000..28f02b1eb
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v3.metadata.json
@@ -0,0 +1,132 @@
+{
+ "format-version": 3,
+ "table-uuid": "3d465034-bbde-4a45-8631-bd21a039fd5f",
+ "location": "hdfs://localhost:20500/test-warehouse/test_complex_default",
+ "last-sequence-number": 1,
+ "last-updated-ms": 1776515250121,
+ "last-column-id": 4,
+ "current-schema-id": 1,
+ "schemas": [
+ {
+ "type": "struct",
+ "schema-id": 0,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "struct",
+ "schema-id": 1,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ },
+ {
+ "id": 2,
+ "name": "j",
+ "required": false,
+ "type": {
+ "type": "struct",
+ "fields": [
+ {
+ "id": 3,
+ "name": "f1",
+ "required": false,
+ "type": "int",
+ "initial-default": -1
+ },
+ {
+ "id": 4,
+ "name": "f2",
+ "required": false,
+ "type": "string"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "default-spec-id": 0,
+ "partition-specs": [
+ {
+ "spec-id": 0,
+ "fields": []
+ }
+ ],
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [
+ {
+ "order-id": 0,
+ "fields": []
+ }
+ ],
+ "properties": {
+ "owner": "root",
+ "write.parquet.compression-codec": "zstd"
+ },
+ "current-snapshot-id": 5009345500136574000,
+ "next-row-id": 1,
+ "refs": {
+ "main": {
+ "snapshot-id": 5009345500136574000,
+ "type": "branch"
+ }
+ },
+ "snapshots": [
+ {
+ "sequence-number": 1,
+ "snapshot-id": 5009345500136574000,
+ "timestamp-ms": 1776515249839,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776515236360",
+ "added-data-files": "1",
+ "added-records": "1",
+ "added-files-size": "392",
+ "changed-partition-count": "1",
+ "total-records": "1",
+ "total-files-size": "392",
+ "total-data-files": "1",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776515236360",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/snap-5009345500136574322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro",
+ "schema-id": 0,
+ "first-row-id": 0,
+ "added-rows": 1
+ }
+ ],
+ "statistics": [],
+ "partition-statistics": [],
+ "snapshot-log": [
+ {
+ "timestamp-ms": 1776515249839,
+ "snapshot-id": 5009345500136574000
+ }
+ ],
+ "metadata-log": [
+ {
+ "timestamp-ms": 1776515247241,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/00000-18de13c6-1211-4e12-9bdd-5ca5ece53cb3.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776515249839,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/00001-bdf07f0f-edee-486c-bb0c-21cbe2ac9dc3.metadata.json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v4.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v4.metadata.json
new file mode 100644
index 000000000..c25b0a30b
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/v4.metadata.json
@@ -0,0 +1,168 @@
+{
+ "format-version": 3,
+ "table-uuid": "3d465034-bbde-4a45-8631-bd21a039fd5f",
+ "location": "hdfs://localhost:20500/test-warehouse/test_complex_default",
+ "last-sequence-number": 2,
+ "last-updated-ms": 1776515250526,
+ "last-column-id": 4,
+ "current-schema-id": 1,
+ "schemas": [
+ {
+ "type": "struct",
+ "schema-id": 0,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "struct",
+ "schema-id": 1,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ },
+ {
+ "id": 2,
+ "name": "j",
+ "required": false,
+ "type": {
+ "type": "struct",
+ "fields": [
+ {
+ "id": 3,
+ "name": "f1",
+ "required": false,
+ "type": "int",
+ "initial-default": -1
+ },
+ {
+ "id": 4,
+ "name": "f2",
+ "required": false,
+ "type": "string"
+ }
+ ]
+ }
+ }
+ ]
+ }
+ ],
+ "default-spec-id": 0,
+ "partition-specs": [
+ {
+ "spec-id": 0,
+ "fields": []
+ }
+ ],
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [
+ {
+ "order-id": 0,
+ "fields": []
+ }
+ ],
+ "properties": {
+ "owner": "root",
+ "write.parquet.compression-codec": "zstd"
+ },
+ "current-snapshot-id": 2556729165577948700,
+ "next-row-id": 2,
+ "refs": {
+ "main": {
+ "snapshot-id": 2556729165577948700,
+ "type": "branch"
+ }
+ },
+ "snapshots": [
+ {
+ "sequence-number": 1,
+ "snapshot-id": 5009345500136574000,
+ "timestamp-ms": 1776515249839,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776515236360",
+ "added-data-files": "1",
+ "added-records": "1",
+ "added-files-size": "392",
+ "changed-partition-count": "1",
+ "total-records": "1",
+ "total-files-size": "392",
+ "total-data-files": "1",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776515236360",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/snap-5009345500136574322-1-4efe63ec-63be-461b-a4fc-2b1347bf40dc.avro",
+ "schema-id": 0,
+ "first-row-id": 0,
+ "added-rows": 1
+ },
+ {
+ "sequence-number": 2,
+ "snapshot-id": 2556729165577948700,
+ "parent-snapshot-id": 5009345500136574000,
+ "timestamp-ms": 1776515250526,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776515236360",
+ "added-data-files": "1",
+ "added-records": "1",
+ "added-files-size": "930",
+ "changed-partition-count": "1",
+ "total-records": "2",
+ "total-files-size": "1322",
+ "total-data-files": "2",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776515236360",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/snap-2556729165577948823-1-45121802-1f9f-49ed-b088-fbbe0c63a810.avro",
+ "schema-id": 1,
+ "first-row-id": 1,
+ "added-rows": 1
+ }
+ ],
+ "statistics": [],
+ "partition-statistics": [],
+ "snapshot-log": [
+ {
+ "timestamp-ms": 1776515249839,
+ "snapshot-id": 5009345500136574000
+ },
+ {
+ "timestamp-ms": 1776515250526,
+ "snapshot-id": 2556729165577948700
+ }
+ ],
+ "metadata-log": [
+ {
+ "timestamp-ms": 1776515247241,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/00000-18de13c6-1211-4e12-9bdd-5ca5ece53cb3.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776515249839,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/00001-bdf07f0f-edee-486c-bb0c-21cbe2ac9dc3.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776515250121,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_complex_default/metadata/00002-b065cf6f-0c36-4e4e-b42a-8a94e5527a2e.metadata.json"
+ }
+ ]
+}
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/version-hint.txt b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/version-hint.txt
new file mode 100644
index 000000000..b8626c4cf
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_complex_default/metadata/version-hint.txt
@@ -0,0 +1 @@
+4
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-0-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-0-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet
new file mode 100644
index 000000000..0ce222477
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-0-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-0-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-0-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet
new file mode 100644
index 000000000..c1a3e60cc
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-0-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-9-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-9-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet
new file mode 100644
index 000000000..c1a3e60cc
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00000-9-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-1-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-1-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet
new file mode 100644
index 000000000..40c1d159f
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-1-04425a97-2aee-40d2-8c81-f1ea0fa96d69-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-1-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-1-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet
new file mode 100644
index 000000000..c62edd6fa
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-1-be7baf77-d645-46bf-85a6-1890ac0dbd4d-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-10-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-10-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet
new file mode 100644
index 000000000..c62edd6fa
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/00001-10-32a1199a-eac8-4050-bd63-6c487f7eb18e-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00002.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00002.parquet
new file mode 100644
index 000000000..e3aaae43f
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00002.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00002.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00002.parquet
new file mode 100644
index 000000000..e3aaae43f
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=10/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00002.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=5/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=5/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00001.parquet
new file mode 100644
index 000000000..eafe520e3
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=5/00000-13-154f9db4-fe9f-433c-ad9b-e6556ca6c828-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=5/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=5/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00001.parquet
new file mode 100644
index 000000000..eafe520e3
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=5/00000-4-ad51b53a-a5bf-4cce-801c-217e50e5c478-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-15-160d8949-4a73-4e64-980c-9139ea31874e-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-15-160d8949-4a73-4e64-980c-9139ea31874e-0-00001.parquet
new file mode 100644
index 000000000..1fef2c5bc
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-15-160d8949-4a73-4e64-980c-9139ea31874e-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-6-51f5db7d-9859-4e03-855b-f79080ff9dea-0-00001.parquet b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-6-51f5db7d-9859-4e03-855b-f79080ff9dea-0-00001.parquet
new file mode 100644
index 000000000..1fef2c5bc
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/data/j=null/00000-6-51f5db7d-9859-4e03-855b-f79080ff9dea-0-00001.parquet differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/255d8a57-3727-4f9a-85bc-77501dbd160c-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/255d8a57-3727-4f9a-85bc-77501dbd160c-m0.avro
new file mode 100644
index 000000000..d3df48e94
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/255d8a57-3727-4f9a-85bc-77501dbd160c-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/3017ab79-5cf7-4b0f-82da-71da3f213a7e-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/3017ab79-5cf7-4b0f-82da-71da3f213a7e-m0.avro
new file mode 100644
index 000000000..3a5b02faf
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/3017ab79-5cf7-4b0f-82da-71da3f213a7e-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/4f406ba3-5a68-49c4-9052-02efa68e4e88-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/4f406ba3-5a68-49c4-9052-02efa68e4e88-m0.avro
new file mode 100644
index 000000000..ca57b53d3
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/4f406ba3-5a68-49c4-9052-02efa68e4e88-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/72aaa146-43b8-4166-afa1-03669e13753a-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/72aaa146-43b8-4166-afa1-03669e13753a-m0.avro
new file mode 100644
index 000000000..48c635e79
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/72aaa146-43b8-4166-afa1-03669e13753a-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/75c8c665-8b6e-44a6-8e5d-74d9e339833c-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/75c8c665-8b6e-44a6-8e5d-74d9e339833c-m0.avro
new file mode 100644
index 000000000..6dfbb55cc
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/75c8c665-8b6e-44a6-8e5d-74d9e339833c-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/80cb93fa-c2c3-40ae-b434-820828b5e52a-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/80cb93fa-c2c3-40ae-b434-820828b5e52a-m0.avro
new file mode 100644
index 000000000..beda05b6b
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/80cb93fa-c2c3-40ae-b434-820828b5e52a-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/cfb04a5f-58ed-458d-85ef-fb6ffb227a89-m0.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/cfb04a5f-58ed-458d-85ef-fb6ffb227a89-m0.avro
new file mode 100644
index 000000000..efbb24534
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/cfb04a5f-58ed-458d-85ef-fb6ffb227a89-m0.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-1848864042488207454-1-255d8a57-3727-4f9a-85bc-77501dbd160c.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-1848864042488207454-1-255d8a57-3727-4f9a-85bc-77501dbd160c.avro
new file mode 100644
index 000000000..9d8660dc9
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-1848864042488207454-1-255d8a57-3727-4f9a-85bc-77501dbd160c.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro
new file mode 100644
index 000000000..3c743a0d8
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5064620223589928700-1-3017ab79-5cf7-4b0f-82da-71da3f213a7e.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5064620223589928700-1-3017ab79-5cf7-4b0f-82da-71da3f213a7e.avro
new file mode 100644
index 000000000..482120a83
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5064620223589928700-1-3017ab79-5cf7-4b0f-82da-71da3f213a7e.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5334867362062016953-1-cfb04a5f-58ed-458d-85ef-fb6ffb227a89.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5334867362062016953-1-cfb04a5f-58ed-458d-85ef-fb6ffb227a89.avro
new file mode 100644
index 000000000..433a233c1
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5334867362062016953-1-cfb04a5f-58ed-458d-85ef-fb6ffb227a89.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5370916276675601204-1-4f406ba3-5a68-49c4-9052-02efa68e4e88.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5370916276675601204-1-4f406ba3-5a68-49c4-9052-02efa68e4e88.avro
new file mode 100644
index 000000000..c0ea7784c
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5370916276675601204-1-4f406ba3-5a68-49c4-9052-02efa68e4e88.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5788798547140819570-1-72aaa146-43b8-4166-afa1-03669e13753a.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5788798547140819570-1-72aaa146-43b8-4166-afa1-03669e13753a.avro
new file mode 100644
index 000000000..e33cd0051
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-5788798547140819570-1-72aaa146-43b8-4166-afa1-03669e13753a.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-7704689856474798162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-7704689856474798162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro
new file mode 100644
index 000000000..9972b159b
Binary files /dev/null and b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/snap-7704689856474798162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro differ
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v1.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v1.metadata.json
new file mode 100644
index 000000000..de834d007
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v1.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"afed8a22-4e46-48dc-9d9a-e8cfa2cd640c","location":"hdfs://localhost:20500/test-warehouse/test_default_part","last-sequence-number":0,"last-updated-ms":1776501427812,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fi [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v2.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v2.metadata.json
new file mode 100644
index 000000000..ccfa352a4
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v2.metadata.json
@@ -0,0 +1 @@
+{"format-version":3,"table-uuid":"afed8a22-4e46-48dc-9d9a-e8cfa2cd640c","location":"hdfs://localhost:20500/test-warehouse/test_default_part","last-sequence-number":1,"last-updated-ms":1776501457059,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"i","required":false,"type":"int"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fi [...]
\ No newline at end of file
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v3.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v3.metadata.json
new file mode 100644
index 000000000..29eb11d23
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v3.metadata.json
@@ -0,0 +1,116 @@
+{
+ "format-version": 3,
+ "table-uuid": "afed8a22-4e46-48dc-9d9a-e8cfa2cd640c",
+ "location": "hdfs://localhost:20500/test-warehouse/test_default_part",
+ "last-sequence-number": 1,
+ "last-updated-ms": 1776501471669,
+ "last-column-id": 2,
+ "current-schema-id": 1,
+ "schemas": [
+ {
+ "type": "struct",
+ "schema-id": 0,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "struct",
+ "schema-id": 1,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ },
+ {
+ "id": 2,
+ "name": "j",
+ "required": false,
+ "type": "int",
+ "initial-default": -1
+ }
+ ]
+ }
+ ],
+ "default-spec-id": 0,
+ "partition-specs": [
+ {
+ "spec-id": 0,
+ "fields": []
+ }
+ ],
+ "last-partition-id": 999,
+ "default-sort-order-id": 0,
+ "sort-orders": [
+ {
+ "order-id": 0,
+ "fields": []
+ }
+ ],
+ "properties": {
+ "owner": "root",
+ "write.parquet.compression-codec": "zstd"
+ },
+ "current-snapshot-id": 4871712590577795000,
+ "next-row-id": 2,
+ "refs": {
+ "main": {
+ "snapshot-id": 4871712590577795000,
+ "type": "branch"
+ }
+ },
+ "snapshots": [
+ {
+ "sequence-number": 1,
+ "snapshot-id": 4871712590577795000,
+ "timestamp-ms": 1776501457059,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "2",
+ "added-records": "2",
+ "added-files-size": "784",
+ "changed-partition-count": "1",
+ "total-records": "2",
+ "total-files-size": "784",
+ "total-data-files": "2",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro",
+ "schema-id": 0,
+ "first-row-id": 0,
+ "added-rows": 2
+ }
+ ],
+ "statistics": [],
+ "partition-statistics": [],
+ "snapshot-log": [
+ {
+ "timestamp-ms": 1776501457059,
+ "snapshot-id": 4871712590577795000
+ }
+ ],
+ "metadata-log": [
+ {
+ "timestamp-ms": 1776501427812,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00000-3c5fd953-6d63-4191-aeed-fd5b27df2b31.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501457059,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00001-135ef3a5-7ff1-4d39-9f53-2629acad2aa8.metadata.json"
+ }
+ ]
+}
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v4.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v4.metadata.json
new file mode 100644
index 000000000..faa8748b4
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v4.metadata.json
@@ -0,0 +1,131 @@
+{
+ "format-version": 3,
+ "table-uuid": "afed8a22-4e46-48dc-9d9a-e8cfa2cd640c",
+ "location": "hdfs://localhost:20500/test-warehouse/test_default_part",
+ "last-sequence-number": 1,
+ "last-updated-ms": 1776501482719,
+ "last-column-id": 2,
+ "current-schema-id": 1,
+ "schemas": [
+ {
+ "type": "struct",
+ "schema-id": 0,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "struct",
+ "schema-id": 1,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ },
+ {
+ "id": 2,
+ "name": "j",
+ "required": false,
+ "type": "int",
+ "initial-default": -1
+ }
+ ]
+ }
+ ],
+ "default-spec-id": 1,
+ "partition-specs": [
+ {
+ "spec-id": 0,
+ "fields": []
+ },
+ {
+ "spec-id": 1,
+ "fields": [
+ {
+ "name": "j",
+ "transform": "identity",
+ "source-id": 2,
+ "field-id": 1000
+ }
+ ]
+ }
+ ],
+ "last-partition-id": 1000,
+ "default-sort-order-id": 0,
+ "sort-orders": [
+ {
+ "order-id": 0,
+ "fields": []
+ }
+ ],
+ "properties": {
+ "owner": "root",
+ "write.parquet.compression-codec": "zstd"
+ },
+ "current-snapshot-id": 4871712590577795000,
+ "next-row-id": 2,
+ "refs": {
+ "main": {
+ "snapshot-id": 4871712590577795000,
+ "type": "branch"
+ }
+ },
+ "snapshots": [
+ {
+ "sequence-number": 1,
+ "snapshot-id": 4871712590577795000,
+ "timestamp-ms": 1776501457059,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "2",
+ "added-records": "2",
+ "added-files-size": "784",
+ "changed-partition-count": "1",
+ "total-records": "2",
+ "total-files-size": "784",
+ "total-data-files": "2",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro",
+ "schema-id": 0,
+ "first-row-id": 0,
+ "added-rows": 2
+ }
+ ],
+ "statistics": [],
+ "partition-statistics": [],
+ "snapshot-log": [
+ {
+ "timestamp-ms": 1776501457059,
+ "snapshot-id": 4871712590577795000
+ }
+ ],
+ "metadata-log": [
+ {
+ "timestamp-ms": 1776501427812,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00000-3c5fd953-6d63-4191-aeed-fd5b27df2b31.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501457059,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00001-135ef3a5-7ff1-4d39-9f53-2629acad2aa8.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501471669,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00002-52873544-e39a-4a4a-99e2-5d49f79635b5.metadata.json"
+ }
+ ]
+}
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v5.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v5.metadata.json
new file mode 100644
index 000000000..0aa49a79e
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v5.metadata.json
@@ -0,0 +1,167 @@
+{
+ "format-version": 3,
+ "table-uuid": "afed8a22-4e46-48dc-9d9a-e8cfa2cd640c",
+ "location": "hdfs://localhost:20500/test-warehouse/test_default_part",
+ "last-sequence-number": 2,
+ "last-updated-ms": 1776501490485,
+ "last-column-id": 2,
+ "current-schema-id": 1,
+ "schemas": [
+ {
+ "type": "struct",
+ "schema-id": 0,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "struct",
+ "schema-id": 1,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ },
+ {
+ "id": 2,
+ "name": "j",
+ "required": false,
+ "type": "int",
+ "initial-default": -1
+ }
+ ]
+ }
+ ],
+ "default-spec-id": 1,
+ "partition-specs": [
+ {
+ "spec-id": 0,
+ "fields": []
+ },
+ {
+ "spec-id": 1,
+ "fields": [
+ {
+ "name": "j",
+ "transform": "identity",
+ "source-id": 2,
+ "field-id": 1000
+ }
+ ]
+ }
+ ],
+ "last-partition-id": 1000,
+ "default-sort-order-id": 0,
+ "sort-orders": [
+ {
+ "order-id": 0,
+ "fields": []
+ }
+ ],
+ "properties": {
+ "owner": "root",
+ "write.parquet.compression-codec": "zstd"
+ },
+ "current-snapshot-id": 7704689856474798000,
+ "next-row-id": 4,
+ "refs": {
+ "main": {
+ "snapshot-id": 7704689856474798000,
+ "type": "branch"
+ }
+ },
+ "snapshots": [
+ {
+ "sequence-number": 1,
+ "snapshot-id": 4871712590577795000,
+ "timestamp-ms": 1776501457059,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "2",
+ "added-records": "2",
+ "added-files-size": "784",
+ "changed-partition-count": "1",
+ "total-records": "2",
+ "total-files-size": "784",
+ "total-data-files": "2",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro",
+ "schema-id": 0,
+ "first-row-id": 0,
+ "added-rows": 2
+ },
+ {
+ "sequence-number": 2,
+ "snapshot-id": 7704689856474798000,
+ "parent-snapshot-id": 4871712590577795000,
+ "timestamp-ms": 1776501490485,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "2",
+ "added-records": "2",
+ "added-files-size": "1206",
+ "changed-partition-count": "2",
+ "total-records": "4",
+ "total-files-size": "1990",
+ "total-data-files": "4",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-7704689856474798162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro",
+ "schema-id": 1,
+ "first-row-id": 2,
+ "added-rows": 2
+ }
+ ],
+ "statistics": [],
+ "partition-statistics": [],
+ "snapshot-log": [
+ {
+ "timestamp-ms": 1776501457059,
+ "snapshot-id": 4871712590577795000
+ },
+ {
+ "timestamp-ms": 1776501490485,
+ "snapshot-id": 7704689856474798000
+ }
+ ],
+ "metadata-log": [
+ {
+ "timestamp-ms": 1776501427812,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00000-3c5fd953-6d63-4191-aeed-fd5b27df2b31.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501457059,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00001-135ef3a5-7ff1-4d39-9f53-2629acad2aa8.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501471669,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00002-52873544-e39a-4a4a-99e2-5d49f79635b5.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501482719,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00003-3750f591-e622-43d1-bf2f-d6bfe883580c.metadata.json"
+ }
+ ]
+}
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json
new file mode 100644
index 000000000..572d1fdfd
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/v6.metadata.json
@@ -0,0 +1,203 @@
+{
+ "format-version": 3,
+ "table-uuid": "afed8a22-4e46-48dc-9d9a-e8cfa2cd640c",
+ "location": "hdfs://localhost:20500/test-warehouse/test_default_part",
+ "last-sequence-number": 3,
+ "last-updated-ms": 1776501490791,
+ "last-column-id": 2,
+ "current-schema-id": 1,
+ "schemas": [
+ {
+ "type": "struct",
+ "schema-id": 0,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ }
+ ]
+ },
+ {
+ "type": "struct",
+ "schema-id": 1,
+ "fields": [
+ {
+ "id": 1,
+ "name": "i",
+ "required": false,
+ "type": "int"
+ },
+ {
+ "id": 2,
+ "name": "j",
+ "required": false,
+ "type": "int",
+ "initial-default": -1
+ }
+ ]
+ }
+ ],
+ "default-spec-id": 1,
+ "partition-specs": [
+ {
+ "spec-id": 0,
+ "fields": []
+ },
+ {
+ "spec-id": 1,
+ "fields": [
+ {
+ "name": "j",
+ "transform": "identity",
+ "source-id": 2,
+ "field-id": 1000
+ }
+ ]
+ }
+ ],
+ "last-partition-id": 1000,
+ "default-sort-order-id": 0,
+ "sort-orders": [
+ {
+ "order-id": 0,
+ "fields": []
+ }
+ ],
+ "properties": {
+ "owner": "root",
+ "write.parquet.compression-codec": "zstd"
+ },
+ "current-snapshot-id": 5788798547140820000,
+ "next-row-id": 5,
+ "refs": {
+ "main": {
+ "snapshot-id": 5788798547140820000,
+ "type": "branch"
+ }
+ },
+ "snapshots": [
+ {
+ "sequence-number": 1,
+ "snapshot-id": 4871712590577795000,
+ "timestamp-ms": 1776501457059,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "2",
+ "added-records": "2",
+ "added-files-size": "784",
+ "changed-partition-count": "1",
+ "total-records": "2",
+ "total-files-size": "784",
+ "total-data-files": "2",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-4871712590577794979-1-80cb93fa-c2c3-40ae-b434-820828b5e52a.avro",
+ "schema-id": 0,
+ "first-row-id": 0,
+ "added-rows": 2
+ },
+ {
+ "sequence-number": 2,
+ "snapshot-id": 7704689856474798000,
+ "parent-snapshot-id": 4871712590577795000,
+ "timestamp-ms": 1776501490485,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "2",
+ "added-records": "2",
+ "added-files-size": "1206",
+ "changed-partition-count": "2",
+ "total-records": "4",
+ "total-files-size": "1990",
+ "total-data-files": "4",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-7704689856474798162-1-75c8c665-8b6e-44a6-8e5d-74d9e339833c.avro",
+ "schema-id": 1,
+ "first-row-id": 2,
+ "added-rows": 2
+ },
+ {
+ "sequence-number": 3,
+ "snapshot-id": 5788798547140820000,
+ "parent-snapshot-id": 7704689856474798000,
+ "timestamp-ms": 1776501490791,
+ "summary": {
+ "operation": "append",
+ "spark.app.id": "local-1776499633819",
+ "added-data-files": "1",
+ "added-records": "1",
+ "added-files-size": "575",
+ "changed-partition-count": "1",
+ "total-records": "5",
+ "total-files-size": "2565",
+ "total-data-files": "5",
+ "total-delete-files": "0",
+ "total-position-deletes": "0",
+ "total-equality-deletes": "0",
+ "engine-version": "4.0.3-SNAPSHOT",
+ "app-id": "local-1776499633819",
+ "engine-name": "spark",
+ "iceberg-version": "Apache Iceberg 1.10.1 (commit ccb8bc435062171e64bc8b7e5f56e6aed9c5b934)"
+ },
+ "manifest-list": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/snap-5788798547140819570-1-72aaa146-43b8-4166-afa1-03669e13753a.avro",
+ "schema-id": 1,
+ "first-row-id": 4,
+ "added-rows": 1
+ }
+ ],
+ "statistics": [],
+ "partition-statistics": [],
+ "snapshot-log": [
+ {
+ "timestamp-ms": 1776501457059,
+ "snapshot-id": 4871712590577795000
+ },
+ {
+ "timestamp-ms": 1776501490485,
+ "snapshot-id": 7704689856474798000
+ },
+ {
+ "timestamp-ms": 1776501490791,
+ "snapshot-id": 5788798547140820000
+ }
+ ],
+ "metadata-log": [
+ {
+ "timestamp-ms": 1776501427812,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00000-3c5fd953-6d63-4191-aeed-fd5b27df2b31.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501457059,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00001-135ef3a5-7ff1-4d39-9f53-2629acad2aa8.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501471669,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00002-52873544-e39a-4a4a-99e2-5d49f79635b5.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501482719,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00003-3750f591-e622-43d1-bf2f-d6bfe883580c.metadata.json"
+ },
+ {
+ "timestamp-ms": 1776501490485,
+ "metadata-file": "hdfs://localhost:20500/test-warehouse/test_default_part/metadata/00004-4ea03624-eec7-4735-9802-d596d7870c44.metadata.json"
+ }
+ ]
+}
diff --git a/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/version-hint.text b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/version-hint.text
new file mode 100644
index 000000000..1e8b31496
--- /dev/null
+++ b/testdata/data/iceberg_test/iceberg_v3/test_default_part/metadata/version-hint.text
@@ -0,0 +1 @@
+6
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test
new file mode 100644
index 000000000..5a1e25d1d
--- /dev/null
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-default-values.test
@@ -0,0 +1,280 @@
+====
+---- QUERY
+# Test Iceberg V3 default values across multiple file formats
+#
+# This test covers:
+# - Parquet: Full read (initial-default)
+# - ORC: Read (initial-default) support only
+# - Avro: Read (initial-default) support only
+#
+# Note: Impala only supports writing to Parquet format Iceberg tables.
+# ORC and Avro formats are read-only for Iceberg in Impala.
+
+# Test initial-default values for Parquet format
+# Column 'j' has initial-default = -1 and write-default = -999
+# First row: i=1, j is missing (should use initial-default -1)
+# Second row: i=2, j=2 (explicit value)
+SELECT * FROM iceberg_v3_default_value ORDER BY i;
+---- RESULTS
+1,-1
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test initial-default values for ORC format
+SELECT * FROM iceberg_v3_default_value_orc ORDER BY i;
+---- RESULTS
+1,-1
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test initial-default values for Avro format
+SELECT * FROM iceberg_v3_default_value_avro ORDER BY i;
+---- RESULTS
+1,-1
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Parquet: Filter exactly on the initial-default value
+SELECT i, j FROM iceberg_v3_default_value WHERE j = -1 ORDER BY i;
+---- RESULTS
+1,-1
+---- TYPES
+INT,INT
+====
+---- QUERY
+# ORC: Filter exactly on the initial-default value
+SELECT i, j FROM iceberg_v3_default_value_orc WHERE j = -1 ORDER BY i;
+---- RESULTS
+1,-1
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Avro: Filter exactly on the initial-default value
+SELECT i, j FROM iceberg_v3_default_value_avro WHERE j = -1 ORDER BY i;
+---- RESULTS
+1,-1
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Parquet: Filter exactly on explicit value
+SELECT i, j FROM iceberg_v3_default_value WHERE j > 0 ORDER BY i;
+---- RESULTS
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# ORC: Filter exactly on explicit value
+SELECT i, j FROM iceberg_v3_default_value_orc WHERE j > 0 ORDER BY i;
+---- RESULTS
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Avro: Filter exactly on explicit value
+SELECT i, j FROM iceberg_v3_default_value_avro WHERE j > 0 ORDER BY i;
+---- RESULTS
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Parquet: initial-default is a real INT, not SQL NULL — no rows match j IS NULL
+SELECT i, j FROM iceberg_v3_default_value WHERE j IS NULL ORDER BY i;
+---- RESULTS
+---- TYPES
+INT,INT
+====
+---- QUERY
+# ORC: no NULL j after default fill
+SELECT i, j FROM iceberg_v3_default_value_orc WHERE j IS NULL ORDER BY i;
+---- RESULTS
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Avro: no NULL j after default fill
+SELECT i, j FROM iceberg_v3_default_value_avro WHERE j IS NULL ORDER BY i;
+---- RESULTS
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Parquet: verify all materialized defaults and explicitly stored values evaluate to NOT NULL
+SELECT i, j FROM iceberg_v3_default_value WHERE j IS NOT NULL ORDER BY i;
+---- RESULTS
+1,-1
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# ORC: verify NOT NULL evaluation
+SELECT i, j FROM iceberg_v3_default_value_orc WHERE j IS NOT NULL ORDER BY i;
+---- RESULTS
+1,-1
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Avro: verify NOT NULL evaluation
+SELECT i, j FROM iceberg_v3_default_value_avro WHERE j IS NOT NULL ORDER BY i;
+---- RESULTS
+1,-1
+2,2
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Parquet: Compound predicate pushdown testing
+SELECT i, j FROM iceberg_v3_default_value WHERE i = 1 AND j = -1;
+---- RESULTS
+1,-1
+---- TYPES
+INT,INT
+====
+---- QUERY
+# ORC: Compound predicate pushdown testing
+SELECT i, j FROM iceberg_v3_default_value_orc WHERE i = 1 AND j = -1;
+---- RESULTS
+1,-1
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Avro: Compound predicate pushdown testing
+SELECT i, j FROM iceberg_v3_default_value_avro WHERE i = 1 AND j = -1;
+---- RESULTS
+1,-1
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Time travel — first snapshot: schema had only i
+SELECT * FROM iceberg_v3_default_value FOR SYSTEM_VERSION AS OF 7752776210164022855;
+---- RESULTS
+1
+---- TYPES
+INT
+====
+---- QUERY
+# Time travel — ORC first snapshot (schema without j)
+SELECT * FROM iceberg_v3_default_value_orc FOR SYSTEM_VERSION AS OF 1536392834658079256;
+---- RESULTS
+1
+---- TYPES
+INT
+====
+---- QUERY
+# Time travel — Avro first snapshot (schema without j)
+SELECT * FROM iceberg_v3_default_value_avro FOR SYSTEM_VERSION AS OF 2819462914350205316;
+---- RESULTS
+1
+---- TYPES
+INT
+====
+---- QUERY
+# Tests Iceberg V3 `initial-default` read projection against partition evolution.
+# Validates that the scanner correctly projects the read default value for older unpartitioned files,
+# while successfully overriding it with partition key values (or explicit NULLs) for newly partitioned data files.
+
+# Table was created initially with column 'i' and records (1), (2) were inserted.
+# It was then partitioned on the newly added column 'j'. Has initial-default = -1 and write-default = -999
+# Three new records (3,5), (4,10) & (5,NULL) were inserted.
+# For row: i=1 & i=2, j is missing (should use initial-default -1)
+# For others, it must use the partition key value, explicit NULL
+SELECT * FROM test_default_part ORDER BY i;
+---- RESULTS
+1,-1
+2,-1
+3,5
+4,10
+5,NULL
+---- TYPES
+INT,INT
+====
+---- QUERY
+# Test Case 2: Filter on the Default Value
+# This validates that partition pruning doesn't accidentally skip old files
+# that lack partition metadata for j.
+SELECT i FROM test_default_part WHERE j = -1 ORDER BY i;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# Test Case 3: Filter on the Partition Key
+# This tests that expression evaluation works correctly on the newly partitioned
+# data and doesn't get confused by the initial default.
+SELECT i FROM test_default_part WHERE j = 5;
+---- RESULTS
+3
+---- TYPES
+INT
+====
+---- QUERY
+# Test Case 4: Aggregation and Grouping
+SELECT j, COUNT(i) as row_count
+FROM test_default_part
+GROUP BY j
+ORDER BY j;
+---- RESULTS
+-1,2
+5,1
+10,1
+NULL,1
+---- TYPES
+INT,BIGINT
+====
+---- QUERY
+# Table test_complex_default has two columns: i (int), j STRUCT<f1: INT, f2: STRING>
+# Nested field j.f1 with an int initial-default in metadata.
+# Selecting only column i should succeed because j is not materialized.
+SELECT i FROM test_complex_default ORDER BY i;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# SELECT * does not materialize complex columns like struct j unless they
+# are explicitly referenced. Since j is not materialized in this query, the nested default
+# on j.f1 is not checked and the scan succeeds.
+SELECT * FROM test_complex_default;
+---- RESULTS
+1
+2
+---- TYPES
+INT
+====
+---- QUERY
+# Nested field j.f2 does not have any initial-default in metadata.
+# Selecting the nested field j.f2 specifically should succeed.
+SELECT j.f2 FROM test_complex_default;
+---- RESULTS
+'hello'
+'NULL'
+---- TYPES
+STRING
+====
+---- QUERY
+SELECT f2 FROM (SELECT j.f1 as f1, j.f2 as f2 FROM test_complex_default) v;
+---- RESULTS
+'hello'
+'NULL'
+---- TYPES
+STRING
+====
diff --git a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
index 4e4ee3ee9..fcd4a8950 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/iceberg-v3-negative.test
@@ -44,58 +44,22 @@ FROM iceberg_v3_deletion_vectors FOR SYSTEM_VERSION AS OF 2700858159721908397;
INT
====
---- QUERY
-SELECT * FROM iceberg_v3_default_value;
+# Default values on nested struct fields are rejected when materialized.
+# Table test_complex_default: nested field j.f1 has an int initial-default in metadata.
+# Selecting j materializes the struct and its nested fields, so the scan should fail.
+SELECT j FROM test_complex_default;
---- CATCH
-Iceberg columns with default values not supported yet.
+Default values are not supported for nested fields
====
---- QUERY
-# Column 'i' does not have a default value.
-SELECT i FROM iceberg_v3_default_value;
----- RESULTS
-1
-2
----- TYPES
-INT
-====
----- QUERY
-# The first snapshot only has column 'i' which does not have a default value.
-SELECT * FROM iceberg_v3_default_value FOR SYSTEM_VERSION AS OF 7752776210164022855;
----- RESULTS
-1
----- TYPES
-INT
-====
----- QUERY
-# 'j' is not materialized.
-select i from (select * from iceberg_v3_default_value) v;
----- RESULTS
-1
-2
----- TYPES
-INT
-====
----- QUERY
-# 'j' is not materialized.
-select i from (select i, j from iceberg_v3_default_value) v;
----- RESULTS
-1
-2
----- TYPES
-INT
-====
----- QUERY
-select i, file__position, input__file__name from iceberg_v3_default_value;
----- RESULTS
-1,0,regex:'.*00000-0-4bcf51b3-40de-43df-a74a-3eb29c0a9634-0-00001.parquet'
-2,0,regex:'.*00000-1-18a720eb-5773-431a-b440-5e9caa4ac641-0-00001.parquet'
----- TYPES
-INT,BIGINT,STRING
+# Selecting j.f1 materializes the nested field, so the scan should fail.
+SELECT j.f1 FROM test_complex_default;
+---- CATCH
+Default values are not supported for nested fields
====
---- QUERY
-# 'j' is not materialized.
-select count(*) from (select j from iceberg_v3_default_value) v;
----- RESULTS
-2
----- TYPES
-BIGINT
+# Negative test: INSERT omitting a column that has a write-default should fail until supported
+INSERT INTO iceberg_v3_default_value (i) VALUES (100);
+---- CATCH
+Write-default values are not yet supported
====
diff --git a/tests/common/file_utils.py b/tests/common/file_utils.py
index 10a0cc5fc..de7e0ab45 100644
--- a/tests/common/file_utils.py
+++ b/tests/common/file_utils.py
@@ -39,8 +39,8 @@ def create_iceberg_table_from_directory(impala_client, unique_database, table_na
if not warehouse_prefix and unique_database:
warehouse_prefix = os.getenv("DEFAULT_FS", WAREHOUSE_PREFIX)
- # Only orc and parquet tested/supported
- assert file_format == "orc" or file_format == "parquet"
+ # Only orc, parquet, and avro tested/supported
+ assert file_format in ["orc", "parquet", "avro"]
table_location = os.path.expandvars(table_location)
local_dir = os.path.join(table_location, table_name)
diff --git a/tests/query_test/test_iceberg.py b/tests/query_test/test_iceberg.py
index 187c82057..3eb4d31da 100644
--- a/tests/query_test/test_iceberg.py
+++ b/tests/query_test/test_iceberg.py
@@ -2332,6 +2332,7 @@ class TestIcebergV3Table(IcebergTestSuite):
def test_v3_negative(self, vector, unique_database):
self.load_table(unique_database, "iceberg_v3_deletion_vectors")
self.load_table(unique_database, "iceberg_v3_default_value")
+ self.load_table(unique_database, "test_complex_default")
self.run_test_case('QueryTest/iceberg-v3-negative', vector, unique_database)
def test_v3_row_lineage(self, vector, unique_database):
@@ -2371,6 +2372,15 @@ class TestIcebergV3Table(IcebergTestSuite):
def test_v3_optimize(self, vector, unique_database):
self.run_test_case('QueryTest/iceberg-v3-optimize', vector, unique_database)
+ def test_v3_default_values(self, vector, unique_database):
+ """Test Iceberg V3 initial-default and write-default values."""
+ self.load_table(unique_database, "iceberg_v3_default_value")
+ self.load_table(unique_database, "iceberg_v3_default_value_orc", format="orc")
+ self.load_table(unique_database, "iceberg_v3_default_value_avro", format="avro")
+ self.load_table(unique_database, "test_default_part")
+ self.load_table(unique_database, "test_complex_default")
+ self.run_test_case('QueryTest/iceberg-v3-default-values', vector, unique_database)
+
# Tests to exercise the DIRECTED distribution mode for V2 Iceberg tables. Note, that most
# of the test coverage is in TestIcebergV2Table.test_read_position_deletes but since it
(impala) 01/03: IMPALA-14897: [Patch 1 of 2] Configure TLS v1.3 Cipher Suites on Thrift Servers
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 417ba2d3eb1d252eb21834d1cf6f9e4f21ba49f9
Author: jasonmfehr <jf...@cloudera.com>
AuthorDate: Thu Apr 16 15:45:07 2026 -0700
IMPALA-14897: [Patch 1 of 2] Configure TLS v1.3 Cipher Suites on Thrift Servers
Adds configuration of Impala Thrift servers to set the
TLS 1.3 allowed ciphersuites.
The '--tls_ciphersuites' parameter sets the list of
TLS 1.3 ciphersuites that each Thrift server is allowed to
choose from during TLS handshakes from inbound client
connections. The value of this parameter is now being set
on all 'ThriftServerBuilder' instances so that each Thrift
server only uses the configured allowed TLS 1.3
ciphersuites. Before, 'ThriftServerBuilder' instances used
all supported OS OpenSSL TLS 1.3 ciphersuites.
The lone exception is the external frontend Thrift server
since that Thrift server does not support any TLS
connections. IMPALA-14898 was opened to add TLS support to
that Thrift server.
The following supporting changes are included:
1. start-impala-cluster.py: New startup flag '--enable_tls'
added to spin up a TLS 1.2/1.3 localhost cluster.
2. localhost.pem: Regenerated certificate to be signed by
the wildcardCA.pem CA certificate. This certificate was
not used during testing until test_server_tls.py.
3. certificates-info.txt: Added details about the localhost
certificate/key.
4. network.py: Added generic TLS-related functions that can
be used in future TLS testing.
5. custom_cluster_test_suite.py: Added 'admissiond_args'
parameter to CustomClusterTestSuite.with_args' to enable
specifying the '--admissiond_args' parameter when
starting the cluster.
Testing accomplished by manual checks using
openssl s_client and a new custom cluster test that does
both positive assertions that ciphersuites configured on
the cluster are allowed and negative assertions that
ciphersuites supported by the OS OpenSSL but not configured
on the cluster are not allowed.
kRPC testing will be handled in patch 2.
Change-Id: Id88a4e10fbad3132d14e35bcbbbc73d343c7761a
Generated-by: Github Copilot (GPT-5.3-Codex)
Reviewed-on: http://gerrit.cloudera.org:8080/24198
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/catalog/catalogd-main.cc | 4 +-
be/src/service/impala-server.cc | 9 +-
be/src/statestore/statestore-subscriber.cc | 4 +-
be/src/statestore/statestore.cc | 7 +-
be/src/testutil/certificates-info.txt | 29 +++
be/src/testutil/localhost-ecdsa.key | 5 +
be/src/testutil/localhost-ecdsa.pem | 16 ++
be/src/testutil/localhost.pem | 33 +--
be/src/testutil/wildcardCA-ecdsa.key | 5 +
be/src/testutil/wildcardCA-ecdsa.pem | 16 ++
bin/start-impala-cluster.py | 43 ++++
tests/common/custom_cluster_test_suite.py | 8 +-
tests/common/impala_cluster.py | 4 +
tests/common/network.py | 67 +++++-
tests/custom_cluster/test_server_tls.py | 353 +++++++++++++++++++++++++++++
15 files changed, 576 insertions(+), 27 deletions(-)
diff --git a/be/src/catalog/catalogd-main.cc b/be/src/catalog/catalogd-main.cc
index 3869b4023..7c815c2a1 100644
--- a/be/src/catalog/catalogd-main.cc
+++ b/be/src/catalog/catalogd-main.cc
@@ -47,6 +47,7 @@ DECLARE_string(ssl_private_key);
DECLARE_string(ssl_private_key_password_cmd);
DECLARE_string(ssl_cipher_list);
DECLARE_string(ssl_minimum_version);
+DECLARE_string(tls_ciphersuites);
#include "common/names.h"
using namespace impala;
@@ -97,7 +98,8 @@ int CatalogdMain(int argc, char** argv) {
builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
ABORT_IF_ERROR(builder.metrics(daemon_env.metrics()).Build(&server));
ABORT_IF_ERROR(server->Start());
diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc
index f7a58469f..c3ee334a2 100644
--- a/be/src/service/impala-server.cc
+++ b/be/src/service/impala-server.cc
@@ -3283,7 +3283,8 @@ Status ImpalaServer::Start(int32_t beeswax_port, int32_t hs2_port,
builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
ThriftServer* server;
@@ -3316,7 +3317,8 @@ Status ImpalaServer::Start(int32_t beeswax_port, int32_t hs2_port,
builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
ThriftServer* server;
@@ -3386,7 +3388,8 @@ Status ImpalaServer::Start(int32_t beeswax_port, int32_t hs2_port,
http_builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
RETURN_IF_ERROR(
diff --git a/be/src/statestore/statestore-subscriber.cc b/be/src/statestore/statestore-subscriber.cc
index 9e7b5ea99..0638c96d8 100644
--- a/be/src/statestore/statestore-subscriber.cc
+++ b/be/src/statestore/statestore-subscriber.cc
@@ -78,6 +78,7 @@ DECLARE_string(ssl_private_key);
DECLARE_string(ssl_private_key_password_cmd);
DECLARE_string(ssl_cipher_list);
DECLARE_string(ssl_minimum_version);
+DECLARE_string(tls_ciphersuites);
namespace impala {
@@ -313,7 +314,8 @@ Status StatestoreSubscriber::Start() {
builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
ThriftServer* server;
diff --git a/be/src/statestore/statestore.cc b/be/src/statestore/statestore.cc
index 98ab753cc..a752a19f8 100644
--- a/be/src/statestore/statestore.cc
+++ b/be/src/statestore/statestore.cc
@@ -171,6 +171,7 @@ DECLARE_string(ssl_private_key);
DECLARE_string(ssl_private_key_password_cmd);
DECLARE_string(ssl_cipher_list);
DECLARE_string(ssl_minimum_version);
+DECLARE_string(tls_ciphersuites);
#ifndef NDEBUG
DECLARE_int32(stress_statestore_startup_delay_ms);
#endif
@@ -796,7 +797,8 @@ Status Statestore::Init(int32_t state_store_port) {
builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
RETURN_IF_ERROR(subscriber_topic_update_threadpool_.Init());
RETURN_IF_ERROR(subscriber_priority_topic_update_threadpool_.Init());
@@ -1844,7 +1846,8 @@ Status Statestore::InitStatestoreHa(
builder.ssl(FLAGS_ssl_server_certificate, FLAGS_ssl_private_key)
.pem_password_cmd(FLAGS_ssl_private_key_password_cmd)
.ssl_version(ssl_version)
- .cipher_list(FLAGS_ssl_cipher_list);
+ .cipher_list(FLAGS_ssl_cipher_list)
+ .tls_ciphersuites(FLAGS_tls_ciphersuites);
}
ThriftServer* ha_server;
RETURN_IF_ERROR(builder.metrics(metrics_).Build(&ha_server));
diff --git a/be/src/testutil/certificates-info.txt b/be/src/testutil/certificates-info.txt
index 72ae886da..320cd2aef 100644
--- a/be/src/testutil/certificates-info.txt
+++ b/be/src/testutil/certificates-info.txt
@@ -82,6 +82,35 @@ responsible for and how they were created:
This was created the same as 1) with a different commonName.
-------------
+ 6) localhost.pem & localhost.key:
+
+ This is a certificate signed by wildcardCA.pem/wildcardCA.key for localhost
+ testing. The subject has no locality or emailAddress and uses:
+ C=US, ST=CA, O=Cloudera, OU=Impala, CN=localhost
+ It has the subject alternative name "DNS:localhost,IP:127.0.0.1".
+
+-------------
+ 7) wildcardCA-ecdsa.pem & wildcardCA-ecdsa.key:
+
+ This is a root CA certificate and it's private key that uses the signature algorithm
+ "ecdsa-with-SHA256" and public key algorithm "id-ecPublicKey". It was used to sign
+ localhost-ecdsa.pem.
+
+ This root CA certificate differs from wildcardCA.pem in the supported TLS 1.2
+ ciphersuites. Can be used to sign client/server certificates for use with the
+ TLS 1.2 ciphersuites who's names start with "ECDHE-ECDSA-".
+
+-------------
+ 8) localhost-ecdsa.pem & localhost-ecdsa.key:
+
+ This is a certificate signed by wildcardCA-ecdsa.pem/wildcardCA-ecdsa.key for
+ localhost testing that uses the signature algorithm "ecdsa-with-SHA256" and public key
+ algorithm "id-ecPublicKey". The subject has no locality or emailAddress and uses:
+ C=US, ST=CA, O=Cloudera, OU=Impala, CN=localhost
+ It has the subject alternative name "DNS:localhost,IP:127.0.0.1".
+
+-------------
+
To verify the contents of any certificate, do the following:
openssl x509 -in <certificate_name> -text
diff --git a/be/src/testutil/localhost-ecdsa.key b/be/src/testutil/localhost-ecdsa.key
new file mode 100644
index 000000000..911575444
--- /dev/null
+++ b/be/src/testutil/localhost-ecdsa.key
@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIDGgeG5gBAo1zAh0jvwbXkPaspxwR4hjY/ctwQHR/uwdoAoGCCqGSM49
+AwEHoUQDQgAEGo6h8EQzDJUidW+B73NEMx1pvlfboLRziwNPLs2vzgr7WKQHPLnX
+8JaTNhuH5pdLJTza3TXVuOKjOor7lbJKfQ==
+-----END EC PRIVATE KEY-----
diff --git a/be/src/testutil/localhost-ecdsa.pem b/be/src/testutil/localhost-ecdsa.pem
new file mode 100644
index 000000000..c1f915519
--- /dev/null
+++ b/be/src/testutil/localhost-ecdsa.pem
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICdjCCAhygAwIBAgIUBuz3jpXfjfflMHlP2WkruVGECVcwCgYIKoZIzj0EAwIw
+gZIxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxETAPBgNV
+BAoMCENsb3VkZXJhMQ8wDQYDVQQLDAZJbXBhbGExIDAeBgNVBAMMF1dsZGNhcmQg
+cm9vdCBDQSAtIEVDRFNBMSMwIQYJKoZIhvcNAQkBFhRzYWlsZXNoQGNsb3VkZXJh
+LmNvbTAgFw0yNjA0MjAyMDAwMDhaGA8yMTI2MDMyNzIwMDAwOFowUjELMAkGA1UE
+BhMCVVMxCzAJBgNVBAgMAkNBMREwDwYDVQQKDAhDbG91ZGVyYTEPMA0GA1UECwwG
+SW1wYWxhMRIwEAYDVQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjOPQMB
+BwNCAAQajqHwRDMMlSJ1b4Hvc0QzHWm+V9ugtHOLA08uza/OCvtYpAc8udfwlpM2
+G4fml0slPNrdNdW44qM6ivuVskp9o4GMMIGJMB8GA1UdIwQYMBaAFGXIaFPmwaX6
+aCMdAWOmoEnb5fvCMB0GA1UdDgQWBBRSMGDSFjqz37tHBVSLPFkudIPk7zAJBgNV
+HRMEAjAAMAsGA1UdDwQEAwIHgDATBgNVHSUEDDAKBggrBgEFBQcDATAaBgNVHREE
+EzARgglsb2NhbGhvc3SHBH8AAAEwCgYIKoZIzj0EAwIDSAAwRQIgHY1um36/V6WB
+paEJQpohFd+EZFEMUQ/6t5yBCtY59pECIQCIK2ebhxlYgTaO/t5v+Hzxf+xZ4XBv
+evqdnz/2ZsS5zQ==
+-----END CERTIFICATE-----
diff --git a/be/src/testutil/localhost.pem b/be/src/testutil/localhost.pem
index 223a994c1..2e301128d 100644
--- a/be/src/testutil/localhost.pem
+++ b/be/src/testutil/localhost.pem
@@ -1,20 +1,23 @@
-----BEGIN CERTIFICATE-----
-MIIDTjCCAjagAwIBAgIUB8U8sXAUFbgKk2FmztEoZkC6JFswDQYJKoZIhvcNAQEL
+MIIDzDCCArSgAwIBAgIUeezc1e2jNAGC8cnaJ9pk6bWnoXEwDQYJKoZIhvcNAQEL
BQAwgYsxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxETAP
BgNVBAoMCENsb3VkZXJhMQ8wDQYDVQQLDAZJbXBhbGExGTAXBgNVBAMMEFdpbGRj
YXJkIFJvb3QgQ0ExIzAhBgkqhkiG9w0BCQEWFHNhaWxlc2hAY2xvdWRlcmEuY29t
-MCAXDTI1MTAxMzE0MTQ0MloYDzIxMjUwOTE5MTQxNDQyWjAUMRIwEAYDVQQDDAls
-b2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCx0nVyp98j
-3GyxDKwscXgiIDMT1hJjVSDeyiI8+wnqrfahYJKAdxX67n9pX2/Q8PYjgsaWbys6
-1KI5sI7oI1asTEHY/Od3TZHC8mL3QEKjwRxMfjjrvr7r1aIUK2t1BvAMk/xdFktY
-2uQ49RMC4w50a5CQq2Mag5OywmZVnPG/5RwlbYE5yHSEJuK9qMH57SqhOtPHWE9G
-Zb/2xNUk0MhU+3SAyhzoRid07P7gE2qHJIG7Z0Z8FxpDY/AI+HxmCfO25vDyLAE8
-izWg1jefd2NY+fGjxkbzacWq70vjfmLLv+EYFkYNFRSBSLM11LzEK3S4vtNc4f+C
-WXIXaHFUf6PFAgMBAAGjHjAcMBoGA1UdEQQTMBGCCWxvY2FsaG9zdIcEfwAAATAN
-BgkqhkiG9w0BAQsFAAOCAQEAmW0tQjNXzysc0iViaRpjapNfgDLIVSgw18dzmBYK
-mMK1kFFxgdXcr2PmzXs7dnuUnO5UbvqXdIsAa0UqSZwQvSjFbwpMLdEJ9fj4H9aH
-N42BkL/oDomI5CS2hKCcY7yFtLZaNQHfJRDbzd9EzmmVURtGDyGwqKfT6ThbT9r+
-ec+jVtR1Lj5tnCh0++7eJfUJaMZ6FXp6vluGvkoSZPywvXp7H8FXqRvcXrBRrvrb
-CoOVqrahmifyBRRL/uOBkakLxd/dbcArlo1I3P40LG7SHJ/QUwiflPVz2pApdMt+
-9HTvP6uRU0PdqxzoEvJ+mZA+ygePUdUF7xu9oM8OfxJXIQ==
+MCAXDTI2MDQxNjIyMDQ0NloYDzIxMjYwMzIzMjIwNDQ2WjBSMQswCQYDVQQGEwJV
+UzELMAkGA1UECAwCQ0ExETAPBgNVBAoMCENsb3VkZXJhMQ8wDQYDVQQLDAZJbXBh
+bGExEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCC
+AQoCggEBALHSdXKn3yPcbLEMrCxxeCIgMxPWEmNVIN7KIjz7Ceqt9qFgkoB3Ffru
+f2lfb9Dw9iOCxpZvKzrUojmwjugjVqxMQdj853dNkcLyYvdAQqPBHEx+OOu+vuvV
+ohQra3UG8AyT/F0WS1ja5Dj1EwLjDnRrkJCrYxqDk7LCZlWc8b/lHCVtgTnIdIQm
+4r2owfntKqE608dYT0Zlv/bE1STQyFT7dIDKHOhGJ3Ts/uATaockgbtnRnwXGkNj
+8Aj4fGYJ87bm8PIsATyLNaDWN593Y1j58aPGRvNpxarvS+N+Ysu/4RgWRg0VFIFI
+szXUvMQrdLi+01zh/4JZchdocVR/o8UCAwEAAaNeMFwwGgYDVR0RBBMwEYIJbG9j
+YWxob3N0hwR/AAABMB0GA1UdDgQWBBQGGVMuL30lgs8ieEZkxl4GGqfkHTAfBgNV
+HSMEGDAWgBQhBLvTLXXNLmU3Lo2l2Qxw8INTBjANBgkqhkiG9w0BAQsFAAOCAQEA
+A009wj1Lc7Tp0k6gwbwMKePJDhEUllcs8q+nNikUgdWQxvTCcj6tdcdtRHzo/u3o
+NczYT2Yk7x8MXMuYAFSCNzy9MC1ShbbJv4ilUw+c40NyDA3iSObohenQpCdLCYdZ
+Qm2vGMiSTY/iVhm2eR9wtYU3+kCvZzJ+FHnBUucqFAUoMo+OS2sZHLXWdO+BIR/u
+eRMiBihvM8H/T/qpipRxfjpn+NN8Ivh/DSaOwJeDSbUBpw1lo8Wp39IHtBGpwThq
+rVS+ky2SXIUnP0LIW3I51xHi51NSDnIUp40ehg9Wlx2FiYupFeoZconWfNXMtZYO
+7asEl/OHxNQEzikF+XwWdw==
-----END CERTIFICATE-----
diff --git a/be/src/testutil/wildcardCA-ecdsa.key b/be/src/testutil/wildcardCA-ecdsa.key
new file mode 100644
index 000000000..a483c72cc
--- /dev/null
+++ b/be/src/testutil/wildcardCA-ecdsa.key
@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEICqXnNyo8IJlQqCsZ9YpsyLHaPKcM8VbbY9vrim077m5oAoGCCqGSM49
+AwEHoUQDQgAE1rljOehQQPBRWc1PRCPjbBHSbHoFvT/LK9yp/mF/iMmtx9g88+M+
+8VvLmUzy3jPkOJsqqICjqmq4tysIcRZc5g==
+-----END EC PRIVATE KEY-----
diff --git a/be/src/testutil/wildcardCA-ecdsa.pem b/be/src/testutil/wildcardCA-ecdsa.pem
new file mode 100644
index 000000000..2fbaae052
--- /dev/null
+++ b/be/src/testutil/wildcardCA-ecdsa.pem
@@ -0,0 +1,16 @@
+-----BEGIN CERTIFICATE-----
+MIICjTCCAjOgAwIBAgIURBWcd8FncSwr2LoDbmwbusLsSPAwCgYIKoZIzj0EAwIw
+gZIxCzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxETAPBgNV
+BAoMCENsb3VkZXJhMQ8wDQYDVQQLDAZJbXBhbGExIDAeBgNVBAMMF1dsZGNhcmQg
+cm9vdCBDQSAtIEVDRFNBMSMwIQYJKoZIhvcNAQkBFhRzYWlsZXNoQGNsb3VkZXJh
+LmNvbTAgFw0yNjA0MjAxOTU2MzdaGA8yMDUzMDkwNTE5NTYzN1owgZIxCzAJBgNV
+BAYTAlVTMQswCQYDVQQIDAJDQTELMAkGA1UEBwwCU0YxETAPBgNVBAoMCENsb3Vk
+ZXJhMQ8wDQYDVQQLDAZJbXBhbGExIDAeBgNVBAMMF1dsZGNhcmQgcm9vdCBDQSAt
+IEVDRFNBMSMwIQYJKoZIhvcNAQkBFhRzYWlsZXNoQGNsb3VkZXJhLmNvbTBZMBMG
+ByqGSM49AgEGCCqGSM49AwEHA0IABNa5YznoUEDwUVnNT0Qj42wR0mx6Bb0/yyvc
+qf5hf4jJrcfYPPPjPvFby5lM8t4z5DibKqiAo6pquLcrCHEWXOajYzBhMA8GA1Ud
+EwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgEGMB0GA1UdDgQWBBRlyGhT5sGl+mgj
+HQFjpqBJ2+X7wjAfBgNVHSMEGDAWgBRlyGhT5sGl+mgjHQFjpqBJ2+X7wjAKBggq
+hkjOPQQDAgNIADBFAiEAoyEdLERB51DpxSIBKoQS5+aLMZMMUbH2qMDVQqp/7yAC
+IC8EVOixChaaxD9OSmHIjRRkgGlaopSZy/vBubmBxrfJ
+-----END CERTIFICATE-----
diff --git a/bin/start-impala-cluster.py b/bin/start-impala-cluster.py
index f452b0a1a..791ec3d67 100755
--- a/bin/start-impala-cluster.py
+++ b/bin/start-impala-cluster.py
@@ -210,6 +210,12 @@ parser.add_option("--use_calcite_planner", type="choice", choices=BOOLEAN_STRING
parser.add_option("--enable_ranger_authz", dest="enable_ranger_authz",
action="store_true", default=False,
help="If true, enable Ranger authorization in Impala cluster.")
+parser.add_option("--enable_tls", dest="enable_tls",
+ action="store_true", default=False,
+ help="If true, enable TLS on webserver and Thrift ports. "
+ "uses be/src/testutil/localhost.pem as the server certificate and "
+ "be/src/testutil/wildcardCA.pem as the CA certificate. Also binds "
+ "all server sockets to localhost instead of 0.0.0.0")
# For testing: list of comma-separated delays, in milliseconds, that delay impalad catalog
# replica initialization. The ith delay is applied to the ith impalad.
@@ -238,6 +244,9 @@ KUDU_RPC_TIMEOUT = build_flavor_timeout(0, slow_build_timeout=60000)
# during builds to make spurious session expiration less likely.
DISCONNECTED_SESSION_TIMEOUT = 60 * 60 * 6
+# Directory where TLS certificates are located.
+CERT_DIR = "{}/be/src/testutil".format(os.environ['IMPALA_HOME'])
+
def check_process_exists(binary, attempts=1):
"""Checks if a process exists given the binary name. The `attempts` count allows us to
control the time a process needs to settle until it becomes available. After each try
@@ -462,6 +471,14 @@ def build_statestored_arg_list(num_statestored, remap_ports):
args.extend(["-enable_catalogd_ha=true"])
if options.enable_statestored_ha:
args.extend(["-enable_statestored_ha=true"])
+ if options.enable_tls:
+ args.extend(["--hostname=localhost",
+ "--webserver_interface=localhost",
+ "--webserver_certificate_file={0}/localhost.pem".format(CERT_DIR),
+ "--webserver_private_key_file={0}/localhost.key".format(CERT_DIR),
+ "--ssl_client_ca_certificate={0}/wildcardCA.pem".format(CERT_DIR),
+ "--ssl_server_certificate={0}/localhost.pem".format(CERT_DIR),
+ "--ssl_private_key={0}/localhost.key".format(CERT_DIR)])
statestored_arg_list.append(args)
return statestored_arg_list
@@ -491,6 +508,14 @@ def build_catalogd_arg_list(num_catalogd, remap_ports):
if options.enable_ranger_authz:
args.extend(["--server-name=server1", "--ranger_service_type=hive",
"--ranger_app_id=impala", "--authorization_provider=ranger"])
+ if options.enable_tls:
+ args.extend(["--hostname=localhost",
+ "--webserver_interface=localhost",
+ "--webserver_certificate_file={0}/localhost.pem".format(CERT_DIR),
+ "--webserver_private_key_file={0}/localhost.key".format(CERT_DIR),
+ "--ssl_client_ca_certificate={0}/wildcardCA.pem".format(CERT_DIR),
+ "--ssl_server_certificate={0}/localhost.pem".format(CERT_DIR),
+ "--ssl_private_key={0}/localhost.key".format(CERT_DIR)])
catalogd_arg_list.append(args)
return catalogd_arg_list
@@ -507,6 +532,14 @@ def build_admissiond_arg_list():
["-state_store_port={0}".format(state_store_port)])
args.extend(
["-state_store_2_port={0}".format(state_store_port + 1)])
+ if options.enable_tls:
+ args.extend(["--hostname=localhost",
+ "--webserver_interface=localhost",
+ "--webserver_certificate_file={0}/localhost.pem".format(CERT_DIR),
+ "--webserver_private_key_file={0}/localhost.key".format(CERT_DIR),
+ "--ssl_client_ca_certificate={0}/wildcardCA.pem".format(CERT_DIR),
+ "--ssl_server_certificate={0}/localhost.pem".format(CERT_DIR),
+ "--ssl_private_key={0}/localhost.key".format(CERT_DIR)])
return args
@@ -727,6 +760,16 @@ def build_impalad_arg_lists(cluster_size, num_coordinators, use_exclusive_coordi
args = ("--server-name=server1 --ranger_service_type=hive --ranger_app_id=impala "
"--authorization_provider=ranger {args}").format(args=args)
+ if options.enable_tls:
+ args = ("--hostname=localhost "
+ "--webserver_interface=localhost "
+ "--webserver_certificate_file={cert_dir}/localhost.pem "
+ "--webserver_private_key_file={cert_dir}/localhost.key "
+ "--ssl_client_ca_certificate={cert_dir}/wildcardCA.pem "
+ "--ssl_server_certificate={cert_dir}/localhost.pem "
+ "--ssl_private_key={cert_dir}/localhost.key {args}") \
+ .format(args=args, cert_dir=CERT_DIR)
+
# Appended at the end so they can override previous args.
if i < len(per_impalad_args):
args = "{args} {per_impalad_args}".format(
diff --git a/tests/common/custom_cluster_test_suite.py b/tests/common/custom_cluster_test_suite.py
index d3a96ef46..3b38ae96c 100644
--- a/tests/common/custom_cluster_test_suite.py
+++ b/tests/common/custom_cluster_test_suite.py
@@ -167,7 +167,8 @@ class CustomClusterTestSuite(ImpalaTestSuite):
impalad_timeout_s=None, expect_cores=None, reset_ranger=False,
tmp_dir_placeholders=[],
expect_startup_fail=False, disable_log_buffering=False, log_symlinks=False,
- workload_mgmt=False, force_restart=False, custom_core_site_dir=None):
+ workload_mgmt=False, force_restart=False, custom_core_site_dir=None,
+ admissiond_args=None):
"""Records arguments to be passed to a cluster by adding them to the decorated
method's func_dict"""
args = dict()
@@ -217,6 +218,8 @@ class CustomClusterTestSuite(ImpalaTestSuite):
# When sharding tests, always restart the cluster to avoid issues with tests
# that depend on a specific test order within a shard.
args[FORCE_RESTART] = True
+ if admissiond_args is not None:
+ args[ADMISSIOND_ARGS] = admissiond_args
def merge_args(args_first, args_last):
result = args_first.copy()
@@ -230,7 +233,8 @@ class CustomClusterTestSuite(ImpalaTestSuite):
CATALOGD_ARGS,
START_ARGS,
JVM_ARGS,
- KUDU_ARGS
+ KUDU_ARGS,
+ ADMISSIOND_ARGS
):
# Let the server decide.
result[key] = " ".join((result[key], args_last[key]))
diff --git a/tests/common/impala_cluster.py b/tests/common/impala_cluster.py
index 41abc41c6..5d17ac0da 100644
--- a/tests/common/impala_cluster.py
+++ b/tests/common/impala_cluster.py
@@ -53,15 +53,19 @@ DEFAULT_EXTERNAL_FE_PORT = 21150
DEFAULT_HS2_HTTP_PORT = 28000
DEFAULT_KRPC_PORT = 27000
DEFAULT_CATALOG_SERVICE_PORT = 26000
+DEFAULT_CATALOG1_SERVICE_PORT = 26001
DEFAULT_STATE_STORE_SUBSCRIBER_PORT = 23000
DEFAULT_CATALOGD_STATE_STORE_SUBSCRIBER_PORT = 23020
+DEFAULT_CATALOGD1_STATE_STORE_SUBSCRIBER_PORT = 23021
DEFAULT_STATESTORE_SERVICE_PORT = 24000
+DEFAULT_STATESTORE1_SERVICE_PORT = 24001
DEFAULT_STATESTORE_HA_SERVICE_PORT = 24020
DEFAULT_PEER_STATESTORE_HA_SERVICE_PORT = 24021
DEFAULT_IMPALAD_WEBSERVER_PORT = 25000
DEFAULT_STATESTORED_WEBSERVER_PORT = 25010
DEFAULT_CATALOGD_WEBSERVER_PORT = 25020
DEFAULT_ADMISSIOND_WEBSERVER_PORT = 25030
+DEFAULT_ADMISSIOND_STATE_STORE_SUBSCRIBER_PORT = 23030
DEFAULT_IMPALAD_JVM_DEBUG_PORT = 30000
DEFAULT_CATALOGD_JVM_DEBUG_PORT = 30030
diff --git a/tests/common/network.py b/tests/common/network.py
index c58baa9b4..48fd6b2e3 100644
--- a/tests/common/network.py
+++ b/tests/common/network.py
@@ -20,8 +20,8 @@
from __future__ import absolute_import, division, print_function
import socket
import ssl
+import subprocess
-from tests.common.environ import IS_REDHAT_DERIVATIVE
# Retrieves the host external IP rather than localhost/127.0.0.1 so we have an IP that
# Impala will consider distinct from storage backends to force remote scheduling.
@@ -58,12 +58,73 @@ def to_host_port(host, port):
return fmt.format(host, port)
+def run_openssl_cmd(args, timeout_s=3):
+ """
+ Runs the 'openssl' command with the provided args array as a subprocess. Force closes
+ STDIN without writing any data to it which avoids openssl commands hanging due to
+ waiting for input.
+
+ Args:
+ args: List of strings forming the arguments to the 'openssl' command.
+ timeout_s: Optional timeout in seconds. If provided and exceeded, the process is
+ killed.
+
+ Returns: Tuple containing the return code and the combined stdout and stderr output
+ as a string.
+ """
+
+ assert args[0] != "openssl", "'openssl' command must not be included in the args list."
+ args = ["openssl"] + args
+
+ proc = subprocess.Popen(args, stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT, universal_newlines=True)
+
+ try:
+ if timeout_s is None:
+ stdout, stderr = proc.communicate(input="")
+ else:
+ stdout, stderr = proc.communicate(input="", timeout=timeout_s)
+ except subprocess.TimeoutExpired as e:
+ # TimeoutExpired exceptions usually indicate the server port does not support TLS.
+ proc.kill()
+ stdout = proc.communicate()
+ raise e
+
+ out = stdout
+ if not isinstance(out, str):
+ out = out.decode("utf-8", "replace")
+
+ return proc.returncode, out
+
+
+def __get_openssl_supported_ciphers(tls_flag):
+ """
+ Note: Do not call this function directly. Use the OPENSSL_TLS_1_2_CIPHERSUITES and
+ OPENSSL_TLS_1_3_CIPHERSUITES variables instead which are initialized at module load.
+
+ Returns a sorted list of ciphersuites supported by the OS OpenSSL for the given
+ TLS version flag. Allowed values for tls_flag are '-tls1_2' and '-tls1_3'.
+
+ If querying openssl fails, returns an empty list.
+ """
+ rc, out = run_openssl_cmd(["ciphers", "-s", tls_flag])
+ if rc != 0:
+ return []
+ ciphers = set([c.strip() for c in out.strip().split(":") if c.strip()])
+ return sorted(ciphers)
+
+
+OPENSSL_TLS_1_2_CIPHERSUITES = __get_openssl_supported_ciphers("-tls1_2")
+OPENSSL_TLS_1_3_CIPHERSUITES = __get_openssl_supported_ciphers("-tls1_3")
+
CERT_TO_CA_MAP = {
"wildcard-cert.pem": "wildcardCA.pem",
- "wildcard-san-cert.pem": "wildcardCA.pem"
+ "wildcard-san-cert.pem": "wildcardCA.pem",
+ "localhost.pem": "wildcardCA.pem",
+ "localhost-ecdsa.pem": "wildcardCA-ecdsa.pem",
}
-REQUIRED_MIN_OPENSSL_VERSION = 0x10001000
+REQUIRED_MIN_OPENSSL_VERSION = 0x10101000
_openssl_version_number = getattr(ssl, "OPENSSL_VERSION_NUMBER", None)
if _openssl_version_number is None:
SKIP_SSL_MSG = "Legacy OpenSSL module detected"
diff --git a/tests/custom_cluster/test_server_tls.py b/tests/custom_cluster/test_server_tls.py
new file mode 100644
index 000000000..24aa6b3a1
--- /dev/null
+++ b/tests/custom_cluster/test_server_tls.py
@@ -0,0 +1,353 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from __future__ import absolute_import, division, print_function
+import os
+import pytest
+
+from tests.common.custom_cluster_test_suite import CustomClusterTestSuite
+from tests.common.impala_cluster import (
+ DEFAULT_BEESWAX_PORT, DEFAULT_HS2_PORT, DEFAULT_CATALOG1_SERVICE_PORT,
+ DEFAULT_STATE_STORE_SUBSCRIBER_PORT, DEFAULT_CATALOGD_STATE_STORE_SUBSCRIBER_PORT,
+ DEFAULT_CATALOGD1_STATE_STORE_SUBSCRIBER_PORT, DEFAULT_HS2_HTTP_PORT,
+ DEFAULT_ADMISSIOND_STATE_STORE_SUBSCRIBER_PORT, DEFAULT_IMPALAD_WEBSERVER_PORT,
+ DEFAULT_STATESTORED_WEBSERVER_PORT, DEFAULT_CATALOGD_WEBSERVER_PORT,
+ DEFAULT_ADMISSIOND_WEBSERVER_PORT, DEFAULT_STATESTORE_SERVICE_PORT,
+ DEFAULT_STATESTORE1_SERVICE_PORT, DEFAULT_STATESTORE_HA_SERVICE_PORT,
+ DEFAULT_PEER_STATESTORE_HA_SERVICE_PORT, DEFAULT_CATALOG_SERVICE_PORT,
+)
+from tests.common.network import (
+ OPENSSL_TLS_1_2_CIPHERSUITES,
+ OPENSSL_TLS_1_3_CIPHERSUITES,
+ SKIP_SSL_MSG,
+ run_openssl_cmd
+)
+from tests.common.test_vector import ImpalaTestDimension
+
+
+# TLS ports to assert.
+TLS_PORTS = [
+ DEFAULT_BEESWAX_PORT,
+ DEFAULT_HS2_PORT,
+ DEFAULT_HS2_HTTP_PORT,
+ DEFAULT_STATE_STORE_SUBSCRIBER_PORT,
+ DEFAULT_STATESTORE_SERVICE_PORT,
+ DEFAULT_STATESTORE1_SERVICE_PORT,
+ DEFAULT_STATESTORED_WEBSERVER_PORT,
+ DEFAULT_PEER_STATESTORE_HA_SERVICE_PORT,
+ DEFAULT_CATALOGD_STATE_STORE_SUBSCRIBER_PORT,
+ DEFAULT_CATALOGD1_STATE_STORE_SUBSCRIBER_PORT,
+ DEFAULT_CATALOG_SERVICE_PORT,
+ DEFAULT_CATALOG1_SERVICE_PORT,
+ DEFAULT_CATALOGD_WEBSERVER_PORT,
+ DEFAULT_IMPALAD_WEBSERVER_PORT,
+ DEFAULT_ADMISSIOND_STATE_STORE_SUBSCRIBER_PORT,
+ DEFAULT_ADMISSIOND_WEBSERVER_PORT,
+ DEFAULT_STATESTORE_HA_SERVICE_PORT,
+]
+
+# TLS 1.2 RSA ciphersuites allowed by the cluster configuration.
+TLS_1_2_CIPHERSUITES_RSA = [
+ "ECDHE-RSA-AES128-GCM-SHA256",
+ "ECDHE-RSA-AES256-GCM-SHA384",
+]
+
+# TLS 1.2 ECDSA ciphersuites allowed by the cluster configuration.
+TLS_1_2_CIPHERSUITES_ECDSA = [
+ "ECDHE-ECDSA-AES128-GCM-SHA256",
+ "ECDHE-ECDSA-AES256-SHA",
+]
+
+# TLS 1.3 ciphersuites allowed by the cluster configuration.
+TLS_1_3_CIPHERSUITES = [
+ "TLS_AES_128_GCM_SHA256",
+ "TLS_AES_256_GCM_SHA384",
+]
+
+# Strings defining each TLS version
+TLS_VER_10 = "1.0"
+TLS_VER_11 = "1.1"
+TLS_VER_12 = "1.2"
+TLS_VER_13 = "1.3"
+
+# Directory that contains the test TLS certificates and private keys.
+CERT_DIR = "{}/be/src/testutil".format(os.environ['IMPALA_HOME'])
+
+# Test dimension specific values. Set up here instead of in add_test_dimensions to cut
+# down test output (which includes everything from the ImpalaTestDimension creating
+# confusing and complicated test output).
+TEST_DIMENSIONS_DATA = {
+ TLS_VER_10: {
+ # TLS 1.0 is not supported, thus there are no test ciphersuites which results in the
+ # tests asserting that all ciphersuites are rejected.
+ "openssl_ver": "-tls1",
+ "openssl_ciphersuites_arg": "-cipher",
+ "extended_master_secret": True,
+ "all_ciphersuites": OPENSSL_TLS_1_2_CIPHERSUITES,
+ "test_ciphersuites": [],
+ },
+ TLS_VER_11: {
+ # TLS 1.1 is not supported, thus there are no test ciphersuites which results in the
+ # tests asserting that all ciphersuites are rejected.
+ "openssl_ver": "-tls1_1",
+ "openssl_ciphersuites_arg": "-cipher",
+ "extended_master_secret": True,
+ "all_ciphersuites": OPENSSL_TLS_1_2_CIPHERSUITES,
+ "test_ciphersuites": [],
+ },
+ TLS_VER_12: {
+ "openssl_ver": "-tls1_2",
+ "openssl_ciphersuites_arg": "-cipher",
+ "extended_master_secret": True,
+ "all_ciphersuites": OPENSSL_TLS_1_2_CIPHERSUITES,
+ "test_ciphersuites": TLS_1_2_CIPHERSUITES_RSA + TLS_1_2_CIPHERSUITES_ECDSA,
+ },
+ TLS_VER_13: {
+ "openssl_ver": "-tls1_3",
+ "openssl_ciphersuites_arg": "-ciphersuites",
+ "extended_master_secret": False, # TLS 1.3 does not use extended master secrets.
+ "all_ciphersuites": OPENSSL_TLS_1_3_CIPHERSUITES,
+ "test_ciphersuites": TLS_1_3_CIPHERSUITES,
+ },
+}
+
+# List of arguments to configure on each cluster daemon to enable TLS on all ports using
+# the above certificates.
+TLS_LOCALHOST_ARGS = (
+ "--hostname=localhost "
+ "--webserver_interface=localhost "
+ "--ssl_cipher_list={0} "
+ "--tls_ciphersuites={1} "
+).format(
+ ":".join(TEST_DIMENSIONS_DATA[TLS_VER_12]["test_ciphersuites"]),
+ ":".join(TEST_DIMENSIONS_DATA[TLS_VER_13]["test_ciphersuites"]),
+)
+
+# Specific values for the RSA and ECDSA tests. Defined outside of test dimensions because
+# these values are used directly in the cluster startup args.
+RSA_CA_FILE = "{}/wildcardCA.pem".format(CERT_DIR)
+
+TLS_RSA_LOCALHOST_ARGS = (
+ "{localhost_arg} "
+ "--ssl_client_ca_certificate={ca_file} "
+ "--webserver_certificate_file={cert_dir}/localhost.pem "
+ "--webserver_private_key_file={cert_dir}/localhost.key "
+ "--ssl_server_certificate={cert_dir}/localhost.pem "
+ "--ssl_private_key={cert_dir}/localhost.key "
+).format(localhost_arg=TLS_LOCALHOST_ARGS, ca_file=RSA_CA_FILE, cert_dir=CERT_DIR)
+
+ECDSA_CA_FILE = "{}/wildcardCA-ecdsa.pem".format(CERT_DIR)
+
+TLS_ECDSA_LOCALHOST_ARGS = (
+ "{localhost_arg} "
+ "--ssl_client_ca_certificate={ca_file} "
+ "--webserver_certificate_file={cert_dir}/localhost-ecdsa.pem "
+ "--webserver_private_key_file={cert_dir}/localhost-ecdsa.key "
+ "--ssl_server_certificate={cert_dir}/localhost-ecdsa.pem "
+ "--ssl_private_key={cert_dir}/localhost-ecdsa.key "
+).format(localhost_arg=TLS_LOCALHOST_ARGS, ca_file=ECDSA_CA_FILE, cert_dir=CERT_DIR)
+
+
+class BaseTestServerTls(CustomClusterTestSuite):
+ """
+ Tests that the cluster properly accepts or rejects TLS connections based on the
+ configured TLS versions and ciphersuites and that extended master secrets are used
+ for TLS 1.2 connections.
+
+ Checks webserver and Thrift server ports.
+
+ The cluster is set up with a separate admissiond and both statestored and catalogd
+ running in HA mode to ensure all TLS-enabled cluster ports are asserted.
+
+ Uses the openssl s_client command to perform assertions because the Python 'ssl'
+ module changed drastically with Python 3.7 and thus doesn't provide a consistent
+ interface across Python versions.
+ """
+
+ @classmethod
+ def setup_class(cls):
+ super(BaseTestServerTls, cls).setup_class()
+
+ unsupported_ciphersuites = []
+
+ # Assert all test TLS 1.2 ciphersuites are supported by OS/OpenSSL.
+ for ciphersuite in TEST_DIMENSIONS_DATA[TLS_VER_12]["test_ciphersuites"]:
+ if ciphersuite not in OPENSSL_TLS_1_2_CIPHERSUITES:
+ unsupported_ciphersuites.append(ciphersuite)
+
+ # Assert all test TLS 1.3 ciphersuites are supported by OS/OpenSSL.
+ for ciphersuite in TEST_DIMENSIONS_DATA[TLS_VER_13]["test_ciphersuites"]:
+ if ciphersuite not in OPENSSL_TLS_1_3_CIPHERSUITES:
+ unsupported_ciphersuites.append(ciphersuite)
+
+ assert len(unsupported_ciphersuites) == 0, \
+ "Configured TLS ciphers are not supported by OS/OpenSSL: {}".format(
+ unsupported_ciphersuites)
+
+ @classmethod
+ def add_test_dimensions(cls):
+ cls.ImpalaTestMatrix.add_dimension(
+ ImpalaTestDimension('tls_version', "1.0", "1.1", TLS_VER_12, TLS_VER_13))
+
+ @classmethod
+ def ca_file(cls):
+ raise NotImplementedError("Subclasses must implement ca_file()")
+
+ def _ciphersuite_filter(self, vector): # noqa: U100
+ raise NotImplementedError("Subclasses must implement _ciphersuite_filter()")
+
+ def _get_dimension_data(self, vector):
+ return TEST_DIMENSIONS_DATA[vector.get_value("tls_version")]
+
+ def _get_tls_ver_str(self, vector):
+ return vector.get_value("tls_version")
+
+ def _ciphersuites_allowed(self, vector):
+ return sorted(ciph for ciph in self._get_dimension_data(vector)["test_ciphersuites"]
+ if self._ciphersuite_filter(vector)(ciph))
+
+ def _ciphersuites_notallowed(self, vector):
+ """
+ Builds a list of ciphersuites that are not allowed by the cluster configuration and
+ that should be rejected by the cluster. This is done by taking all OS/OpenSSL
+ ciphersuites and removing the test ciphersuites (that should be allowed).
+ """
+ ret = []
+ for ciph in set(self._get_dimension_data(vector)["all_ciphersuites"]) \
+ - set(self._get_dimension_data(vector)["test_ciphersuites"]):
+ if self._ciphersuite_filter(vector)(ciph):
+ ret.append(ciph)
+ return sorted(ret)
+
+ def _openssl_handshake(self, vector, port, ciphersuites):
+ """
+ Uses the openssl s_client command to attempt a TLS handshake on the given port with
+ the specified TLS version and ciphersuites. Returns a tuple of (success, output)
+ where success is a boolean indicating whether the handshake succeeded and output is
+ the combined stdout/stderr output from the openssl command as a string.
+ """
+ cmd = ["s_client", "-connect", "localhost:{}".format(port), "-servername",
+ "localhost", "-CAfile", self.ca_file()]
+
+ # Add the correct TLS version and ciphersuite flags based on the tls_version argument.
+ cmd.extend([
+ self._get_dimension_data(vector)["openssl_ver"],
+ self._get_dimension_data(vector)["openssl_ciphersuites_arg"],
+ ciphersuites,
+ ])
+
+ rc, out = run_openssl_cmd(cmd)
+ return rc == 0, out
+
+ @pytest.mark.execute_serially
+ @pytest.mark.skipif(SKIP_SSL_MSG != "", reason=SKIP_SSL_MSG)
+ def test_webserver_thrift_ciphersuites_allowed(self, vector):
+ """
+ Asserts the cluster accepts TLS connections with the configured ciphersuites and
+ TLS version on webserver and Thrift server ports.
+ """
+ # Skip test if there are no allowed ciphersuites for the TLS version.
+ if len(self._ciphersuites_allowed(vector)) == 0:
+ pytest.skip("No configured ciphersuites for TLS version {}"
+ .format(self._get_tls_ver_str(vector)))
+
+ # Loop through all ports attempting a TLS handshake with each TLS ciphersuite.
+ for port in TLS_PORTS:
+ # Assert a successful TLS handshake for all TLS 1.2 ciphersuites configured on the
+ # cluster.
+ for cipher in self._ciphersuites_allowed(vector):
+ success, out = self._openssl_handshake(vector, port, cipher)
+
+ assert success, "Expected TLS {} handshake to succeed on 'localhost:{}' with " \
+ "cipher '{}', output:\n{}" \
+ .format(self._get_tls_ver_str(vector), port, cipher, out)
+
+ assert \
+ "Ciphersuite: {}".format(cipher) in out \
+ or "Cipher is {}".format(cipher) in out, \
+ "Expected TLS {} cipher '{}' to be negotiated on 'localhost:{}', " \
+ "output:\n{}".format(self._get_tls_ver_str(vector), cipher, port, out)
+
+ if self._get_dimension_data(vector)["extended_master_secret"]:
+ assert "Extended master secret: yes" in out, \
+ "Expected TLS {} on 'localhost:{}' with cipher '{}' to use extended " \
+ "master secret, output:\n{}".format(self._get_tls_ver_str(vector), port,
+ cipher, out)
+
+ @pytest.mark.execute_serially
+ @pytest.mark.skipif(SKIP_SSL_MSG != "", reason=SKIP_SSL_MSG)
+ def test_webserver_thrift_ciphersuites_notallowed(self, vector):
+ """
+ Asserts the cluster denies TLS connections with ciphersuites that are not
+ configured on webserver and Thrift server ports.
+ """
+ # Loop through all ports attempting a TLS handshake with each disallowed cipher.
+ for port in TLS_PORTS:
+ for cipher in self._ciphersuites_notallowed(vector):
+ success, out = self._openssl_handshake(vector, port, cipher)
+ assert not success, "Unexpected TLS {} cipher '{}' accepted on 'localhost:{}', " \
+ "output:\n{}".format(self._get_tls_ver_str(vector), cipher, port, out)
+
+
+@CustomClusterTestSuite.with_args(cluster_size=1,
+ impalad_args=TLS_RSA_LOCALHOST_ARGS,
+ statestored_args=TLS_RSA_LOCALHOST_ARGS,
+ catalogd_args=TLS_RSA_LOCALHOST_ARGS,
+ admissiond_args=TLS_RSA_LOCALHOST_ARGS,
+ start_args="--enable_admission_service "
+ "--enable_statestored_ha "
+ "--enable_catalogd_ha")
+class TestServerTlsRSA(BaseTestServerTls):
+ """
+ Spins up a cluster configured to use TLS with RSA certificates and runs webserver
+ and Thrift server TLS tests.
+ """
+
+ @classmethod
+ def ca_file(cls):
+ return RSA_CA_FILE
+
+ def _ciphersuite_filter(self, vector):
+ """
+ Filters TLS ciphersuites to include RSA ciphersuites.
+ """
+ return lambda ciph: self._get_tls_ver_str(vector) == TLS_VER_13 or "ECDSA" not in ciph
+
+
+@CustomClusterTestSuite.with_args(cluster_size=1,
+ impalad_args=TLS_ECDSA_LOCALHOST_ARGS,
+ statestored_args=TLS_ECDSA_LOCALHOST_ARGS,
+ catalogd_args=TLS_ECDSA_LOCALHOST_ARGS,
+ admissiond_args=TLS_ECDSA_LOCALHOST_ARGS,
+ start_args="--enable_admission_service "
+ "--enable_statestored_ha "
+ "--enable_catalogd_ha")
+class TestServerTlsECDSA(BaseTestServerTls):
+ """
+ Spins up a cluster configured to use TLS with ECDSA certificates and runs webserver
+ and Thrift server TLS tests.
+ """
+
+ @classmethod
+ def ca_file(cls):
+ return ECDSA_CA_FILE
+
+ def _ciphersuite_filter(self, vector):
+ """
+ Filters TLS ciphersuites to only include ECDSA ciphersuites.
+ """
+ return lambda ciph: self._get_tls_ver_str(vector) == TLS_VER_13 or "ECDSA" in ciph
(impala) 02/03: IMPALA-14763: Prevent admissiond OOM during request decompression
Posted by ja...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
asf-gitbox-commits pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 1a73414d729e4ab8e519444aac7bfd6b3019e60a
Author: Yida Wu <yi...@cloudera.com>
AuthorDate: Thu Feb 26 23:08:12 2026 -0800
IMPALA-14763: Prevent admissiond OOM during request decompression
When admissiond is close to its memory limit and a very large
queued query is dequeued, decompression of this compressed
request can push memory usage over the limit and cause an OOM.
Previously in IMPALA-14493, memory checks TCMalloc's BYTES_IN_USE
to provide memory safeguard on Submission for uncompressed
requests, but after IMPALA-14661, we need to consider the
decompression cases. This patch adds memory safeguard for
compressed requests, mainly the decompression will happen on
Submission or Dequeue.
We put all the rejection logic into a static function
RejectForAdmissionServiceMemory(), and introduce a new memory
tracker, pending_decompression_mem_tracker, to track the total
uncompressed size of pending compressed requests.
RejectForAdmissionServiceMemory() compares the current
tcmalloc bytes-in-use plus the additional memory to reserve
against the process memory limit.
For compressed requests, we first add the request’s uncompressed
size to pending_decompression_mem_tracker, then pass the total
pending uncompressed size as the additional reserved memory to
RejectForAdmissionServiceMemory(), ensuring thread safety.
For uncompressed requests, the additional memory is zero.
If the check fails, RejectForAdmissionServiceMemory() returns
an error and admissiond rejects the query.
Additionally, to prevent early decompression for queued
compressed requests when GetQueryStatus() is called, we now
cache the TQueryOptions inside AdmissionExecRequestCompressed
during the first decompression. WaitOnQueued() uses these
cached options for the AC_AFTER_ADMISSION_OUTCOME debug
action instead of redecompressing the whole request.
Testing:
Added a new test to check compressed requests being rejected
on Submission. Manually verified that the safeguard also works
at Dequeue, an automated test for the Dequeue case was a bit
flaky to include.
Passed exhaustive test test_admission_controller.py.
Change-Id: I196455f445f0644d89467a23b4ec1f64f184f2db
Reviewed-on: http://gerrit.cloudera.org:8080/24055
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/scheduling/admission-controller.cc | 141 ++++++++++++++--------
be/src/scheduling/admission-controller.h | 28 +++++
tests/custom_cluster/test_admission_controller.py | 13 ++
3 files changed, 130 insertions(+), 52 deletions(-)
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index 5fd0ac5f4..7a233593c 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -353,6 +353,36 @@ static inline bool ParsePoolTopicKey(
return true;
}
+// Checks a query will exceed the admission service process memory limit.
+// If the limit is exceeded, returns true and the 'rejection_reason'.
+// 'additional_mem' allows to preserve the memory needed like from decompression.
+static bool RejectForAdmissionServiceMemory(
+ int64_t additional_mem, string* rejection_reason) {
+ // Only works when in a admissiond process.
+ if (AdmissiondEnv::GetInstance() == nullptr) return false;
+ DCHECK(rejection_reason != nullptr);
+
+ int64_t mem_limit = AdmissiondEnv::GetInstance()->admission_service_mem_limit();
+ if (mem_limit <= 0) return false;
+
+#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
+ DCHECK(SanitizerMallocMetric::BYTES_ALLOCATED != nullptr);
+ int64_t bytes_inuse = SanitizerMallocMetric::BYTES_ALLOCATED->GetValue();
+#else
+ DCHECK(TcmallocMetric::BYTES_IN_USE != nullptr);
+ int64_t bytes_inuse = TcmallocMetric::BYTES_IN_USE->GetValue();
+#endif
+
+ if (bytes_inuse + additional_mem > mem_limit) {
+ LOG(INFO) << "Rejected admission with additional_mem: " << additional_mem
+ << ", bytes_inuse: " << bytes_inuse << ", mem_limit: " << mem_limit;
+ *rejection_reason =
+ Substitute(REASON_EXCEED_MEMORY_LIMIT, bytes_inuse + additional_mem, mem_limit);
+ return true;
+ }
+ return false;
+}
+
Status AdmissionExecRequestCompressed::GetQueryExecRequest(
const TQueryExecRequest** out_req) const {
lock_guard<std::mutex> l(lock_);
@@ -362,23 +392,36 @@ Status AdmissionExecRequestCompressed::GetQueryExecRequest(
}
DCHECK(req_ != nullptr);
+ int64_t uncomp_size = req_->uncompressed_size;
+ MemTracker* tracker = admission_controller_->pending_decompression_mem_tracker_.get();
+ DCHECK(tracker != nullptr);
+ tracker->Consume(uncomp_size);
+ const auto tracker_releaser =
+ MakeScopeExitTrigger([&]() { tracker->Release(uncomp_size); });
+ string rejection_reason;
+ if (RejectForAdmissionServiceMemory(tracker->consumption(), &rejection_reason)) {
+ // It is okay we set the pool's name to N/A as the pool is not related.
+ return Status::Expected(
+ ErrorMsg(TErrorCode::ADMISSION_REJECTED, "N/A", rejection_reason));
+ }
+
unique_ptr<TQueryExecRequest> new_req = std::make_unique<TQueryExecRequest>();
RETURN_IF_ERROR(DecompressThrift(*req_, new_req.get()));
+ DCHECK(new_req != nullptr);
int64_t comp_size = req_->compressed_data.size();
- int64_t uncomp_size = req_->uncompressed_size;
float ratio = (comp_size > 0) ? (static_cast<float>(uncomp_size) / comp_size) : 0.0f;
-
LOG(INFO) << "Decompress TQueryExecRequest for query "
<< PrintId(new_req->query_ctx.query_id) << ": Compressed size=" << comp_size
- << " B"
- << ", Uncompressed size=" << uncomp_size << " B"
- << ", Ratio=" << ratio << "x";
+ << " B, Uncompressed size=" << uncomp_size << " B, Ratio=" << ratio << "x";
// Update the compression stats only once.
- if (first_decompress_ && admission_controller_ != nullptr) {
- admission_controller_->UpdateAdmissionRequestCompressionStats(
- comp_size, uncomp_size, ratio);
+ if (first_decompress_) {
+ cached_query_options_ = new_req->query_ctx.client_request.query_options;
+ if (admission_controller_ != nullptr) {
+ admission_controller_->UpdateAdmissionRequestCompressionStats(
+ comp_size, uncomp_size, ratio);
+ }
}
// Store the object in our cache.
@@ -807,6 +850,11 @@ Status AdmissionController::Init() {
if (!status.ok()) {
status.AddDetail("AdmissionController failed to register request queue topic");
}
+ if (AdmissiondEnv::GetInstance() != nullptr) {
+ DCHECK(AdmissiondEnv::GetInstance()->process_mem_tracker() != nullptr);
+ pending_decompression_mem_tracker_.reset(new MemTracker(-1, "Pending Decompression",
+ AdmissiondEnv::GetInstance()->process_mem_tracker()));
+ }
return status;
}
@@ -1718,31 +1766,19 @@ Status AdmissionController::SubmitForAdmission(const AdmissionRequest& request,
return Status::Expected(rejected_msg);
}
- if (AdmissiondEnv::GetInstance() != nullptr) {
- // See RegisterMemoryMetrics() in memory-metrics.cc for where the relevant metrics
- // are registered.
-#if defined(ADDRESS_SANITIZER) || defined(THREAD_SANITIZER)
- DCHECK(SanitizerMallocMetric::BYTES_ALLOCATED != nullptr);
- int64_t bytes_inuse = SanitizerMallocMetric::BYTES_ALLOCATED->GetValue();
-#else
- DCHECK(TcmallocMetric::BYTES_IN_USE != nullptr);
- int64_t bytes_inuse = TcmallocMetric::BYTES_IN_USE->GetValue();
-#endif
- if (!is_trivial && AdmissiondEnv::GetInstance()->admission_service_mem_limit() > 0
- && bytes_inuse > AdmissiondEnv::GetInstance()->admission_service_mem_limit()) {
- queue_node->not_admitted_reason = Substitute(REASON_EXCEED_MEMORY_LIMIT,
- bytes_inuse, AdmissiondEnv::GetInstance()->admission_service_mem_limit());
- request.summary_profile->AddInfoString(
- PROFILE_INFO_KEY_ADMISSION_RESULT, PROFILE_INFO_VAL_REJECTED);
- stats->metrics()->total_rejected->Increment(1);
- const ErrorMsg& rejected_msg = ErrorMsg(TErrorCode::ADMISSION_REJECTED,
- queue_node->pool_name, queue_node->not_admitted_reason);
- VLOG_QUERY << "query_id=" << PrintId(request.query_id) << " "
- << rejected_msg.msg();
- return Status::Expected(rejected_msg);
- }
+ // Check if the admissiond is at capacity and reject immediately to prevent OOM.
+ // Only needed when in admissiond mode, RejectForAdmissionServiceMemory() will return
+ // false immediately if not in admissiond process.
+ if (!is_trivial
+ && RejectForAdmissionServiceMemory(0, &queue_node->not_admitted_reason)) {
+ request.summary_profile->AddInfoString(
+ PROFILE_INFO_KEY_ADMISSION_RESULT, PROFILE_INFO_VAL_REJECTED);
+ stats->metrics()->total_rejected->Increment(1);
+ const ErrorMsg& rejected_msg = ErrorMsg(TErrorCode::ADMISSION_REJECTED,
+ queue_node->pool_name, queue_node->not_admitted_reason);
+ VLOG_QUERY << "query_id=" << PrintId(request.query_id) << " " << rejected_msg.msg();
+ return Status::Expected(rejected_msg);
}
-
string user;
const TQueryExecRequest* exec_req_queue_node = nullptr;
RETURN_IF_ERROR(
@@ -1861,12 +1897,8 @@ Status AdmissionController::WaitOnQueued(const UniqueIdPB& query_id,
queue_nodes_.erase(query_id);
});
- const TQueryExecRequest* exec_req_queue_node = nullptr;
- RETURN_IF_ERROR(
- queue_node->admission_request.request.GetQueryExecRequest(&exec_req_queue_node));
- DCHECK(exec_req_queue_node != nullptr);
// Disallow the FAIL action here. It would leave the queue in an inconsistent state.
- DebugActionNoFail(exec_req_queue_node->query_ctx.client_request.query_options,
+ DebugActionNoFail(queue_node->admission_request.request.GetQueryOptions(),
"AC_AFTER_ADMISSION_OUTCOME");
{
@@ -2686,11 +2718,27 @@ void AdmissionController::TryDequeue() {
bool coordinator_resource_limited = false;
bool is_trivial = false;
- bool is_rejected = !is_cancelled
- && !FindGroupToAdmitOrReject(membership_snapshot, pool_config,
+ bool is_rejected = false;
+ const UniqueIdPB& query_id = queue_node->admission_request.query_id;
+ const TQueryExecRequest* exec_req = nullptr;
+ if (!is_cancelled) {
+ Status status =
+ queue_node->admission_request.request.GetQueryExecRequest(&exec_req);
+ if (!status.ok()) {
+ LOG(WARNING) << "Failed to get query execution request for dequeued query "
+ << PrintId(query_id) << ": " << status.GetDetail();
+ // Fall to rejection handling.
+ queue_node->not_admitted_reason =
+ Substitute("Failed to get query execution request: $0", status.GetDetail());
+ is_rejected = true;
+ }
+ if (!is_rejected) {
+ is_rejected = !FindGroupToAdmitOrReject(membership_snapshot, pool_config,
queue_node->root_cfg,
/* admit_from_queue=*/true, stats, queue_node, coordinator_resource_limited,
&is_trivial);
+ }
+ }
if (!is_cancelled && !is_rejected
&& queue_node->admitted_schedule.get() == nullptr) {
@@ -2712,18 +2760,6 @@ void AdmissionController::TryDequeue() {
--max_to_dequeue;
VLOG(3) << "Dequeueing from stats for pool " << pool_name;
stats->Dequeue(false);
- const UniqueIdPB& query_id = queue_node->admission_request.query_id;
- const TQueryExecRequest* exec_req = nullptr;
- Status status =
- queue_node->admission_request.request.GetQueryExecRequest(&exec_req);
- if (!status.ok()) {
- LOG(WARNING) << "Failed to get query execution request for dequeued query "
- << PrintId(query_id) << ": " << status.GetDetail();
- // Fall to rejection handling.
- queue_node->not_admitted_reason =
- Substitute("Failed to get query execution request: $0", status.GetDetail());
- is_rejected = true;
- }
if (is_rejected) {
AdmissionOutcome outcome =
queue_node->admit_outcome->Set(AdmissionOutcome::REJECTED);
@@ -2762,8 +2798,9 @@ void AdmissionController::TryDequeue() {
DCHECK(!is_cancelled);
DCHECK(!is_rejected);
DCHECK(queue_node->admitted_schedule != nullptr);
+ DCHECK(exec_req != nullptr);
string user;
- status = GetEffectiveShortUser(exec_req->query_ctx.session, &user);
+ Status status = GetEffectiveShortUser(exec_req->query_ctx.session, &user);
DCHECK_OK(status); // Should never happen because user name was checked at query
// entry.
AdmitQuery(queue_node, user, true /* was_queued */, is_trivial);
diff --git a/be/src/scheduling/admission-controller.h b/be/src/scheduling/admission-controller.h
index 8604452c1..5077a304d 100644
--- a/be/src/scheduling/admission-controller.h
+++ b/be/src/scheduling/admission-controller.h
@@ -68,10 +68,17 @@ class AdmissionExecRequest {
/// Avoids Status checks, DCHECKs if called on compressed requests.
virtual const TQueryExecRequest* request() const = 0;
+ // Returns the query options. For compressed requests, this returns a cached
+ // copy to help unnecessary decompression.
+ virtual const TQueryOptions& GetQueryOptions() const = 0;
+
/// For compressed only, clears the cached decompressed object to save memory.
/// Returns true if cache is empty, returns false if no compressed data or non-empty
/// after operation.
virtual bool ClearDecompressedCache() const = 0;
+
+ /// Returns if the request is compressed
+ virtual bool IsCompressed() const = 0;
};
class AdmissionExecRequestUncompressed : public AdmissionExecRequest {
@@ -90,11 +97,18 @@ class AdmissionExecRequestUncompressed : public AdmissionExecRequest {
return req_;
}
+ const TQueryOptions& GetQueryOptions() const override {
+ DCHECK(req_ != nullptr);
+ return req_->query_ctx.client_request.query_options;
+ }
+
bool ClearDecompressedCache() const override {
// we don't have a compressed data, so return false.
return false;
}
+ bool IsCompressed() const override { return false; }
+
private:
const TQueryExecRequest* req_;
};
@@ -117,6 +131,12 @@ class AdmissionExecRequestCompressed : public AdmissionExecRequest {
return nullptr;
}
+ const TQueryOptions& GetQueryOptions() const override {
+ std::lock_guard<std::mutex> l(lock_);
+ DCHECK(!first_decompress_) << "GetQueryOptions() called before first decompression";
+ return cached_query_options_;
+ }
+
bool ClearDecompressedCache() const override {
std::lock_guard<std::mutex> l(lock_);
if (decompressed_req_) {
@@ -127,12 +147,15 @@ class AdmissionExecRequestCompressed : public AdmissionExecRequest {
return true;
}
+ bool IsCompressed() const override { return true; }
+
private:
const TQueryExecRequestCompressed* req_;
const AdmissionController* admission_controller_;
mutable std::mutex lock_;
mutable std::unique_ptr<TQueryExecRequest> decompressed_req_;
mutable bool first_decompress_ = true;
+ mutable TQueryOptions cached_query_options_;
};
/// The AdmissionController is used to throttle requests (e.g. queries, DML) based
@@ -579,6 +602,7 @@ class AdmissionController {
private:
class PoolStats;
friend class PoolStats;
+ friend class AdmissionExecRequestCompressed;
/// Pointer to the cluster membership manager. Not owned by the AdmissionController.
ClusterMembershipMgr* cluster_membership_mgr_;
@@ -629,6 +653,10 @@ class AdmissionController {
/// executor groups).
IntCounter* total_dequeue_failed_coordinator_limited_ = nullptr;
+ /// MemTracker tracking the size of uncompressed requests that have been approved for
+ /// decompression, but haven't been allocated by tcmalloc yet.
+ mutable std::unique_ptr<MemTracker> pending_decompression_mem_tracker_;
+
/// Histograms for tracking the compressed/uncompressed sizes and compression ratios
/// of the admission requests of TQueryExecRequest.
mutable HistogramMetric* compressed_size_metric_;
diff --git a/tests/custom_cluster/test_admission_controller.py b/tests/custom_cluster/test_admission_controller.py
index 923edb23b..bc79258eb 100644
--- a/tests/custom_cluster/test_admission_controller.py
+++ b/tests/custom_cluster/test_admission_controller.py
@@ -2598,6 +2598,19 @@ class TestAdmissionControllerWithACServiceCompress(TestAdmissionController):
self.enable_admission_compress(method, 1)
super(TestAdmissionControllerWithACServiceCompress, self).setup_method(method)
+ @SkipIfNotHdfsMinicluster.tuned_for_minicluster
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--vmodule admission-controller=3 --mem_limit=20MB ")
+ def test_admission_service_safeguard_large_compressed_query(self):
+ """Test compressed request being rejected at Submission due to memory
+ pressure."""
+ large_query = " union all ".join(["select * from functional.alltypes"] * 30)
+ handle = self.client.execute_async(large_query)
+ self.client.wait_for_impala_state(handle, ERROR, 20)
+ profile = self.client.get_runtime_profile(handle)
+ # Only compressed requests rejected will end up with pool N/A.
+ assert "pool N/A: Admission rejected due to memory pressure" in profile
+
class TestAdmissionControllerStress(TestAdmissionControllerBase):
"""Submits a number of queries (parameterized) with some delay between submissions