You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by li...@apache.org on 2026/07/23 09:23:29 UTC
(iceberg-rust) branch main updated: feat(encryption): support AES-256 keys for reading Parquet data files (#2878)
This is an automated email from the ASF dual-hosted git repository.
blackmwk pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
The following commit(s) were added to refs/heads/main by this push:
new 4532da80f feat(encryption): support AES-256 keys for reading Parquet data files (#2878)
4532da80f is described below
commit 4532da80f3930cdbda1cf2effe05b50165da9875
Author: Matt Butrovich <mb...@users.noreply.github.com>
AuthorDate: Thu Jul 23 05:23:23 2026 -0400
feat(encryption): support AES-256 keys for reading Parquet data files (#2878)
## Which issue does this PR close?
- Closes #.
Depends on the arrow-rs 58.4.0 release tracked in apache/arrow-rs#10349.
## What changes are included in this PR?
- Bump `arrow`/`parquet` deps from `58` to `58.4`. arrow-rs 58.4.0 adds
AES-256 for Parquet modular encryption (58.3.0 was AES-128 only), landed
via apache/arrow-rs#9203.
- Add an AES-256 encrypted-Parquet read test. Refactored the existing
128-bit test into a shared `assert_encrypted_parquet_roundtrip(key)`
helper with `_aes_128` and `_aes_256` callers.
Note: 192-bit is not supported for data files. arrow-rs uses `ring`,
which has no AES-192-GCM. Can revisit later.
## Are these changes tested?
Yes. Unit test `test_read_encrypted_parquet_aes_256` writes and reads
back a 256-bit encrypted Parquet file. It fails on arrow-rs 58.3.0
(`ring` rejects the 32-byte key) and passes on 58.4.0.
---
Cargo.lock | 74 ++++++++++++++---------------
Cargo.toml | 20 ++++----
crates/iceberg/src/arrow/reader/pipeline.rs | 17 +++++--
3 files changed, 61 insertions(+), 50 deletions(-)
diff --git a/Cargo.lock b/Cargo.lock
index ead0b2e13..87b228a87 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -227,9 +227,9 @@ checksum = "d3fb67a6e08acf24fdeccbac2cb6ac4305825bd1f117462e0e6f2f193345ad56"
[[package]]
name = "arrow"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "378530e55cd479eda3c14eb345310799717e6f76d0c332041e8487022166b471"
+checksum = "6cfdd0833e32a9874d2b55089333ad310c0be208aafa277385ce2461dec90be3"
dependencies = [
"arrow-arith",
"arrow-array",
@@ -249,9 +249,9 @@ dependencies = [
[[package]]
name = "arrow-arith"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a0ab212d2c1886e802f51c5212d78ebbcbb0bec980fff9dadc1eb8d45cd0b738"
+checksum = "0a41203398f0eaa6f7ec8e62c0da742a21abf282c148fc157f6c35c90e29981a"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -263,9 +263,9 @@ dependencies = [
[[package]]
name = "arrow-array"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfd33d3e92f207444098c75b42de99d329562be0cf686b307b097cc52b4e999e"
+checksum = "ae33dad492b7df00a217563a7b0ef2874df68a0deea1b1a3acf628152f7f7a69"
dependencies = [
"ahash",
"arrow-buffer",
@@ -306,9 +306,9 @@ dependencies = [
[[package]]
name = "arrow-buffer"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0c6cd424c2693bcdbc150d843dc9d4d137dd2de4782ce6df491ad11a3a0416c0"
+checksum = "b9552f96391c005e6ab449fa941420935e7e062489b12b8b1b08879b2163f5b5"
dependencies = [
"bytes",
"half",
@@ -318,9 +318,9 @@ dependencies = [
[[package]]
name = "arrow-cast"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4c5aefb56a2c02e9e2b30746241058b85f8983f0fcff2ba0c6d09006e1cded7f"
+checksum = "3a8a327c9649f30d8406995f27642b68df354713cca3baaaf100f076f18d5f34"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -340,9 +340,9 @@ dependencies = [
[[package]]
name = "arrow-csv"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e94e8cf7e517657a52b91ea1263acf38c4ca62a84655d72458a3359b12ab97de"
+checksum = "af0dd6d90d1955e9f9a014c1e563ee8aeffc21909085d25623e1da44d96eca26"
dependencies = [
"arrow-array",
"arrow-cast",
@@ -355,9 +355,9 @@ dependencies = [
[[package]]
name = "arrow-data"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3c88210023a2bfee1896af366309a3028fc3bcbd6515fa29a7990ee1baa08ee0"
+checksum = "2b24852db04738907e06c04ea61e42fe7fda962a34513022dc0d0e754fb7976b"
dependencies = [
"arrow-buffer",
"arrow-schema",
@@ -368,9 +368,9 @@ dependencies = [
[[package]]
name = "arrow-ipc"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "238438f0834483703d88896db6fe5a7138b2230debc31b34c0336c2996e3c64f"
+checksum = "29a908a11fcfb3fb2f6730f4ac15e367bc644e419155e96238f68cf3adde572b"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -384,9 +384,9 @@ dependencies = [
[[package]]
name = "arrow-json"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "205ca2119e6d679d5c133c6f30e68f027738d95ed948cf77677ea69c7800036b"
+checksum = "b8a96aed3931c076adee39ec2a40d8219fc7f09e79bcdaca1df16272993e1e14"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -409,9 +409,9 @@ dependencies = [
[[package]]
name = "arrow-ord"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1bffd8fd2579286a5d63bac898159873e5094a79009940bcb42bbfce4f19f1d0"
+checksum = "63a083ec750f5c043f02946b4baf05fcdbb55f4560a3277055caca5cc99f3eb0"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -422,9 +422,9 @@ dependencies = [
[[package]]
name = "arrow-pyarrow"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d29abdf672a81c1aeb57fd2661457f9918964d49aed0e9f18932535f2a9e49ce"
+checksum = "3ffb9be5a873590f825aef50df20e0f8dff5fd42a77058e28bbe7bd44bb53dec"
dependencies = [
"arrow-array",
"arrow-data",
@@ -434,9 +434,9 @@ dependencies = [
[[package]]
name = "arrow-row"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bab5994731204603c73ba69267616c50f80780774c6bb0476f1f830625115e0c"
+checksum = "514ba0ef0d4c5896202dae736251ce415abb43a950bed570fb7981b8716c0e4c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -447,9 +447,9 @@ dependencies = [
[[package]]
name = "arrow-schema"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f633dbfdf39c039ada1bf9e34c694816eb71fbb7dc78f613993b7245e078a1ed"
+checksum = "21ca356ad6425cecb6eb7b28e4f659f1ee7880fbb1a16127de7dd62901efee9e"
dependencies = [
"bitflags 2.13.0",
"serde_core",
@@ -458,9 +458,9 @@ dependencies = [
[[package]]
name = "arrow-select"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8cd065c54172ac787cf3f2f8d4107e0d3fdc26edba76fdf4f4cc170258942222"
+checksum = "c58da39eb3d8350ad4a549e5c2bc49284dac554016c69829310350f1731b0aad"
dependencies = [
"ahash",
"arrow-array",
@@ -472,9 +472,9 @@ dependencies = [
[[package]]
name = "arrow-string"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "29dd7cda3ab9692f43a2e4acc444d760cc17b12bb6d8232ddf64e9bab7c06b42"
+checksum = "b6789b388467525e3271326b6b4915666ecfdf5142aef09779445c954b67543c"
dependencies = [
"arrow-array",
"arrow-buffer",
@@ -1483,7 +1483,7 @@ version = "3.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "faf9468729b8cbcea668e36183cb69d317348c2e08e994829fb56ebfdfbaac34"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
]
[[package]]
@@ -3718,7 +3718,7 @@ dependencies = [
"libc",
"percent-encoding",
"pin-project-lite",
- "socket2 0.5.10",
+ "socket2 0.6.4",
"system-configuration",
"tokio",
"tower-service",
@@ -5435,9 +5435,9 @@ dependencies = [
[[package]]
name = "parquet"
-version = "58.3.0"
+version = "58.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5dafa7d01085b62a47dd0c1829550a0a36710ea9c4fe358a05a85477cec8a908"
+checksum = "d298093b2dec60289dce0684c986d0f7679e9dd15771c2c65406e1aaf604a704"
dependencies = [
"ahash",
"arrow-array",
@@ -5923,7 +5923,7 @@ dependencies = [
"quinn-udp",
"rustc-hash",
"rustls",
- "socket2 0.5.10",
+ "socket2 0.6.4",
"thiserror 2.0.18",
"tokio",
"tracing",
@@ -5962,7 +5962,7 @@ dependencies = [
"cfg_aliases",
"libc",
"once_cell",
- "socket2 0.5.10",
+ "socket2 0.6.4",
"tracing",
"windows-sys 0.61.2",
]
@@ -8541,7 +8541,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
- "windows-sys 0.48.0",
+ "windows-sys 0.61.2",
]
[[package]]
diff --git a/Cargo.toml b/Cargo.toml
index c7315a6d1..a394ec107 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -49,15 +49,15 @@ aes-gcm = "0.10"
anyhow = "1.0.72"
apache-avro = { version = "0.21", features = ["snappy", "zstandard"] }
array-init = "2"
-arrow = "58"
-arrow-arith = "58"
-arrow-array = "58"
-arrow-buffer = "58"
-arrow-cast = "58"
-arrow-ord = "58"
-arrow-schema = "58"
-arrow-select = "58"
-arrow-string = "58"
+arrow = "58.4"
+arrow-arith = "58.4"
+arrow-array = "58.4"
+arrow-buffer = "58.4"
+arrow-cast = "58.4"
+arrow-ord = "58.4"
+arrow-schema = "58.4"
+arrow-select = "58.4"
+arrow-string = "58.4"
as-any = "0.3.2"
async-trait = "0.1.89"
aws-config = "1.8.7"
@@ -121,7 +121,7 @@ murmur3 = "0.5.2"
once_cell = "1.20"
opendal = "0.57"
ordered-float = "4"
-parquet = "58"
+parquet = "58.4"
pilota = "0.11.10"
pretty_assertions = "1.4"
pyo3 = "0.28"
diff --git a/crates/iceberg/src/arrow/reader/pipeline.rs b/crates/iceberg/src/arrow/reader/pipeline.rs
index ccc7679ed..579f04726 100644
--- a/crates/iceberg/src/arrow/reader/pipeline.rs
+++ b/crates/iceberg/src/arrow/reader/pipeline.rs
@@ -737,9 +737,10 @@ mod tests {
}
}
- #[tokio::test]
- async fn test_read_encrypted_parquet() {
- let encryption_key = b"0123456789abcdef";
+ /// Writes a single-column Parquet file encrypted with `encryption_key`, then reads it
+ /// back through `ArrowReader` and asserts the round-tripped values. The key length
+ /// selects the AES-GCM variant in arrow-rs (16 -> AES-128, 32 -> AES-256).
+ async fn assert_encrypted_parquet_roundtrip(encryption_key: &[u8]) {
let aad_prefix = b"aad_prefix";
let schema = Arc::new(
@@ -807,6 +808,16 @@ mod tests {
assert_eq!(ids.values(), &[10, 20, 30]);
}
+ #[tokio::test]
+ async fn test_read_encrypted_parquet_aes_128() {
+ assert_encrypted_parquet_roundtrip(b"0123456789abcdef").await;
+ }
+
+ #[tokio::test]
+ async fn test_read_encrypted_parquet_aes_256() {
+ assert_encrypted_parquet_roundtrip(b"0123456789abcdef0123456789abcdef").await;
+ }
+
#[tokio::test]
async fn test_read_encrypted_parquet_without_key_metadata_fails() {
let encryption_key = b"0123456789abcdef";