You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by gi...@apache.org on 2026/09/15 15:15:17 UTC
(iceberg-rust) 01/01: feat(python) add standard key metadata python bindings (#3206)
This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/main/pr-3206-aa5ff7e004b01137010923aaac640532d214f94a
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git
commit 930c10298e43daeb7316b20b83049e9f724ae58c
Author: Xander <za...@googlemail.com>
AuthorDate: Tue Sep 15 15:14:51 2026 +0000
feat(python) add standard key metadata python bindings (#3206)
* add standard ket metadata python bindings
* fix
* fix: clarify unsupported key metadata version error
Report the received and supported key metadata versions, matching the wording expected by PyIceberg. Add Rust and Python regression coverage.
Co-authored-by: Copilot App <22...@users.noreply.github.com>
* named struct
* improve tests
* tests
---------
Co-authored-by: Kevin Liu <ke...@gmail.com>
Co-authored-by: Copilot App <22...@users.noreply.github.com>
---
bindings/python/src/encryption.rs | 121 ++++++++++++++++++++++++++
bindings/python/src/lib.rs | 2 +
bindings/python/tests/test_encryption.py | 117 +++++++++++++++++++++++++
crates/iceberg/src/encryption/key_metadata.rs | 37 +++++++-
4 files changed, 276 insertions(+), 1 deletion(-)
diff --git a/bindings/python/src/encryption.rs b/bindings/python/src/encryption.rs
new file mode 100644
index 000000000..c0b14875b
--- /dev/null
+++ b/bindings/python/src/encryption.rs
@@ -0,0 +1,121 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use iceberg::encryption::StandardKeyMetadata;
+use pyo3::prelude::*;
+use pyo3::types::PyBytes;
+
+use crate::error::to_py_err;
+
+/// The encryption key, AAD prefix and file length needed to decrypt a single file.
+#[pyclass(
+ frozen,
+ name = "StandardKeyMetadata",
+ module = "pyiceberg_core.encryption"
+)]
+pub struct PyStandardKeyMetadata {
+ // Python objects, so attribute access increfs rather than copying the key on every read.
+ encryption_key: Py<PyBytes>,
+ aad_prefix: Option<Py<PyBytes>>,
+ file_length: Option<u64>,
+}
+
+#[pymethods]
+impl PyStandardKeyMetadata {
+ #[getter]
+ fn encryption_key(&self, py: Python<'_>) -> Py<PyBytes> {
+ self.encryption_key.clone_ref(py)
+ }
+
+ #[getter]
+ fn aad_prefix(&self, py: Python<'_>) -> Option<Py<PyBytes>> {
+ self.aad_prefix.as_ref().map(|prefix| prefix.clone_ref(py))
+ }
+
+ #[getter]
+ fn file_length(&self) -> Option<u64> {
+ self.file_length
+ }
+
+ /// Redacts the key, so logging an instance cannot leak key material.
+ fn __repr__(&self, py: Python<'_>) -> PyResult<String> {
+ let key_length = self.encryption_key.bind(py).as_bytes().len();
+ let aad_prefix = match &self.aad_prefix {
+ Some(prefix) => prefix.bind(py).repr()?.to_string(),
+ None => "None".to_string(),
+ };
+ let file_length = match self.file_length {
+ Some(file_length) => file_length.to_string(),
+ None => "None".to_string(),
+ };
+
+ Ok(format!(
+ "StandardKeyMetadata(encryption_key=<redacted, {key_length} bytes>, aad_prefix={aad_prefix}, file_length={file_length})"
+ ))
+ }
+}
+
+/// Decode `StandardKeyMetadata` from its wire format.
+#[pyfunction]
+pub fn decode_standard_key_metadata(
+ py: Python<'_>,
+ data: &[u8],
+) -> PyResult<PyStandardKeyMetadata> {
+ let metadata = StandardKeyMetadata::decode(data).map_err(to_py_err)?;
+
+ Ok(PyStandardKeyMetadata {
+ encryption_key: PyBytes::new(py, metadata.encryption_key().as_bytes()).unbind(),
+ aad_prefix: metadata
+ .aad_prefix()
+ .map(|aad_prefix| PyBytes::new(py, aad_prefix).unbind()),
+ file_length: metadata.file_length(),
+ })
+}
+
+/// Encode `StandardKeyMetadata` to its wire format.
+#[pyfunction]
+#[pyo3(signature = (encryption_key, aad_prefix=None, file_length=None))]
+pub fn encode_standard_key_metadata<'py>(
+ py: Python<'py>,
+ encryption_key: &[u8],
+ aad_prefix: Option<&[u8]>,
+ file_length: Option<u64>,
+) -> PyResult<Bound<'py, PyBytes>> {
+ let mut metadata = StandardKeyMetadata::try_new(encryption_key).map_err(to_py_err)?;
+
+ if let Some(aad_prefix) = aad_prefix {
+ metadata = metadata.with_aad_prefix(aad_prefix);
+ }
+ if let Some(file_length) = file_length {
+ metadata = metadata.with_file_length(file_length);
+ }
+
+ Ok(PyBytes::new(py, &metadata.encode().map_err(to_py_err)?))
+}
+
+pub fn register_module(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
+ let this = PyModule::new(py, "encryption")?;
+
+ this.add_class::<PyStandardKeyMetadata>()?;
+ this.add_function(wrap_pyfunction!(decode_standard_key_metadata, &this)?)?;
+ this.add_function(wrap_pyfunction!(encode_standard_key_metadata, &this)?)?;
+
+ m.add_submodule(&this)?;
+ py.import("sys")?
+ .getattr("modules")?
+ .set_item("pyiceberg_core.encryption", this)
+}
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index baddbe93a..c50b1ce15 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -18,6 +18,7 @@
use pyo3::prelude::*;
mod data_file;
+mod encryption;
mod error;
mod manifest;
mod transform;
@@ -26,5 +27,6 @@ mod transform;
fn pyiceberg_core_rust(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
transform::register_module(py, m)?;
manifest::register_module(py, m)?;
+ encryption::register_module(py, m)?;
Ok(())
}
diff --git a/bindings/python/tests/test_encryption.py b/bindings/python/tests/test_encryption.py
new file mode 100644
index 000000000..1e4715607
--- /dev/null
+++ b/bindings/python/tests/test_encryption.py
@@ -0,0 +1,117 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import pytest
+from pyiceberg_core import encryption
+
+AES128_KEY = b"0123456789012345"
+# A version byte, then the Avro-encoded encryption_key: zigzag length 16, then the key.
+ENCODED_PREFIX = b"\x01\x20" + AES128_KEY
+
+
+def fields(metadata):
+ return (metadata.encryption_key, metadata.aad_prefix, metadata.file_length)
+
+
+# Pinned in both directions so the encoding stays compatible with the Java and Python
+# implementations, including the null union tags written for absent optional fields.
+@pytest.mark.parametrize(
+ "aad_prefix, file_length, encoded",
+ [
+ (None, None, ENCODED_PREFIX + b"\x00\x00"),
+ (b"ad", None, ENCODED_PREFIX + b"\x02\x04ad\x00"),
+ (b"ad", 1024, ENCODED_PREFIX + b"\x02\x04ad\x02\x80\x10"),
+ (b"", None, ENCODED_PREFIX + b"\x02\x00\x00"),
+ ],
+)
+def test_wire_format(aad_prefix, file_length, encoded):
+ assert fields(encryption.decode_standard_key_metadata(encoded)) == (
+ AES128_KEY,
+ aad_prefix,
+ file_length,
+ )
+ assert (
+ encryption.encode_standard_key_metadata(AES128_KEY, aad_prefix, file_length)
+ == encoded
+ )
+
+
+def test_repr_redacts_encryption_key():
+ encoded = encryption.encode_standard_key_metadata(AES128_KEY, b"ad", 1024)
+
+ rendered = repr(encryption.decode_standard_key_metadata(encoded))
+
+ assert AES128_KEY not in rendered.encode()
+ assert rendered == (
+ "StandardKeyMetadata(encryption_key=<redacted, 16 bytes>, "
+ "aad_prefix=b'ad', file_length=1024)"
+ )
+
+
+@pytest.mark.parametrize("key_length", [16, 24, 32])
+def test_encode_accepts_aes_key_lengths(key_length):
+ encoded = encryption.encode_standard_key_metadata(bytes(key_length))
+ metadata = encryption.decode_standard_key_metadata(encoded)
+
+ assert fields(metadata) == (bytes(key_length), None, None)
+
+
+@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33])
+def test_encode_rejects_invalid_key_length(key_length):
+ with pytest.raises(ValueError, match="key length"):
+ encryption.encode_standard_key_metadata(bytes(key_length))
+
+
+@pytest.mark.parametrize("key_length", [0, 4, 15, 20, 33])
+def test_decode_rejects_invalid_key_length(key_length):
+ # Key length is validated on decode too, not only on encode. The Avro zigzag length
+ # is 2 * key_length, a single byte for every length here.
+ data = b"\x01" + bytes([key_length * 2]) + bytes(key_length) + b"\x00\x00"
+
+ with pytest.raises(ValueError, match="Invalid encryption key in key metadata"):
+ encryption.decode_standard_key_metadata(data)
+
+
+@pytest.mark.parametrize("data", [b"\x02", b"\x02\x20" + AES128_KEY + b"\x00\x00"])
+def test_decode_rejects_unsupported_version(data):
+ with pytest.raises(ValueError, match=r"version: 2"):
+ encryption.decode_standard_key_metadata(data)
+
+
+def test_decode_rejects_empty_buffer():
+ with pytest.raises(ValueError, match="Empty key metadata"):
+ encryption.decode_standard_key_metadata(b"")
+
+
+def test_decode_tolerates_trailing_bytes():
+ # Deliberate, matching Java: a reader must tolerate fields a newer writer appended.
+ encoded = encryption.encode_standard_key_metadata(AES128_KEY, b"ad", 1024)
+
+ metadata = encryption.decode_standard_key_metadata(encoded + b"\xde\xad\xbe\xef")
+
+ assert fields(metadata) == (AES128_KEY, b"ad", 1024)
+
+
+@pytest.mark.xfail(
+ reason="apache/avro-rs#664: a missing union tag decodes as null; fixed upstream in 0.23",
+ strict=True,
+)
+def test_decode_rejects_truncated_union_tags():
+ # Version and key, with both union tags omitted, currently decodes as if the
+ # optional fields were absent rather than erroring.
+ with pytest.raises(ValueError):
+ encryption.decode_standard_key_metadata(ENCODED_PREFIX)
diff --git a/crates/iceberg/src/encryption/key_metadata.rs b/crates/iceberg/src/encryption/key_metadata.rs
index 1169698e2..765d600b5 100644
--- a/crates/iceberg/src/encryption/key_metadata.rs
+++ b/crates/iceberg/src/encryption/key_metadata.rs
@@ -206,7 +206,7 @@ mod _serde {
if version != V1 {
return Err(Error::new(
ErrorKind::FeatureUnsupported,
- format!("Cannot resolve schema for version: {version}"),
+ format!("Unsupported key metadata version: {version} (supported: {V1})"),
));
}
@@ -302,6 +302,10 @@ mod tests {
assert!(result.is_err());
let err = result.unwrap_err();
assert_eq!(err.kind(), ErrorKind::FeatureUnsupported);
+ assert_eq!(
+ err.message(),
+ "Unsupported key metadata version: 2 (supported: 1)"
+ );
}
#[test]
@@ -358,4 +362,35 @@ mod tests {
);
}
}
+
+ #[test]
+ fn test_decode_tolerates_trailing_bytes() {
+ // Deliberate, not an oversight: Java's RawDecoder reads a single datum without
+ // checking for EOF, and Iceberg depends on that leniency — `file_length` was
+ // appended to SCHEMA_V1 without bumping the version byte. Requiring EOF here
+ // would stop older readers tolerating fields a newer writer appended.
+ let key = b"0123456789012345";
+ let aad = b"1234567890123456";
+ let file_length = 1024;
+
+ let serialized = StandardKeyMetadata::try_new(key)
+ .unwrap()
+ .with_aad_prefix(aad)
+ .with_file_length(file_length)
+ .encode()
+ .unwrap();
+
+ // Arbitrary junk, then bytes shaped like a further optional field appended by a
+ // newer writer (union tag "present", then a 4-byte value).
+ for trailing in [b"\xde\xad\xbe\xef".as_slice(), b"\x02\x08more".as_slice()] {
+ let mut extended = serialized.to_vec();
+ extended.extend_from_slice(trailing);
+
+ let parsed = StandardKeyMetadata::decode(&extended).unwrap();
+
+ assert_eq!(parsed.encryption_key().as_bytes(), key);
+ assert_eq!(parsed.aad_prefix(), Some(aad.as_slice()));
+ assert_eq!(parsed.file_length(), Some(file_length));
+ }
+ }
}