You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by xu...@apache.org on 2025/05/28 09:20:59 UTC

(iceberg-rust) branch main updated: feat: Introduce snapshot summary properties (#1336)

This is an automated email from the ASF dual-hosted git repository.

xuanwo pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg-rust.git


The following commit(s) were added to refs/heads/main by this push:
     new 8bb9a882 feat: Introduce snapshot summary properties (#1336)
8bb9a882 is described below

commit 8bb9a882c7e5f6cfd2a9762136e874421cd7fb59
Author: dentiny <de...@gmail.com>
AuthorDate: Wed May 28 02:20:54 2025 -0700

    feat: Introduce snapshot summary properties (#1336)
    
    ## Which issue does this PR close?
    
    - Closes https://github.com/apache/iceberg-rust/issues/1329
    
    TLDR:
    - I want to add customized metadata (with size controlled within 200B)
    for each snapshot, summary properties is the perfect place for this
    feature request
    
    ## What changes are included in this PR?
    
    I add a new table update action for snapshot summary properties.
    
    ## Are these changes tested?
    
    Yes, unit tests are added.
---
 crates/iceberg/src/catalog/mod.rs                 | 29 +++++++++
 crates/iceberg/src/spec/snapshot.rs               |  5 ++
 crates/iceberg/src/spec/table_metadata_builder.rs | 74 +++++++++++++++++++++++
 crates/iceberg/src/transaction/mod.rs             | 12 ++++
 4 files changed, 120 insertions(+)

diff --git a/crates/iceberg/src/catalog/mod.rs b/crates/iceberg/src/catalog/mod.rs
index 3457f836..545a2bbb 100644
--- a/crates/iceberg/src/catalog/mod.rs
+++ b/crates/iceberg/src/catalog/mod.rs
@@ -495,6 +495,12 @@ pub enum TableUpdate {
         /// Schema IDs to remove.
         schema_ids: Vec<i32>,
     },
+    /// Add snapshot summary properties.
+    #[serde(rename_all = "kebab-case")]
+    AddSnapshotSummaryProperties {
+        /// Additional properties to add.
+        properties: HashMap<String, String>,
+    },
 }
 
 impl TableUpdate {
@@ -539,6 +545,9 @@ impl TableUpdate {
                 Ok(builder.remove_partition_statistics(snapshot_id))
             }
             TableUpdate::RemoveSchemas { schema_ids } => builder.remove_schemas(&schema_ids),
+            TableUpdate::AddSnapshotSummaryProperties { properties } => {
+                builder.add_snapshot_summary_properties(properties)
+            }
         }
     }
 }
@@ -2098,4 +2107,24 @@ mod tests {
             },
         );
     }
+
+    #[test]
+    fn test_add_snapshot_summary_properties() {
+        let mut expected_properties = HashMap::new();
+        expected_properties.insert("prop-key".to_string(), "prop-value".to_string());
+
+        test_serde_json(
+            r#"
+                {
+                    "action": "add-snapshot-summary-properties",
+                    "properties": {
+                            "prop-key": "prop-value"
+                    }
+                }        
+            "#,
+            TableUpdate::AddSnapshotSummaryProperties {
+                properties: expected_properties,
+            },
+        );
+    }
 }
diff --git a/crates/iceberg/src/spec/snapshot.rs b/crates/iceberg/src/spec/snapshot.rs
index a2716ad9..12e862f1 100644
--- a/crates/iceberg/src/spec/snapshot.rs
+++ b/crates/iceberg/src/spec/snapshot.rs
@@ -214,6 +214,11 @@ impl Snapshot {
             snapshot_id: self.snapshot_id,
         }
     }
+
+    /// Add the given properties map to snapshot summary.
+    pub(crate) fn add_summary_properties(&mut self, props: HashMap<String, String>) {
+        self.summary.additional_properties.extend(props);
+    }
 }
 
 pub(super) mod _serde {
diff --git a/crates/iceberg/src/spec/table_metadata_builder.rs b/crates/iceberg/src/spec/table_metadata_builder.rs
index 1f3f8953..2408e57d 100644
--- a/crates/iceberg/src/spec/table_metadata_builder.rs
+++ b/crates/iceberg/src/spec/table_metadata_builder.rs
@@ -1244,6 +1244,33 @@ impl TableMetadataBuilder {
 
         Ok(self)
     }
+
+    /// Add summary properties to the latest snapshot for the table metadata.
+    pub fn add_snapshot_summary_properties(
+        mut self,
+        properties: HashMap<String, String>,
+    ) -> Result<Self> {
+        if properties.is_empty() {
+            return Ok(self);
+        }
+
+        let snapshot_id = self.metadata.current_snapshot_id.unwrap();
+        let mut cur_snapshot = self
+            .metadata
+            .snapshots
+            .remove(&snapshot_id)
+            .unwrap()
+            .as_ref()
+            .clone();
+        cur_snapshot.add_summary_properties(properties.clone());
+        self.metadata
+            .snapshots
+            .insert(snapshot_id, Arc::new(cur_snapshot));
+        self.changes
+            .push(TableUpdate::AddSnapshotSummaryProperties { properties });
+
+        Ok(self)
+    }
 }
 
 impl From<TableMetadataBuildResult> for TableMetadata {
@@ -2496,4 +2523,51 @@ mod tests {
             };
         assert_eq!(remove_schema_ids, &[0]);
     }
+
+    #[test]
+    fn test_add_snapshot_summary_properties() {
+        let file = File::open(format!(
+            "{}/testdata/table_metadata/{}",
+            env!("CARGO_MANIFEST_DIR"),
+            "TableMetadataV2Valid.json"
+        ))
+        .unwrap();
+        let reader = BufReader::new(file);
+        let resp = serde_json::from_reader::<_, TableMetadata>(reader).unwrap();
+
+        let table = Table::builder()
+            .metadata(resp)
+            .metadata_location("s3://bucket/test/location/metadata/v1.json".to_string())
+            .identifier(TableIdent::from_strs(["ns1", "test1"]).unwrap())
+            .file_io(FileIOBuilder::new("memory").build().unwrap())
+            .build()
+            .unwrap();
+        assert!(
+            table
+                .metadata()
+                .current_snapshot()
+                .unwrap()
+                .summary()
+                .additional_properties
+                .is_empty()
+        );
+
+        let mut new_properties = HashMap::new();
+        new_properties.insert("prop-key".to_string(), "prop-value".to_string());
+
+        let mut meta_data_builder = table.metadata().clone().into_builder(None);
+        meta_data_builder = meta_data_builder
+            .add_snapshot_summary_properties(new_properties.clone())
+            .unwrap();
+        let build_result = meta_data_builder.build().unwrap();
+        assert_eq!(
+            build_result
+                .metadata
+                .current_snapshot()
+                .unwrap()
+                .summary()
+                .additional_properties,
+            new_properties
+        );
+    }
 }
diff --git a/crates/iceberg/src/transaction/mod.rs b/crates/iceberg/src/transaction/mod.rs
index ba79d60b..7f949c3a 100644
--- a/crates/iceberg/src/transaction/mod.rs
+++ b/crates/iceberg/src/transaction/mod.rs
@@ -128,6 +128,18 @@ impl<'a> Transaction<'a> {
         Ok(self)
     }
 
+    /// Add snapshot summary properties.
+    pub fn add_snapshot_summary_properties(
+        mut self,
+        props: HashMap<String, String>,
+    ) -> Result<Self> {
+        self.apply(
+            vec![TableUpdate::AddSnapshotSummaryProperties { properties: props }],
+            vec![],
+        )?;
+        Ok(self)
+    }
+
     fn generate_unique_snapshot_id(&self) -> i64 {
         let generate_random_id = || -> i64 {
             let (lhs, rhs) = Uuid::new_v4().as_u64_pair();