You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2024/11/28 19:44:54 UTC

(iceberg) branch main updated: Default to `overwrite` when operation is missing (#11421)

This is an automated email from the ASF dual-hosted git repository.

fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new 3a04257e49 Default to `overwrite` when operation is missing (#11421)
3a04257e49 is described below

commit 3a04257e49b555b6013f1b984e1c1802e53e1956
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Thu Nov 28 20:44:49 2024 +0100

    Default to `overwrite` when operation is missing (#11421)
    
    * Default to `overwrite` when operation is missing
    
    * Update wording
    
    Co-authored-by: Russell Spitzer <ru...@GMAIL.COM>
    
    * Feedback
    
    * Spotless..
    
    * Convert an empty summary to `null`
    
    * Modify behavior and add some tests
    
    ---------
    
    Co-authored-by: Russell Spitzer <ru...@GMAIL.COM>
---
 .../java/org/apache/iceberg/SnapshotParser.java    | 32 +++++++---
 .../java/org/apache/iceberg/TestSnapshotJson.java  | 73 ++++++++++++++++++++++
 2 files changed, 96 insertions(+), 9 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/SnapshotParser.java b/core/src/main/java/org/apache/iceberg/SnapshotParser.java
index bc5ef60946..b5ac3ec718 100644
--- a/core/src/main/java/org/apache/iceberg/SnapshotParser.java
+++ b/core/src/main/java/org/apache/iceberg/SnapshotParser.java
@@ -31,8 +31,11 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
 import org.apache.iceberg.util.JsonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
 
 public class SnapshotParser {
+  private static final Logger LOG = LoggerFactory.getLogger(SnapshotParser.class);
 
   private SnapshotParser() {}
 
@@ -129,17 +132,28 @@ public class SnapshotParser {
           "Cannot parse summary from non-object value: %s",
           sNode);
 
-      ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
-      Iterator<String> fields = sNode.fieldNames();
-      while (fields.hasNext()) {
-        String field = fields.next();
-        if (field.equals(OPERATION)) {
-          operation = JsonUtil.getString(OPERATION, sNode);
-        } else {
-          builder.put(field, JsonUtil.getString(field, sNode));
+      if (sNode.size() > 0) {
+        ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+        Iterator<String> fields = sNode.fieldNames();
+        while (fields.hasNext()) {
+          String field = fields.next();
+          if (field.equals(OPERATION)) {
+            operation = JsonUtil.getString(OPERATION, sNode);
+          } else {
+            builder.put(field, JsonUtil.getString(field, sNode));
+          }
+        }
+        summary = builder.build();
+
+        // When the operation is not found, default to overwrite
+        // to ensure that we can read the summary without raising an exception
+        if (operation == null) {
+          LOG.warn(
+              "Encountered invalid summary for snapshot {}: the field 'operation' is required but missing, setting 'operation' to overwrite",
+              snapshotId);
+          operation = DataOperations.OVERWRITE;
         }
       }
-      summary = builder.build();
     }
 
     Integer schemaId = JsonUtil.getIntOrNull(SCHEMA_ID, node);
diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java
index ee12390749..8a067fc4dc 100644
--- a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java
+++ b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java
@@ -176,4 +176,77 @@ public class TestSnapshotJson {
 
     return localInput(manifestList).location();
   }
+
+  @Test
+  public void testJsonConversionSummaryWithoutOperation() {
+    // This behavior is out of spec, but we don't want to fail on it.
+    // Instead, the operation will be set to overwrite, to ensure that it will produce
+    // correct metadata when it is written
+
+    long currentMs = System.currentTimeMillis();
+    String json =
+        String.format(
+            "{\n"
+                + "  \"snapshot-id\" : 2,\n"
+                + "  \"parent-snapshot-id\" : 1,\n"
+                + "  \"timestamp-ms\" : %s,\n"
+                + "  \"summary\" : {\n"
+                + "    \"files-added\" : \"4\",\n"
+                + "    \"files-deleted\" : \"100\"\n"
+                + "  },\n"
+                + "  \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+                + "  \"schema-id\" : 3\n"
+                + "}",
+            currentMs);
+
+    Snapshot snap = SnapshotParser.fromJson(json);
+    String expected =
+        String.format(
+            "{\n"
+                + "  \"snapshot-id\" : 2,\n"
+                + "  \"parent-snapshot-id\" : 1,\n"
+                + "  \"timestamp-ms\" : %s,\n"
+                + "  \"summary\" : {\n"
+                + "    \"operation\" : \"overwrite\",\n"
+                + "    \"files-added\" : \"4\",\n"
+                + "    \"files-deleted\" : \"100\"\n"
+                + "  },\n"
+                + "  \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+                + "  \"schema-id\" : 3\n"
+                + "}",
+            currentMs);
+    assertThat(SnapshotParser.toJson(snap)).isEqualTo(expected);
+  }
+
+  @Test
+  public void testJsonConversionEmptySummary() {
+    // This behavior is out of spec, but we don't want to fail on it.
+    // Instead, when we find an empty summary, we'll just set it to null
+
+    long currentMs = System.currentTimeMillis();
+    String json =
+        String.format(
+            "{\n"
+                + "  \"snapshot-id\" : 2,\n"
+                + "  \"parent-snapshot-id\" : 1,\n"
+                + "  \"timestamp-ms\" : %s,\n"
+                + "  \"summary\" : { },\n"
+                + "  \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+                + "  \"schema-id\" : 3\n"
+                + "}",
+            currentMs);
+
+    Snapshot snap = SnapshotParser.fromJson(json);
+    String expected =
+        String.format(
+            "{\n"
+                + "  \"snapshot-id\" : 2,\n"
+                + "  \"parent-snapshot-id\" : 1,\n"
+                + "  \"timestamp-ms\" : %s,\n"
+                + "  \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+                + "  \"schema-id\" : 3\n"
+                + "}",
+            currentMs);
+    assertThat(SnapshotParser.toJson(snap)).isEqualTo(expected);
+  }
 }