You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by fo...@apache.org on 2024/11/28 19:44:54 UTC
(iceberg) branch main updated: Default to `overwrite` when operation is missing (#11421)
This is an automated email from the ASF dual-hosted git repository.
fokko pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 3a04257e49 Default to `overwrite` when operation is missing (#11421)
3a04257e49 is described below
commit 3a04257e49b555b6013f1b984e1c1802e53e1956
Author: Fokko Driesprong <fo...@apache.org>
AuthorDate: Thu Nov 28 20:44:49 2024 +0100
Default to `overwrite` when operation is missing (#11421)
* Default to `overwrite` when operation is missing
* Update wording
Co-authored-by: Russell Spitzer <ru...@GMAIL.COM>
* Feedback
* Spotless..
* Convert an empty summary to `null`
* Modify behavior and add some tests
---------
Co-authored-by: Russell Spitzer <ru...@GMAIL.COM>
---
.../java/org/apache/iceberg/SnapshotParser.java | 32 +++++++---
.../java/org/apache/iceberg/TestSnapshotJson.java | 73 ++++++++++++++++++++++
2 files changed, 96 insertions(+), 9 deletions(-)
diff --git a/core/src/main/java/org/apache/iceberg/SnapshotParser.java b/core/src/main/java/org/apache/iceberg/SnapshotParser.java
index bc5ef60946..b5ac3ec718 100644
--- a/core/src/main/java/org/apache/iceberg/SnapshotParser.java
+++ b/core/src/main/java/org/apache/iceberg/SnapshotParser.java
@@ -31,8 +31,11 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.util.JsonUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class SnapshotParser {
+ private static final Logger LOG = LoggerFactory.getLogger(SnapshotParser.class);
private SnapshotParser() {}
@@ -129,17 +132,28 @@ public class SnapshotParser {
"Cannot parse summary from non-object value: %s",
sNode);
- ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
- Iterator<String> fields = sNode.fieldNames();
- while (fields.hasNext()) {
- String field = fields.next();
- if (field.equals(OPERATION)) {
- operation = JsonUtil.getString(OPERATION, sNode);
- } else {
- builder.put(field, JsonUtil.getString(field, sNode));
+ if (sNode.size() > 0) {
+ ImmutableMap.Builder<String, String> builder = ImmutableMap.builder();
+ Iterator<String> fields = sNode.fieldNames();
+ while (fields.hasNext()) {
+ String field = fields.next();
+ if (field.equals(OPERATION)) {
+ operation = JsonUtil.getString(OPERATION, sNode);
+ } else {
+ builder.put(field, JsonUtil.getString(field, sNode));
+ }
+ }
+ summary = builder.build();
+
+ // When the operation is not found, default to overwrite
+ // to ensure that we can read the summary without raising an exception
+ if (operation == null) {
+ LOG.warn(
+ "Encountered invalid summary for snapshot {}: the field 'operation' is required but missing, setting 'operation' to overwrite",
+ snapshotId);
+ operation = DataOperations.OVERWRITE;
}
}
- summary = builder.build();
}
Integer schemaId = JsonUtil.getIntOrNull(SCHEMA_ID, node);
diff --git a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java
index ee12390749..8a067fc4dc 100644
--- a/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java
+++ b/core/src/test/java/org/apache/iceberg/TestSnapshotJson.java
@@ -176,4 +176,77 @@ public class TestSnapshotJson {
return localInput(manifestList).location();
}
+
+ @Test
+ public void testJsonConversionSummaryWithoutOperation() {
+ // This behavior is out of spec, but we don't want to fail on it.
+ // Instead, the operation will be set to overwrite, to ensure that it will produce
+ // correct metadata when it is written
+
+ long currentMs = System.currentTimeMillis();
+ String json =
+ String.format(
+ "{\n"
+ + " \"snapshot-id\" : 2,\n"
+ + " \"parent-snapshot-id\" : 1,\n"
+ + " \"timestamp-ms\" : %s,\n"
+ + " \"summary\" : {\n"
+ + " \"files-added\" : \"4\",\n"
+ + " \"files-deleted\" : \"100\"\n"
+ + " },\n"
+ + " \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+ + " \"schema-id\" : 3\n"
+ + "}",
+ currentMs);
+
+ Snapshot snap = SnapshotParser.fromJson(json);
+ String expected =
+ String.format(
+ "{\n"
+ + " \"snapshot-id\" : 2,\n"
+ + " \"parent-snapshot-id\" : 1,\n"
+ + " \"timestamp-ms\" : %s,\n"
+ + " \"summary\" : {\n"
+ + " \"operation\" : \"overwrite\",\n"
+ + " \"files-added\" : \"4\",\n"
+ + " \"files-deleted\" : \"100\"\n"
+ + " },\n"
+ + " \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+ + " \"schema-id\" : 3\n"
+ + "}",
+ currentMs);
+ assertThat(SnapshotParser.toJson(snap)).isEqualTo(expected);
+ }
+
+ @Test
+ public void testJsonConversionEmptySummary() {
+ // This behavior is out of spec, but we don't want to fail on it.
+ // Instead, when we find an empty summary, we'll just set it to null
+
+ long currentMs = System.currentTimeMillis();
+ String json =
+ String.format(
+ "{\n"
+ + " \"snapshot-id\" : 2,\n"
+ + " \"parent-snapshot-id\" : 1,\n"
+ + " \"timestamp-ms\" : %s,\n"
+ + " \"summary\" : { },\n"
+ + " \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+ + " \"schema-id\" : 3\n"
+ + "}",
+ currentMs);
+
+ Snapshot snap = SnapshotParser.fromJson(json);
+ String expected =
+ String.format(
+ "{\n"
+ + " \"snapshot-id\" : 2,\n"
+ + " \"parent-snapshot-id\" : 1,\n"
+ + " \"timestamp-ms\" : %s,\n"
+ + " \"manifests\" : [ \"/tmp/manifest1.avro\", \"/tmp/manifest2.avro\" ],\n"
+ + " \"schema-id\" : 3\n"
+ + "}",
+ currentMs);
+ assertThat(SnapshotParser.toJson(snap)).isEqualTo(expected);
+ }
}