You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/10/21 00:05:23 UTC
[iceberg] branch master updated: Nessie: Do not post metadata JSON to Nessie (#5999)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new a62ff8de79 Nessie: Do not post metadata JSON to Nessie (#5999)
a62ff8de79 is described below
commit a62ff8de7908c7c79ddb40fd2d3e81f779a1e5fb
Author: Robert Stupp <sn...@snazy.de>
AuthorDate: Fri Oct 21 02:05:18 2022 +0200
Nessie: Do not post metadata JSON to Nessie (#5999)
---
.../iceberg/nessie/NessieTableOperations.java | 9 +--
.../java/org/apache/iceberg/nessie/NessieUtil.java | 43 -------------
.../org/apache/iceberg/nessie/TestNessieUtil.java | 74 ----------------------
3 files changed, 2 insertions(+), 124 deletions(-)
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
index 99a579da95..2d949fcd7d 100644
--- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
+++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
@@ -18,12 +18,12 @@
*/
package org.apache.iceberg.nessie;
-import com.fasterxml.jackson.databind.JsonNode;
import java.util.Map;
import org.apache.iceberg.BaseMetastoreTableOperations;
import org.apache.iceberg.Snapshot;
import org.apache.iceberg.SnapshotRef;
import org.apache.iceberg.TableMetadata;
+import org.apache.iceberg.TableMetadataParser;
import org.apache.iceberg.TableProperties;
import org.apache.iceberg.exceptions.CommitFailedException;
import org.apache.iceberg.exceptions.CommitStateUnknownException;
@@ -36,7 +36,6 @@ import org.projectnessie.error.NessieNotFoundException;
import org.projectnessie.model.Branch;
import org.projectnessie.model.Content;
import org.projectnessie.model.ContentKey;
-import org.projectnessie.model.GenericMetadata;
import org.projectnessie.model.IcebergTable;
import org.projectnessie.model.ImmutableCommitMeta;
import org.projectnessie.model.ImmutableIcebergTable;
@@ -81,8 +80,7 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
private TableMetadata loadTableMetadata(String metadataLocation, Reference reference) {
// Update the TableMetadata with the Content of NessieTableState.
- TableMetadata deserialized =
- NessieUtil.tableMetadataFromIcebergTable(io(), table, metadataLocation);
+ TableMetadata deserialized = TableMetadataParser.read(io(), metadataLocation);
Map<String, String> newProperties = Maps.newHashMap(deserialized.properties());
newProperties.put(NESSIE_COMMIT_ID_PROPERTY, reference.getHash());
// To prevent accidental deletion of files that are still referenced by other branches/tags,
@@ -177,7 +175,6 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
Snapshot snapshot = metadata.currentSnapshot();
long snapshotId = snapshot != null ? snapshot.snapshotId() : -1L;
- JsonNode newMetadata = NessieUtil.tableMetadataAsJsonNode(metadata);
IcebergTable newTable =
newTableBuilder
.snapshotId(snapshotId)
@@ -185,8 +182,6 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
.specId(metadata.defaultSpecId())
.sortOrderId(metadata.defaultSortOrderId())
.metadataLocation(newMetadataLocation)
- .metadata(
- GenericMetadata.of("org.apache:iceberg:" + metadata.formatVersion(), newMetadata))
.build();
LOG.debug(
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
index 8741151f2f..0e57d8ccc3 100644
--- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
+++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieUtil.java
@@ -18,28 +18,18 @@
*/
package org.apache.iceberg.nessie;
-import com.fasterxml.jackson.core.JsonGenerator;
-import com.fasterxml.jackson.core.JsonParser;
-import com.fasterxml.jackson.databind.JsonNode;
-import java.io.IOException;
-import java.io.StringWriter;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import javax.annotation.Nullable;
import org.apache.iceberg.CatalogProperties;
-import org.apache.iceberg.TableMetadata;
-import org.apache.iceberg.TableMetadataParser;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
-import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
-import org.apache.iceberg.util.JsonUtil;
import org.projectnessie.model.CommitMeta;
import org.projectnessie.model.ContentKey;
-import org.projectnessie.model.IcebergTable;
import org.projectnessie.model.ImmutableCommitMeta;
public final class NessieUtil {
@@ -99,37 +89,4 @@ public final class NessieUtil {
return Optional.ofNullable(catalogOptions.get(CatalogProperties.USER))
.orElseGet(() -> System.getProperty("user.name"));
}
-
- static TableMetadata tableMetadataFromIcebergTable(
- FileIO io, IcebergTable table, String metadataLocation) {
- TableMetadata deserialized;
- if (table.getMetadata() != null) {
- String jsonString;
- try (StringWriter writer = new StringWriter()) {
- try (JsonGenerator generator = JsonUtil.factory().createGenerator(writer)) {
- generator.writeObject(table.getMetadata().getMetadata());
- }
- jsonString = writer.toString();
- } catch (IOException e) {
- throw new RuntimeException("Failed to generate JSON string from metadata", e);
- }
- deserialized = TableMetadataParser.fromJson(metadataLocation, jsonString);
- } else {
- deserialized = TableMetadataParser.read(io, metadataLocation);
- }
- return deserialized;
- }
-
- static JsonNode tableMetadataAsJsonNode(TableMetadata metadata) {
- JsonNode newMetadata;
- try {
- String jsonString = TableMetadataParser.toJson(metadata);
- try (JsonParser parser = JsonUtil.factory().createParser(jsonString)) {
- newMetadata = parser.readValueAs(JsonNode.class);
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- return newMetadata;
- }
}
diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieUtil.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieUtil.java
index 6d7b4e825f..624ce99adf 100644
--- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieUtil.java
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieUtil.java
@@ -18,88 +18,14 @@
*/
package org.apache.iceberg.nessie;
-import com.fasterxml.jackson.databind.JsonNode;
-import java.util.Collections;
-import java.util.Map;
import org.apache.iceberg.CatalogProperties;
-import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.Schema;
-import org.apache.iceberg.SortOrder;
-import org.apache.iceberg.TableMetadata;
-import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
-import org.apache.iceberg.types.Types.NestedField;
-import org.apache.iceberg.types.Types.StringType;
import org.assertj.core.api.Assertions;
-import org.assertj.core.api.InstanceOfAssertFactories;
import org.junit.jupiter.api.Test;
-import org.mockito.Mockito;
import org.projectnessie.model.CommitMeta;
-import org.projectnessie.model.GenericMetadata;
-import org.projectnessie.model.IcebergTable;
-import org.projectnessie.model.ImmutableIcebergTable;
public class TestNessieUtil {
- @Test
- public void testTableMetadataJsonRoundtrip() {
- // Construct a dummy TableMetadata object
- Map<String, String> properties = Collections.singletonMap("property-key", "property-value");
- String location = "obj://foo/bar/baz";
- TableMetadata tableMetadata =
- TableMetadata.newTableMetadata(
- new Schema(1, NestedField.of(1, false, "column", StringType.get())),
- PartitionSpec.unpartitioned(),
- SortOrder.unsorted(),
- location,
- properties);
-
- // Produce a generic JsonNode from the TableMetadata
- JsonNode jsonNode = NessieUtil.tableMetadataAsJsonNode(tableMetadata);
- Assertions.assertThat(jsonNode)
- .asInstanceOf(InstanceOfAssertFactories.type(JsonNode.class))
- .extracting(
- n -> n.get("format-version").asLong(-2L),
- n -> n.get("location").asText("x"),
- n -> n.get("properties").get("property-key").asText())
- .containsExactly(1L, location, "property-value");
-
- // Create a Nessie IcebergTable object with the JsonNode as the metadata
- IcebergTable icebergTableNoMetadata = IcebergTable.of(location, 0L, 1, 2, 3, "cid");
- IcebergTable icebergTable =
- ImmutableIcebergTable.builder()
- .from(icebergTableNoMetadata)
- .metadata(GenericMetadata.of("iceberg", jsonNode))
- .build();
-
- // Deserialize the TableMetadata from Nessie IcebergTable
- TableMetadata deserializedMetadata =
- NessieUtil.tableMetadataFromIcebergTable(null, icebergTable, location);
-
- // (Could compare tableMetadata against deserializedMetadata, but TableMetadata has no equals())
-
- // Produce a JsonNode from the deserializedMetadata and compare that against jsonNode
- JsonNode deserializedJsonNode = NessieUtil.tableMetadataAsJsonNode(deserializedMetadata);
- Assertions.assertThat(deserializedJsonNode).isEqualTo(jsonNode);
- }
-
- @Test
- public void testTableMetadataFromFileIO() {
- String location = "obj://foo/bar/baz";
- FileIO fileIoMock = Mockito.mock(FileIO.class);
- IcebergTable icebergTableNoMetadata = IcebergTable.of(location, 0L, 1, 2, 3, "cid");
-
- // Check that newInputFile() is called when IcebergTable has no metadata
- Mockito.when(fileIoMock.newInputFile(location))
- .thenThrow(new RuntimeException("newInputFile called"));
- Assertions.assertThatThrownBy(
- () ->
- NessieUtil.tableMetadataFromIcebergTable(
- fileIoMock, icebergTableNoMetadata, location))
- .isInstanceOf(RuntimeException.class)
- .hasMessage("newInputFile called");
- }
-
@Test
public void testBuildingCommitMetadataWithNullCatalogOptions() {
Assertions.assertThatThrownBy(() -> NessieUtil.buildCommitMetadata("msg", null))