You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2023/10/11 17:28:35 UTC

[iceberg] branch master updated: Core: Support view metadata compression (#8552)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new b7e5d685c0 Core: Support view metadata compression (#8552)
b7e5d685c0 is described below

commit b7e5d685c00a0b57d7b363146e1c15546a1ac1d1
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Wed Oct 11 19:28:28 2023 +0200

    Core: Support view metadata compression (#8552)
---
 .../apache/iceberg/view/BaseViewOperations.java    |  9 ++-
 .../apache/iceberg/view/ViewMetadataParser.java    | 12 +++-
 .../org/apache/iceberg/view/ViewProperties.java    |  3 +
 .../iceberg/view/TestViewMetadataParser.java       | 67 ++++++++++++++++++++++
 4 files changed, 88 insertions(+), 3 deletions(-)

diff --git a/core/src/main/java/org/apache/iceberg/view/BaseViewOperations.java b/core/src/main/java/org/apache/iceberg/view/BaseViewOperations.java
index f7270b9a35..766d217346 100644
--- a/core/src/main/java/org/apache/iceberg/view/BaseViewOperations.java
+++ b/core/src/main/java/org/apache/iceberg/view/BaseViewOperations.java
@@ -22,6 +22,7 @@ import java.util.UUID;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Function;
 import java.util.function.Predicate;
+import org.apache.iceberg.TableMetadataParser;
 import org.apache.iceberg.exceptions.AlreadyExistsException;
 import org.apache.iceberg.exceptions.CommitFailedException;
 import org.apache.iceberg.exceptions.NoSuchViewException;
@@ -147,8 +148,14 @@ public abstract class BaseViewOperations implements ViewOperations {
   }
 
   private String newMetadataFilePath(ViewMetadata metadata, int newVersion) {
+    String codecName =
+        metadata
+            .properties()
+            .getOrDefault(
+                ViewProperties.METADATA_COMPRESSION, ViewProperties.METADATA_COMPRESSION_DEFAULT);
+    String fileExtension = TableMetadataParser.getFileExtension(codecName);
     return metadataFileLocation(
-        metadata, String.format("%05d-%s%s", newVersion, UUID.randomUUID(), ".metadata.json"));
+        metadata, String.format("%05d-%s%s", newVersion, UUID.randomUUID(), fileExtension));
   }
 
   private String metadataFileLocation(ViewMetadata metadata, String filename) {
diff --git a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
index 7a29c87bad..efce2b725e 100644
--- a/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
+++ b/core/src/main/java/org/apache/iceberg/view/ViewMetadataParser.java
@@ -28,8 +28,11 @@ import java.io.UncheckedIOException;
 import java.nio.charset.StandardCharsets;
 import java.util.List;
 import java.util.Map;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.GZIPOutputStream;
 import org.apache.iceberg.Schema;
 import org.apache.iceberg.SchemaParser;
+import org.apache.iceberg.TableMetadataParser.Codec;
 import org.apache.iceberg.io.InputFile;
 import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
@@ -163,7 +166,9 @@ public class ViewMetadataParser {
   }
 
   public static ViewMetadata read(InputFile file) {
-    try (InputStream is = file.newStream()) {
+    Codec codec = Codec.fromFileName(file.location());
+    try (InputStream is =
+        codec == Codec.GZIP ? new GZIPInputStream(file.newStream()) : file.newStream()) {
       return fromJson(file.location(), JsonUtil.mapper().readValue(is, JsonNode.class));
     } catch (IOException e) {
       throw new UncheckedIOException(String.format("Failed to read json file: %s", file), e);
@@ -172,8 +177,11 @@ public class ViewMetadataParser {
 
   private static void internalWrite(
       ViewMetadata metadata, OutputFile outputFile, boolean overwrite) {
+    boolean isGzip = Codec.fromFileName(outputFile.location()) == Codec.GZIP;
     OutputStream stream = overwrite ? outputFile.createOrOverwrite() : outputFile.create();
-    try (OutputStreamWriter writer = new OutputStreamWriter(stream, StandardCharsets.UTF_8)) {
+    try (OutputStreamWriter writer =
+        new OutputStreamWriter(
+            isGzip ? new GZIPOutputStream(stream) : stream, StandardCharsets.UTF_8)) {
       JsonGenerator generator = JsonUtil.factory().createGenerator(writer);
       generator.useDefaultPrettyPrinter();
       toJson(metadata, generator);
diff --git a/core/src/main/java/org/apache/iceberg/view/ViewProperties.java b/core/src/main/java/org/apache/iceberg/view/ViewProperties.java
index 07cd3cbf65..b356db7ce0 100644
--- a/core/src/main/java/org/apache/iceberg/view/ViewProperties.java
+++ b/core/src/main/java/org/apache/iceberg/view/ViewProperties.java
@@ -23,5 +23,8 @@ public class ViewProperties {
   public static final String VERSION_HISTORY_SIZE = "version.history.num-entries";
   public static final int VERSION_HISTORY_SIZE_DEFAULT = 10;
 
+  public static final String METADATA_COMPRESSION = "write.metadata.compression-codec";
+  public static final String METADATA_COMPRESSION_DEFAULT = "gzip";
+
   private ViewProperties() {}
 }
diff --git a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
index 267195c133..77baadb7a1 100644
--- a/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
+++ b/core/src/test/java/org/apache/iceberg/view/TestViewMetadataParser.java
@@ -22,14 +22,25 @@ import static org.assertj.core.api.Assertions.assertThat;
 import static org.assertj.core.api.Assertions.assertThatThrownBy;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
+import java.util.zip.GZIPInputStream;
+import java.util.zip.ZipException;
 import org.apache.iceberg.Schema;
+import org.apache.iceberg.TableMetadataParser.Codec;
 import org.apache.iceberg.catalog.Namespace;
+import org.apache.iceberg.io.OutputFile;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
 import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
 import org.apache.iceberg.types.Types;
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
 
 public class TestViewMetadataParser {
 
@@ -39,6 +50,8 @@ public class TestViewMetadataParser {
           Types.NestedField.required(2, "y", Types.LongType.get(), "comment"),
           Types.NestedField.required(3, "z", Types.LongType.get()));
 
+  @TempDir private Path tmp;
+
   @Test
   public void nullAndEmptyCheck() {
     assertThatThrownBy(() -> ViewMetadataParser.fromJson((String) null))
@@ -308,4 +321,58 @@ public class TestViewMetadataParser {
 
     assertThat(replaced.currentVersion()).isEqualTo(viewVersion);
   }
+
+  @ParameterizedTest
+  @ValueSource(strings = {"v1.metadata.json", "v1.gz.metadata.json"})
+  public void metadataCompression(String fileName) throws IOException {
+    Codec codec = fileName.startsWith("v1.gz") ? Codec.GZIP : Codec.NONE;
+    String location = Paths.get(tmp.toString(), fileName).toString();
+    OutputFile outputFile = org.apache.iceberg.Files.localOutput(location);
+
+    Schema schema = new Schema(Types.NestedField.required(1, "x", Types.LongType.get()));
+    ViewVersion viewVersion =
+        ImmutableViewVersion.builder()
+            .schemaId(0)
+            .versionId(1)
+            .timestampMillis(23L)
+            .putSummary("operation", "create")
+            .defaultNamespace(Namespace.of("ns"))
+            .build();
+
+    ViewMetadata metadata =
+        ViewMetadata.buildFrom(
+                ViewMetadata.builder()
+                    .setLocation(location)
+                    .addSchema(schema)
+                    .setProperties(
+                        ImmutableMap.of(ViewProperties.METADATA_COMPRESSION, codec.name()))
+                    .addVersion(viewVersion)
+                    .setCurrentVersionId(1)
+                    .build())
+            .setMetadataLocation(outputFile.location())
+            .build();
+
+    ViewMetadataParser.write(metadata, outputFile);
+    assertThat(Codec.GZIP == codec).isEqualTo(isCompressed(location));
+
+    ViewMetadata actualMetadata =
+        ViewMetadataParser.read(org.apache.iceberg.Files.localInput(location));
+
+    assertThat(actualMetadata)
+        .usingRecursiveComparison()
+        .ignoringFieldsOfTypes(Schema.class)
+        .isEqualTo(metadata);
+  }
+
+  private boolean isCompressed(String path) throws IOException {
+    try (InputStream ignored = new GZIPInputStream(Files.newInputStream(new File(path).toPath()))) {
+      return true;
+    } catch (ZipException e) {
+      if (e.getMessage().equals("Not in GZIP format")) {
+        return false;
+      } else {
+        throw e;
+      }
+    }
+  }
 }