You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ry...@apache.org on 2022/04/01 15:49:35 UTC

[iceberg] branch master updated: Nessie: Add Catalog tests for NessieCatalog (#4392)

This is an automated email from the ASF dual-hosted git repository.

rymurr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new dd3137d  Nessie: Add Catalog tests for NessieCatalog (#4392)
dd3137d is described below

commit dd3137d792f3716dd63687762a64691a98db40ab
Author: Eduard Tudenhöfner <et...@gmail.com>
AuthorDate: Fri Apr 1 17:49:21 2022 +0200

    Nessie: Add Catalog tests for NessieCatalog (#4392)
---
 build.gradle                                       |   1 +
 .../iceberg/nessie/NessieTableOperations.java      |   4 +-
 .../apache/iceberg/nessie/TestNessieCatalog.java   | 162 +++++++++++++++++++++
 .../org/apache/iceberg/nessie/TestNessieTable.java |   2 +-
 4 files changed, 166 insertions(+), 3 deletions(-)

diff --git a/build.gradle b/build.gradle
index 260b760..25bf761 100644
--- a/build.gradle
+++ b/build.gradle
@@ -597,6 +597,7 @@ project(':iceberg-nessie') {
     testImplementation "org.apache.avro:avro"
 
     testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+    testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
   }
 }
 
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
index d106990..ebc294b 100644
--- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
+++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
@@ -165,7 +165,7 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
 
       delete = false;
     } catch (NessieConflictException ex) {
-      throw new CommitFailedException(ex, "Commit failed: Reference hash is out of date. " +
+      throw new CommitFailedException(ex, "Cannot commit: Reference hash is out of date. " +
           "Update the reference %s and try again", reference.getName());
     } catch (HttpClientException ex) {
       // Intentionally catch all nessie-client-exceptions here and not just the "timeout" variant
@@ -176,7 +176,7 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
       throw new CommitStateUnknownException(ex);
     } catch (NessieNotFoundException ex) {
       throw new RuntimeException(
-          String.format("Commit failed: Reference %s no longer exist", reference.getName()), ex);
+          String.format("Cannot commit: Reference %s no longer exists", reference.getName()), ex);
     } finally {
       if (delete) {
         io().deleteFile(newMetadataLocation);
diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
new file mode 100644
index 0000000..bf11e03
--- /dev/null
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.nessie;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Path;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.io.TempDir;
+import org.projectnessie.client.api.NessieApiV1;
+import org.projectnessie.client.http.HttpClientBuilder;
+import org.projectnessie.error.NessieConflictException;
+import org.projectnessie.error.NessieNotFoundException;
+import org.projectnessie.jaxrs.ext.NessieJaxRsExtension;
+import org.projectnessie.jaxrs.ext.NessieUri;
+import org.projectnessie.model.Branch;
+import org.projectnessie.model.Reference;
+import org.projectnessie.model.Tag;
+import org.projectnessie.server.store.TableCommitMetaStoreWorker;
+import org.projectnessie.versioned.persist.adapter.DatabaseAdapter;
+import org.projectnessie.versioned.persist.inmem.InmemoryDatabaseAdapterFactory;
+import org.projectnessie.versioned.persist.inmem.InmemoryTestConnectionProviderSource;
+import org.projectnessie.versioned.persist.tests.extension.DatabaseAdapterExtension;
+import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapter;
+import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapterName;
+import org.projectnessie.versioned.persist.tests.extension.NessieExternalDatabase;
+
+@ExtendWith(DatabaseAdapterExtension.class)
+@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
+@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
+public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
+
+  @NessieDbAdapter(storeWorker = TableCommitMetaStoreWorker.class)
+  static DatabaseAdapter databaseAdapter;
+
+  @RegisterExtension
+  static NessieJaxRsExtension server = new NessieJaxRsExtension(() -> databaseAdapter);
+
+  @TempDir public Path temp;
+
+  private NessieCatalog catalog;
+  private NessieApiV1 api;
+  private Configuration hadoopConfig;
+  private String uri;
+
+  @BeforeEach
+  public void beforeEach(@NessieUri URI nessieUri) throws IOException {
+    this.uri = nessieUri.toString();
+    this.api = HttpClientBuilder.builder().withUri(this.uri).build(NessieApiV1.class);
+
+    resetData();
+
+    try {
+      api.createReference().reference(Branch.of("main", null)).create();
+    } catch (Exception e) {
+      // ignore, already created. Can't run this in BeforeAll as quarkus hasn't disabled auth
+    }
+
+    hadoopConfig = new Configuration();
+    catalog = initNessieCatalog("main");
+  }
+
+  @AfterEach
+  public void afterEach() throws NessieConflictException, NessieNotFoundException {
+    resetData();
+    try {
+      if (catalog != null) {
+        catalog.close();
+      }
+      api.close();
+    } finally {
+      catalog = null;
+      api = null;
+      hadoopConfig = null;
+    }
+  }
+
+  private void resetData() throws NessieConflictException, NessieNotFoundException {
+    for (Reference r : api.getAllReferences().get().getReferences()) {
+      if (r instanceof Branch) {
+        api.deleteBranch().branch((Branch) r).delete();
+      } else {
+        api.deleteTag().tag((Tag) r).delete();
+      }
+    }
+    api.createReference().reference(Branch.of("main", null)).create();
+  }
+
+  private NessieCatalog initNessieCatalog(String ref) {
+    NessieCatalog newCatalog = new NessieCatalog();
+    newCatalog.setConf(hadoopConfig);
+    newCatalog.initialize(
+        "nessie",
+        ImmutableMap.of(
+            "ref",
+            ref,
+            CatalogProperties.URI,
+            uri,
+            "auth-type",
+            "NONE",
+            CatalogProperties.WAREHOUSE_LOCATION,
+            temp.toUri().toString()));
+    return newCatalog;
+  }
+
+  @Override
+  protected NessieCatalog catalog() {
+    return catalog;
+  }
+
+  @Override
+  protected boolean supportsNamespaceProperties() {
+    return false;
+  }
+
+  @Override
+  protected boolean supportsServerSideRetry() {
+    // TODO: we do support retries, but a bunch of tests are currently failing
+    return false;
+  }
+
+  @Test
+  @Override
+  @Disabled(
+      "Nessie does not differentiate between table creates & updates, thus a concurrent transaction does not fail")
+  public void testConcurrentCreateTransaction() {
+    super.testConcurrentCreateTransaction();
+  }
+
+  @Test
+  @Override
+  @Disabled("currently not validated within doCommit")
+  public void testUUIDValidation() {
+    super.testUUIDValidation();
+  }
+}
diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
index 410c649..20d86ea 100644
--- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
@@ -338,7 +338,7 @@ public class TestNessieTable extends BaseTestIceberg {
     Assertions.assertThatThrownBy(() -> icebergTable.updateSchema().addColumn("data", Types.LongType.get()).commit())
         .isInstanceOf(CommitFailedException.class)
         .hasMessage(
-            "Commit failed: Reference hash is out of date. Update the reference iceberg-table-test and try again");
+            "Cannot commit: Reference hash is out of date. Update the reference iceberg-table-test and try again");
   }
 
   @Test