You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by ry...@apache.org on 2022/04/01 15:49:35 UTC
[iceberg] branch master updated: Nessie: Add Catalog tests for NessieCatalog (#4392)
This is an automated email from the ASF dual-hosted git repository.
rymurr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new dd3137d Nessie: Add Catalog tests for NessieCatalog (#4392)
dd3137d is described below
commit dd3137d792f3716dd63687762a64691a98db40ab
Author: Eduard Tudenhöfner <et...@gmail.com>
AuthorDate: Fri Apr 1 17:49:21 2022 +0200
Nessie: Add Catalog tests for NessieCatalog (#4392)
---
build.gradle | 1 +
.../iceberg/nessie/NessieTableOperations.java | 4 +-
.../apache/iceberg/nessie/TestNessieCatalog.java | 162 +++++++++++++++++++++
.../org/apache/iceberg/nessie/TestNessieTable.java | 2 +-
4 files changed, 166 insertions(+), 3 deletions(-)
diff --git a/build.gradle b/build.gradle
index 260b760..25bf761 100644
--- a/build.gradle
+++ b/build.gradle
@@ -597,6 +597,7 @@ project(':iceberg-nessie') {
testImplementation "org.apache.avro:avro"
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
+ testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
}
}
diff --git a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
index d106990..ebc294b 100644
--- a/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
+++ b/nessie/src/main/java/org/apache/iceberg/nessie/NessieTableOperations.java
@@ -165,7 +165,7 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
delete = false;
} catch (NessieConflictException ex) {
- throw new CommitFailedException(ex, "Commit failed: Reference hash is out of date. " +
+ throw new CommitFailedException(ex, "Cannot commit: Reference hash is out of date. " +
"Update the reference %s and try again", reference.getName());
} catch (HttpClientException ex) {
// Intentionally catch all nessie-client-exceptions here and not just the "timeout" variant
@@ -176,7 +176,7 @@ public class NessieTableOperations extends BaseMetastoreTableOperations {
throw new CommitStateUnknownException(ex);
} catch (NessieNotFoundException ex) {
throw new RuntimeException(
- String.format("Commit failed: Reference %s no longer exist", reference.getName()), ex);
+ String.format("Cannot commit: Reference %s no longer exists", reference.getName()), ex);
} finally {
if (delete) {
io().deleteFile(newMetadataLocation);
diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
new file mode 100644
index 0000000..bf11e03
--- /dev/null
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieCatalog.java
@@ -0,0 +1,162 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.nessie;
+
+import java.io.IOException;
+import java.net.URI;
+import java.nio.file.Path;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.catalog.CatalogTests;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.junit.jupiter.api.extension.RegisterExtension;
+import org.junit.jupiter.api.io.TempDir;
+import org.projectnessie.client.api.NessieApiV1;
+import org.projectnessie.client.http.HttpClientBuilder;
+import org.projectnessie.error.NessieConflictException;
+import org.projectnessie.error.NessieNotFoundException;
+import org.projectnessie.jaxrs.ext.NessieJaxRsExtension;
+import org.projectnessie.jaxrs.ext.NessieUri;
+import org.projectnessie.model.Branch;
+import org.projectnessie.model.Reference;
+import org.projectnessie.model.Tag;
+import org.projectnessie.server.store.TableCommitMetaStoreWorker;
+import org.projectnessie.versioned.persist.adapter.DatabaseAdapter;
+import org.projectnessie.versioned.persist.inmem.InmemoryDatabaseAdapterFactory;
+import org.projectnessie.versioned.persist.inmem.InmemoryTestConnectionProviderSource;
+import org.projectnessie.versioned.persist.tests.extension.DatabaseAdapterExtension;
+import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapter;
+import org.projectnessie.versioned.persist.tests.extension.NessieDbAdapterName;
+import org.projectnessie.versioned.persist.tests.extension.NessieExternalDatabase;
+
+@ExtendWith(DatabaseAdapterExtension.class)
+@NessieDbAdapterName(InmemoryDatabaseAdapterFactory.NAME)
+@NessieExternalDatabase(InmemoryTestConnectionProviderSource.class)
+public class TestNessieCatalog extends CatalogTests<NessieCatalog> {
+
+ @NessieDbAdapter(storeWorker = TableCommitMetaStoreWorker.class)
+ static DatabaseAdapter databaseAdapter;
+
+ @RegisterExtension
+ static NessieJaxRsExtension server = new NessieJaxRsExtension(() -> databaseAdapter);
+
+ @TempDir public Path temp;
+
+ private NessieCatalog catalog;
+ private NessieApiV1 api;
+ private Configuration hadoopConfig;
+ private String uri;
+
+ @BeforeEach
+ public void beforeEach(@NessieUri URI nessieUri) throws IOException {
+ this.uri = nessieUri.toString();
+ this.api = HttpClientBuilder.builder().withUri(this.uri).build(NessieApiV1.class);
+
+ resetData();
+
+ try {
+ api.createReference().reference(Branch.of("main", null)).create();
+ } catch (Exception e) {
+ // ignore, already created. Can't run this in BeforeAll as quarkus hasn't disabled auth
+ }
+
+ hadoopConfig = new Configuration();
+ catalog = initNessieCatalog("main");
+ }
+
+ @AfterEach
+ public void afterEach() throws NessieConflictException, NessieNotFoundException {
+ resetData();
+ try {
+ if (catalog != null) {
+ catalog.close();
+ }
+ api.close();
+ } finally {
+ catalog = null;
+ api = null;
+ hadoopConfig = null;
+ }
+ }
+
+ private void resetData() throws NessieConflictException, NessieNotFoundException {
+ for (Reference r : api.getAllReferences().get().getReferences()) {
+ if (r instanceof Branch) {
+ api.deleteBranch().branch((Branch) r).delete();
+ } else {
+ api.deleteTag().tag((Tag) r).delete();
+ }
+ }
+ api.createReference().reference(Branch.of("main", null)).create();
+ }
+
+ private NessieCatalog initNessieCatalog(String ref) {
+ NessieCatalog newCatalog = new NessieCatalog();
+ newCatalog.setConf(hadoopConfig);
+ newCatalog.initialize(
+ "nessie",
+ ImmutableMap.of(
+ "ref",
+ ref,
+ CatalogProperties.URI,
+ uri,
+ "auth-type",
+ "NONE",
+ CatalogProperties.WAREHOUSE_LOCATION,
+ temp.toUri().toString()));
+ return newCatalog;
+ }
+
+ @Override
+ protected NessieCatalog catalog() {
+ return catalog;
+ }
+
+ @Override
+ protected boolean supportsNamespaceProperties() {
+ return false;
+ }
+
+ @Override
+ protected boolean supportsServerSideRetry() {
+ // TODO: we do support retries, but a bunch of tests are currently failing
+ return false;
+ }
+
+ @Test
+ @Override
+ @Disabled(
+ "Nessie does not differentiate between table creates & updates, thus a concurrent transaction does not fail")
+ public void testConcurrentCreateTransaction() {
+ super.testConcurrentCreateTransaction();
+ }
+
+ @Test
+ @Override
+ @Disabled("currently not validated within doCommit")
+ public void testUUIDValidation() {
+ super.testUUIDValidation();
+ }
+}
diff --git a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
index 410c649..20d86ea 100644
--- a/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
+++ b/nessie/src/test/java/org/apache/iceberg/nessie/TestNessieTable.java
@@ -338,7 +338,7 @@ public class TestNessieTable extends BaseTestIceberg {
Assertions.assertThatThrownBy(() -> icebergTable.updateSchema().addColumn("data", Types.LongType.get()).commit())
.isInstanceOf(CommitFailedException.class)
.hasMessage(
- "Commit failed: Reference hash is out of date. Update the reference iceberg-table-test and try again");
+ "Cannot commit: Reference hash is out of date. Update the reference iceberg-table-test and try again");
}
@Test