You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by dw...@apache.org on 2024/12/16 18:34:10 UTC
(iceberg) branch main updated: Spark 3.4: Add REST catalog to Spark integration tests (#11698)
This is an automated email from the ASF dual-hosted git repository.
dweeks pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 791d0fa6cf Spark 3.4: Add REST catalog to Spark integration tests (#11698)
791d0fa6cf is described below
commit 791d0fa6cf9408ad3b8a1ef633a059ba1f2255ad
Author: Eduard Tudenhoefner <et...@gmail.com>
AuthorDate: Mon Dec 16 19:34:05 2024 +0100
Spark 3.4: Add REST catalog to Spark integration tests (#11698)
---
spark/v3.4/build.gradle | 22 +++++
.../spark/extensions/TestMetadataTables.java | 2 +
.../extensions/TestRemoveOrphanFilesProcedure.java | 13 ++-
.../org/apache/iceberg/rest/RESTServerRule.java | 107 +++++++++++++++++++++
.../apache/iceberg/spark/SparkCatalogConfig.java | 4 +
.../apache/iceberg/spark/SparkCatalogTestBase.java | 10 ++
.../iceberg/spark/SparkTestBaseWithCatalog.java | 66 +++++++++++--
.../spark/actions/TestComputeTableStatsAction.java | 1 +
.../apache/iceberg/spark/sql/TestAlterTable.java | 13 ++-
.../iceberg/spark/sql/TestCreateTableAsSelect.java | 2 +
.../apache/iceberg/spark/sql/TestNamespaceSQL.java | 14 ++-
.../apache/iceberg/spark/sql/TestRefreshTable.java | 8 +-
12 files changed, 242 insertions(+), 20 deletions(-)
diff --git a/spark/v3.4/build.gradle b/spark/v3.4/build.gradle
index a34ea6b928..3273968558 100644
--- a/spark/v3.4/build.gradle
+++ b/spark/v3.4/build.gradle
@@ -107,9 +107,14 @@ project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}") {
testImplementation project(path: ':iceberg-api', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-data', configuration: 'testArtifacts')
+ testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
+ transitive = false
+ }
testImplementation libs.sqlite.jdbc
testImplementation libs.awaitility
testImplementation libs.junit.vintage.engine
+ // runtime dependencies for running REST Catalog based integration test
+ testRuntimeOnly libs.jetty.servlet
}
test {
@@ -174,6 +179,12 @@ project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVer
testImplementation project(path: ':iceberg-core', configuration: 'testArtifacts')
testImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
testImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+ testImplementation (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
+ transitive = false
+ }
+ // runtime dependencies for running REST Catalog based integration test
+ testRuntimeOnly libs.jetty.servlet
+ testRuntimeOnly libs.sqlite.jdbc
testImplementation libs.avro.avro
testImplementation libs.parquet.hadoop
@@ -252,6 +263,17 @@ project(":iceberg-spark:iceberg-spark-runtime-${sparkMajorVersion}_${scalaVersio
integrationImplementation project(path: ':iceberg-hive-metastore', configuration: 'testArtifacts')
integrationImplementation project(path: ":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
integrationImplementation project(path: ":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}", configuration: 'testArtifacts')
+
+ // runtime dependencies for running Hive Catalog based integration test
+ integrationRuntimeOnly project(':iceberg-hive-metastore')
+ // runtime dependencies for running REST Catalog based integration test
+ integrationRuntimeOnly project(path: ':iceberg-core', configuration: 'testArtifacts')
+ integrationRuntimeOnly (project(path: ':iceberg-open-api', configuration: 'testFixturesRuntimeElements')) {
+ transitive = false
+ }
+ integrationRuntimeOnly libs.jetty.servlet
+ integrationRuntimeOnly libs.sqlite.jdbc
+
// Not allowed on our classpath, only the runtime jar is allowed
integrationCompileOnly project(":iceberg-spark:iceberg-spark-extensions-${sparkMajorVersion}_${scalaVersion}")
integrationCompileOnly project(":iceberg-spark:iceberg-spark-${sparkMajorVersion}_${scalaVersion}")
diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
index 2143916384..37d590f974 100644
--- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
+++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestMetadataTables.java
@@ -528,6 +528,8 @@ public class TestMetadataTables extends SparkExtensionsTestBase {
spark.createDataFrame(newRecords, newSparkSchema).coalesce(1).writeTo(tableName).append();
+ table.refresh();
+
Long currentSnapshotId = table.currentSnapshot().snapshotId();
Dataset<Row> actualFilesDs =
diff --git a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java
index 01e341eead..2b99eecab3 100644
--- a/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java
+++ b/spark/v3.4/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRemoveOrphanFilesProcedure.java
@@ -460,12 +460,15 @@ public class TestRemoveOrphanFilesProcedure extends SparkExtensionsTestBase {
Table table = Spark3Util.loadIcebergTable(spark, tableName);
String statsFileName = "stats-file-" + UUID.randomUUID();
+ String location = table.location();
+ // not every catalog will return file proto for local directories
+ // i.e. Hadoop and Hive Catalog do, Jdbc and REST do not
+ if (!location.startsWith("file:")) {
+ location = "file:" + location;
+ }
+
File statsLocation =
- new File(new URI(table.location()))
- .toPath()
- .resolve("data")
- .resolve(statsFileName)
- .toFile();
+ new File(new URI(location)).toPath().resolve("data").resolve(statsFileName).toFile();
StatisticsFile statisticsFile;
try (PuffinWriter puffinWriter = Puffin.write(Files.localOutput(statsLocation)).build()) {
long snapshotId = table.currentSnapshot().snapshotId();
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/rest/RESTServerRule.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/rest/RESTServerRule.java
new file mode 100644
index 0000000000..f42106e6d8
--- /dev/null
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/rest/RESTServerRule.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.rest;
+
+import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.relocated.com.google.common.collect.Maps;
+import org.junit.rules.ExternalResource;
+
+/**
+ * This class is to make the {@link RESTCatalogServer} usable for JUnit4 in a similar way to {@link
+ * RESTServerExtension}.
+ */
+public class RESTServerRule extends ExternalResource {
+ public static final String FREE_PORT = "0";
+
+ private volatile RESTCatalogServer localServer;
+ private RESTCatalog client;
+ private final Map<String, String> config;
+
+ public RESTServerRule() {
+ config = Maps.newHashMap();
+ }
+
+ public RESTServerRule(Map<String, String> config) {
+ Map<String, String> conf = Maps.newHashMap(config);
+ if (conf.containsKey(RESTCatalogServer.REST_PORT)
+ && conf.get(RESTCatalogServer.REST_PORT).equals(FREE_PORT)) {
+ conf.put(RESTCatalogServer.REST_PORT, String.valueOf(RCKUtils.findFreePort()));
+ }
+ this.config = conf;
+ }
+
+ public Map<String, String> config() {
+ return config;
+ }
+
+ public RESTCatalog client() {
+ if (null == client) {
+ try {
+ maybeInitClientAndServer();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ return client;
+ }
+
+ public String uri() {
+ return client().properties().get(CatalogProperties.URI);
+ }
+
+ private void maybeInitClientAndServer() throws Exception {
+ if (null == localServer) {
+ synchronized (this) {
+ if (null == localServer) {
+ this.localServer = new RESTCatalogServer(config);
+ this.localServer.start(false);
+ this.client = RCKUtils.initCatalogClient(config);
+ }
+ }
+ }
+ }
+
+ @Override
+ protected void before() throws Throwable {
+ maybeShutdownClientAndServer();
+ maybeInitClientAndServer();
+ }
+
+ @Override
+ protected void after() {
+ maybeShutdownClientAndServer();
+ }
+
+ private void maybeShutdownClientAndServer() {
+ try {
+ if (localServer != null) {
+ localServer.stop();
+ localServer = null;
+ }
+ if (client != null) {
+ client.close();
+ client = null;
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+}
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
index abfd7da0c7..e03f500dc4 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogConfig.java
@@ -34,6 +34,10 @@ public enum SparkCatalogConfig {
"testhadoop",
SparkCatalog.class.getName(),
ImmutableMap.of("type", "hadoop", "cache-enabled", "false")),
+ REST(
+ "testrest",
+ SparkCatalog.class.getName(),
+ ImmutableMap.of("type", "rest", "cache-enabled", "false")),
SPARK(
"spark_catalog",
SparkSessionCatalog.class.getName(),
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java
index 89323c2610..6b2b9a1b80 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkCatalogTestBase.java
@@ -19,6 +19,8 @@
package org.apache.iceberg.spark;
import java.util.Map;
+import org.apache.iceberg.CatalogProperties;
+import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
@@ -45,6 +47,14 @@ public abstract class SparkCatalogTestBase extends SparkTestBaseWithCatalog {
SparkCatalogConfig.SPARK.catalogName(),
SparkCatalogConfig.SPARK.implementation(),
SparkCatalogConfig.SPARK.properties()
+ },
+ {
+ SparkCatalogConfig.REST.catalogName(),
+ SparkCatalogConfig.REST.implementation(),
+ ImmutableMap.builder()
+ .putAll(SparkCatalogConfig.REST.properties())
+ .put(CatalogProperties.URI, REST_SERVER_RULE.uri())
+ .build()
}
};
}
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java
index d5708c9e57..1c5d9b711b 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/SparkTestBaseWithCatalog.java
@@ -18,6 +18,12 @@
*/
package org.apache.iceberg.spark;
+import static org.apache.iceberg.CatalogProperties.CATALOG_IMPL;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HADOOP;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_HIVE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_REST;
+
import java.io.File;
import java.io.IOException;
import java.util.Map;
@@ -31,20 +37,42 @@ import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.SupportsNamespaces;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.hadoop.HadoopCatalog;
+import org.apache.iceberg.inmemory.InMemoryCatalog;
+import org.apache.iceberg.rest.RESTCatalog;
+import org.apache.iceberg.rest.RESTCatalogServer;
+import org.apache.iceberg.rest.RESTServerRule;
import org.apache.iceberg.util.PropertyUtil;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
+import org.junit.ClassRule;
import org.junit.Rule;
import org.junit.rules.TemporaryFolder;
public abstract class SparkTestBaseWithCatalog extends SparkTestBase {
protected static File warehouse = null;
+ @ClassRule
+ public static final RESTServerRule REST_SERVER_RULE =
+ new RESTServerRule(
+ Map.of(
+ RESTCatalogServer.REST_PORT,
+ RESTServerRule.FREE_PORT,
+ // In-memory sqlite database by default is private to the connection that created it.
+ // If more than 1 jdbc connection backed by in-memory sqlite is created behind one
+ // JdbcCatalog, then different jdbc connections could provide different views of table
+ // status even belonging to the same catalog. Reference:
+ // https://www.sqlite.org/inmemorydb.html
+ CatalogProperties.CLIENT_POOL_SIZE,
+ "1"));
+
+ protected static RESTCatalog restCatalog;
+
@BeforeClass
public static void createWarehouse() throws IOException {
SparkTestBaseWithCatalog.warehouse = File.createTempFile("warehouse", null);
Assert.assertTrue(warehouse.delete());
+ restCatalog = REST_SERVER_RULE.client();
}
@AfterClass
@@ -60,8 +88,8 @@ public abstract class SparkTestBaseWithCatalog extends SparkTestBase {
protected final String catalogName;
protected final Map<String, String> catalogConfig;
- protected final Catalog validationCatalog;
- protected final SupportsNamespaces validationNamespaceCatalog;
+ protected Catalog validationCatalog;
+ protected SupportsNamespaces validationNamespaceCatalog;
protected final TableIdentifier tableIdent = TableIdentifier.of(Namespace.of("default"), "table");
protected final String tableName;
@@ -77,11 +105,7 @@ public abstract class SparkTestBaseWithCatalog extends SparkTestBase {
String catalogName, String implementation, Map<String, String> config) {
this.catalogName = catalogName;
this.catalogConfig = config;
- this.validationCatalog =
- catalogName.equals("testhadoop")
- ? new HadoopCatalog(spark.sessionState().newHadoopConf(), "file:" + warehouse)
- : catalog;
- this.validationNamespaceCatalog = (SupportsNamespaces) validationCatalog;
+ configureValidationCatalog();
spark.conf().set("spark.sql.catalog." + catalogName, implementation);
config.forEach(
@@ -127,4 +151,32 @@ public abstract class SparkTestBaseWithCatalog extends SparkTestBase {
TableProperties.DELETE_PLANNING_MODE,
planningMode.modeName());
}
+
+ private void configureValidationCatalog() {
+ if (catalogConfig.containsKey(ICEBERG_CATALOG_TYPE)) {
+ switch (catalogConfig.get(ICEBERG_CATALOG_TYPE)) {
+ case ICEBERG_CATALOG_TYPE_HADOOP:
+ this.validationCatalog =
+ new HadoopCatalog(spark.sessionState().newHadoopConf(), "file:" + warehouse);
+ break;
+ case ICEBERG_CATALOG_TYPE_REST:
+ this.validationCatalog = restCatalog;
+ break;
+ case ICEBERG_CATALOG_TYPE_HIVE:
+ this.validationCatalog = catalog;
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown catalog type");
+ }
+ } else if (catalogConfig.containsKey(CATALOG_IMPL)) {
+ switch (catalogConfig.get(CATALOG_IMPL)) {
+ case "org.apache.iceberg.inmemory.InMemoryCatalog":
+ this.validationCatalog = new InMemoryCatalog();
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown catalog impl");
+ }
+ }
+ this.validationNamespaceCatalog = (SupportsNamespaces) validationCatalog;
+ }
}
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java
index 88805a070c..26d53832a4 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/actions/TestComputeTableStatsAction.java
@@ -116,6 +116,7 @@ public class TestComputeTableStatsAction extends SparkCatalogTestBase {
new SimpleRecord(4, "d"));
spark.createDataset(records, Encoders.bean(SimpleRecord.class)).writeTo(tableName).append();
SparkActions actions = SparkActions.get();
+ table.refresh();
ComputeTableStats.Result results =
actions.computeTableStats(table).columns("id", "data").execute();
assertThat(results).isNotNull();
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java
index 89857a65e2..e5c6fcd0aa 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestAlterTable.java
@@ -18,8 +18,11 @@
*/
package org.apache.iceberg.spark.sql;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_REST;
import static org.assertj.core.api.Assertions.assertThat;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assumptions.assumeThat;
import java.util.Map;
import org.apache.iceberg.catalog.Namespace;
@@ -32,7 +35,6 @@ import org.apache.spark.SparkException;
import org.apache.spark.sql.AnalysisException;
import org.junit.After;
import org.junit.Assert;
-import org.junit.Assume;
import org.junit.Before;
import org.junit.Test;
@@ -293,8 +295,13 @@ public class TestAlterTable extends SparkCatalogTestBase {
@Test
public void testTableRename() {
- Assume.assumeFalse(
- "Hadoop catalog does not support rename", validationCatalog instanceof HadoopCatalog);
+ assumeThat(catalogConfig.get(ICEBERG_CATALOG_TYPE))
+ .as(
+ "need to fix https://github.com/apache/iceberg/issues/11154 before enabling this for the REST catalog")
+ .isNotEqualTo(ICEBERG_CATALOG_TYPE_REST);
+ assumeThat(validationCatalog)
+ .as("Hadoop catalog does not support rename")
+ .isNotInstanceOf(HadoopCatalog.class);
Assert.assertTrue("Initial name should exist", validationCatalog.tableExists(tableIdent));
Assert.assertFalse("New name should not exist", validationCatalog.tableExists(renamedIdent));
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
index 13dd8bc853..ff067291c8 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTableAsSelect.java
@@ -394,6 +394,8 @@ public class TestCreateTableAsSelect extends SparkCatalogTestBase {
+ "FROM %s ORDER BY 3, 1",
tableName, sourceName);
+ rtasTable.refresh();
+
Schema expectedSchema =
new Schema(
Types.NestedField.optional(1, "id", Types.LongType.get()),
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
index a0573c5b09..f7ad4a6e91 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestNamespaceSQL.java
@@ -18,7 +18,10 @@
*/
package org.apache.iceberg.spark.sql;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE;
+import static org.apache.iceberg.CatalogUtil.ICEBERG_CATALOG_TYPE_REST;
import static org.assertj.core.api.Assertions.assertThatThrownBy;
+import static org.assertj.core.api.Assumptions.assumeThat;
import java.io.File;
import java.util.List;
@@ -27,6 +30,7 @@ import java.util.Set;
import java.util.stream.Collectors;
import org.apache.iceberg.catalog.Namespace;
import org.apache.iceberg.catalog.TableIdentifier;
+import org.apache.iceberg.exceptions.BadRequestException;
import org.apache.iceberg.exceptions.NamespaceNotEmptyException;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableSet;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
@@ -67,6 +71,9 @@ public class TestNamespaceSQL extends SparkCatalogTestBase {
@Test
public void testDefaultNamespace() {
Assume.assumeFalse("Hadoop has no default namespace configured", isHadoopCatalog);
+ assumeThat(catalogConfig.get(ICEBERG_CATALOG_TYPE))
+ .as("REST has no default namespace configured")
+ .isNotEqualTo(ICEBERG_CATALOG_TYPE_REST);
sql("USE %s", catalogName);
@@ -105,8 +112,8 @@ public class TestNamespaceSQL extends SparkCatalogTestBase {
"Table should exist", validationCatalog.tableExists(TableIdentifier.of(NS, "table")));
assertThatThrownBy(() -> sql("DROP NAMESPACE %s", fullNamespace))
- .isInstanceOf(NamespaceNotEmptyException.class)
- .hasMessageStartingWith("Namespace db is not empty.");
+ .isInstanceOfAny(NamespaceNotEmptyException.class, BadRequestException.class)
+ .hasMessageContaining("Namespace db is not empty.");
sql("DROP TABLE %s.table", fullNamespace);
}
@@ -141,7 +148,8 @@ public class TestNamespaceSQL extends SparkCatalogTestBase {
List<Object[]> namespaces = sql("SHOW NAMESPACES IN %s", catalogName);
- if (isHadoopCatalog) {
+ if (isHadoopCatalog
+ || catalogConfig.get(ICEBERG_CATALOG_TYPE).equals(ICEBERG_CATALOG_TYPE_REST)) {
Assert.assertEquals("Should have 1 namespace", 1, namespaces.size());
Set<String> namespaceNames =
namespaces.stream().map(arr -> arr[0].toString()).collect(Collectors.toSet());
diff --git a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
index 7da2dc0882..ccbad7311a 100644
--- a/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
+++ b/spark/v3.4/spark/src/test/java/org/apache/iceberg/spark/sql/TestRefreshTable.java
@@ -20,6 +20,7 @@ package org.apache.iceberg.spark.sql;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.iceberg.DataFile;
import org.apache.iceberg.Table;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
@@ -50,8 +51,11 @@ public class TestRefreshTable extends SparkCatalogTestBase {
public void testRefreshCommand() {
// We are not allowed to change the session catalog after it has been initialized, so build a
// new one
- if (catalogName.equals(SparkCatalogConfig.SPARK.catalogName())
- || catalogName.equals(SparkCatalogConfig.HADOOP.catalogName())) {
+ if (Set.of(
+ SparkCatalogConfig.SPARK.catalogName(),
+ SparkCatalogConfig.HADOOP.catalogName(),
+ SparkCatalogConfig.REST.catalogName())
+ .contains(catalogName)) {
spark.conf().set("spark.sql.catalog." + catalogName + ".cache-enabled", true);
spark = spark.cloneSession();
}