You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2024/05/30 07:40:12 UTC
(paimon) 20/31: [spark] Fix drop database cascade when database with view with SparkGenericCatalog (#3347)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch release-0.8
in repository https://gitbox.apache.org/repos/asf/paimon.git
commit 0b63cb8a965b6aeec27ef9fb5f033da6755755f7
Author: Zouxxyy <zo...@alibaba-inc.com>
AuthorDate: Mon May 20 10:52:53 2024 +0800
[spark] Fix drop database cascade when database with view with SparkGenericCatalog (#3347)
---
.../main/java/org/apache/paimon/spark/SparkGenericCatalog.java | 9 ++++++++-
.../test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala | 5 +++++
.../org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala | 8 ++++++++
3 files changed, 21 insertions(+), 1 deletion(-)
diff --git a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java
index b947523fe..9576379f8 100644
--- a/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java
+++ b/paimon-spark/paimon-spark-common/src/main/java/org/apache/paimon/spark/SparkGenericCatalog.java
@@ -134,7 +134,14 @@ public class SparkGenericCatalog extends SparkBaseCatalog implements CatalogExte
throws NoSuchNamespaceException, NonEmptyNamespaceException {
if (namespace.length == 1 && namespaceExists(namespace) && cascade) {
for (Identifier table : listTables(namespace)) {
- dropTable(table);
+ try {
+ dropTable(table);
+ } catch (Exception e) {
+ LOG.warn(
+ "Failed to drop {}, fallback to use sessionCatalog to drop, for {}",
+ table,
+ e.getMessage());
+ }
}
}
return asNamespaceCatalog().dropNamespace(namespace, cascade);
diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala
index 6ccf08016..814001bc9 100644
--- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala
+++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/PaimonSparkTestBase.scala
@@ -84,6 +84,11 @@ class PaimonSparkTestBase
}
}
+ protected def reset(): Unit = {
+ afterAll()
+ beforeAll()
+ }
+
/** Default is paimon catalog */
override protected def beforeEach(): Unit = {
super.beforeAll()
diff --git a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
index c03f61bf6..c542633dc 100644
--- a/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
+++ b/paimon-spark/paimon-spark-common/src/test/scala/org/apache/paimon/spark/sql/DDLWithHiveCatalogTestBase.scala
@@ -99,7 +99,9 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase {
}
}
+ // Since we created a new sparkContext, we need to stop it and reset the default sparkContext
reusedSpark.stop()
+ reset()
}
test("Paimon DDL with hive catalog: drop database cascade which contains paimon table") {
@@ -111,6 +113,12 @@ abstract class DDLWithHiveCatalogTestBase extends PaimonHiveTestBase {
spark.sql(s"CREATE DATABASE paimon_db")
spark.sql(s"USE paimon_db")
spark.sql(s"CREATE TABLE paimon_tbl (id int, name string, dt string) using paimon")
+ // Currently, only spark_catalog supports create other table or view
+ if (catalogName.equals(sparkCatalogName)) {
+ spark.sql(s"CREATE TABLE parquet_tbl (id int, name string, dt string) using parquet")
+ spark.sql(s"CREATE VIEW parquet_tbl_view AS SELECT * FROM parquet_tbl")
+ spark.sql(s"CREATE VIEW paimon_tbl_view AS SELECT * FROM paimon_tbl")
+ }
spark.sql(s"USE default")
spark.sql(s"DROP DATABASE paimon_db CASCADE")
}