You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2022/08/25 02:33:00 UTC
[spark] branch master updated: [SPARK-40055][SQL] listCatalogs should also return spark_catalog even when spark_catalog implementation is defaultSessionCatalog
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 245ed9002d1 [SPARK-40055][SQL] listCatalogs should also return spark_catalog even when spark_catalog implementation is defaultSessionCatalog
245ed9002d1 is described below
commit 245ed9002d1da47b96d0fada1745ba6686d47464
Author: Rui Wang <ru...@databricks.com>
AuthorDate: Thu Aug 25 10:32:37 2022 +0800
[SPARK-40055][SQL] listCatalogs should also return spark_catalog even when spark_catalog implementation is defaultSessionCatalog
### What changes were proposed in this pull request?
This is to fix a bug in CatalogManager that `spark_catalog` is not updated into `catalogs` when its implementation is defaultSessionCatalog, thus leads to missing result (which is `spark_catalog`) when calling `spark.catalog.listCatalogs()`.
### Why are the changes needed?
This is bug fix. We are supposed to see `spark_catalog` from `listCatalogs` call.
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
UT
Closes #37488 from amaliujia/SPARK-40055.
Authored-by: Rui Wang <ru...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../org/apache/spark/sql/connector/catalog/CatalogManager.scala | 2 +-
.../scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala | 5 +++++
.../test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala | 1 +
.../src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala | 3 ++-
4 files changed, 9 insertions(+), 2 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala
index 0380621b285..e7afb733362 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/connector/catalog/CatalogManager.scala
@@ -84,7 +84,7 @@ class CatalogManager(
private[sql] def v2SessionCatalog: CatalogPlugin = {
conf.getConf(SQLConf.V2_SESSION_CATALOG_IMPLEMENTATION).map { _ =>
catalogs.getOrElseUpdate(SESSION_CATALOG_NAME, loadV2SessionCatalog())
- }.getOrElse(defaultSessionCatalog)
+ }.getOrElse(catalogs.getOrElseUpdate(SESSION_CATALOG_NAME, defaultSessionCatalog))
}
private var _currentNamespace: Option[Array[String]] = None
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
index 1a277ae957a..6dd302d798f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DataSourceV2SQLSuite.scala
@@ -258,6 +258,7 @@ class DataSourceV2SQLSuiteV1Filter extends DataSourceV2SQLSuite with AlterTableT
withSQLConf(SQLConf.LEGACY_CREATE_HIVE_TABLE_BY_DEFAULT.key -> "false") {
// unset this config to use the default v2 session catalog.
spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+ spark.sessionState.catalogManager.reset()
val testCatalog = catalog("testcat").asTableCatalog
sql("CREATE TABLE testcat.t1 (id int)")
@@ -727,6 +728,7 @@ class DataSourceV2SQLSuiteV1Filter extends DataSourceV2SQLSuite with AlterTableT
test("CreateTableAsSelect: v2 session catalog can load v1 source table") {
// unset this config to use the default v2 session catalog.
spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+ spark.sessionState.catalogManager.reset()
val df = spark.createDataFrame(Seq((1L, "a"), (2L, "b"), (3L, "c"))).toDF("id", "data")
df.createOrReplaceTempView("source")
@@ -1010,6 +1012,7 @@ class DataSourceV2SQLSuiteV1Filter extends DataSourceV2SQLSuite with AlterTableT
spark.conf.set(
V2_SESSION_CATALOG_IMPLEMENTATION.key, classOf[InMemoryTableSessionCatalog].getName)
}
+ spark.sessionState.catalogManager.reset()
withTable("t") {
sql(s"CREATE TABLE t USING $format AS SELECT 1 AS i")
@@ -1822,6 +1825,7 @@ class DataSourceV2SQLSuiteV1Filter extends DataSourceV2SQLSuite with AlterTableT
test("SPARK-30001: session catalog name can be specified in SQL statements") {
// unset this config to use the default v2 session catalog.
spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+ spark.sessionState.catalogManager.reset()
withTable("t") {
sql("CREATE TABLE t USING json AS SELECT 1 AS i")
@@ -1886,6 +1890,7 @@ class DataSourceV2SQLSuiteV1Filter extends DataSourceV2SQLSuite with AlterTableT
test("SPARK-30094: current namespace is used during table resolution") {
// unset this config to use the default v2 session catalog.
spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+ spark.sessionState.catalogManager.reset()
withTable("spark_catalog.default.t", "testcat.ns.t") {
sql("CREATE TABLE t USING parquet AS SELECT 1")
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala
index 5ed64df6280..74e3567be38 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/connector/DeleteFromTests.scala
@@ -102,6 +102,7 @@ trait DeleteFromTests extends DatasourceV2SQLBase {
test("DeleteFrom: DELETE is only supported with v2 tables") {
// unset this config to use the default v2 session catalog.
spark.conf.unset(V2_SESSION_CATALOG_IMPLEMENTATION.key)
+ spark.sessionState.catalogManager.reset()
val v1Table = "tbl"
withTable(v1Table) {
sql(s"CREATE TABLE $v1Table" +
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
index ab26a4fcc35..851a87ad2c0 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala
@@ -869,7 +869,8 @@ class CatalogSuite extends SharedSparkSession with AnalysisTest with BeforeAndAf
assert(spark.catalog.currentCatalog().equals("testcat"))
spark.catalog.setCurrentCatalog("spark_catalog")
assert(spark.catalog.currentCatalog().equals("spark_catalog"))
- assert(spark.catalog.listCatalogs().collect().map(c => c.name).toSet == Set("testcat"))
+ assert(spark.catalog.listCatalogs().collect().map(c => c.name).toSet
+ == Set("spark_catalog", "testcat"))
}
test("SPARK-39583: Make RefreshTable be compatible with 3 layer namespace") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org