You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2022/06/23 20:18:42 UTC
[GitHub] [spark] amaliujia commented on a diff in pull request #36968: [SPARK-39235][SQL] make getDatabase compatible with 3 layer namespace
amaliujia commented on code in PR #36968:
URL: https://github.com/apache/spark/pull/36968#discussion_r905419637
##########
sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala:
##########
@@ -743,4 +743,25 @@ class CatalogSuite extends SharedSparkSession with AnalysisTest with BeforeAndAf
val catalogName2 = "catalog_not_exists"
assert(!spark.catalog.databaseExists(Array(catalogName2, dbName).mkString(".")))
}
+
+ test("three layer namespace compatibility - get database") {
+ Seq(("testcat", "somedb"), ("testcat", "ns.somedb")).foreach { case (catalog, dbName) =>
Review Comment:
how about adding another case of ("spark_catalog", "somedb")?
##########
sql/core/src/test/scala/org/apache/spark/sql/internal/CatalogSuite.scala:
##########
@@ -743,4 +743,25 @@ class CatalogSuite extends SharedSparkSession with AnalysisTest with BeforeAndAf
val catalogName2 = "catalog_not_exists"
assert(!spark.catalog.databaseExists(Array(catalogName2, dbName).mkString(".")))
}
+
+ test("three layer namespace compatibility - get database") {
+ Seq(("testcat", "somedb"), ("testcat", "ns.somedb")).foreach { case (catalog, dbName) =>
+ val qualifiedDb = s"$catalog.$dbName"
+ // TODO test properties? WITH DBPROPERTIES (prop='val')
Review Comment:
+1 to test the complete namespace metadata.
##########
sql/core/src/main/scala/org/apache/spark/sql/internal/CatalogImpl.scala:
##########
@@ -243,7 +243,29 @@ class CatalogImpl(sparkSession: SparkSession) extends Catalog {
* `Database` can be found.
*/
override def getDatabase(dbName: String): Database = {
- makeDatabase(dbName)
+ // `dbName` could be either a single database name (behavior in Spark 3.3 and prior) or a
+ // qualified namespace with catalog name. To maintain backwards compatibility, we first assume
+ // it's a single database name and return the database from sessionCatalog if it exists.
+ // Otherwise we try 3-part name parsing and locate the database. If the parased identifier
+ // contains both catalog name and database name, we then search the database in the catalog.
+ if (sessionCatalog.databaseExists(dbName) || sessionCatalog.isGlobalTempViewDB(dbName)) {
+ makeDatabase(dbName)
+ } else {
+ val ident = sparkSession.sessionState.sqlParser.parseMultipartIdentifier(dbName)
+ val plan = UnresolvedNamespace(ident)
+ val resolved = sparkSession.sessionState.executePlan(plan).analyzed
+ val db = ident.tail
+ val metadata = resolved match {
+ case ResolvedNamespace(catalog: SupportsNamespaces, _) =>
+ catalog.loadNamespaceMetadata(db.toArray)
+ // TODO what to do if it doesn't support namespaces
+ case _ => throw new RuntimeException(s"unexpected catalog resolved: $resolved")
Review Comment:
cc @cloud-fan
WDYT?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org