[spark] branch branch-3.2 updated: [SPARK-36466][SQL] Table in unloaded catalog referenced by view should load correctly

[we...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

wenchen pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-3.2 by this push:
     new 45acd00  [SPARK-36466][SQL] Table in unloaded catalog referenced by view should load correctly
45acd00 is described below

commit 45acd00dd6abe3bc0ed0fdb6b748b207884ae777
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Tue Aug 10 17:31:21 2021 +0800

    [SPARK-36466][SQL] Table in unloaded catalog referenced by view should load correctly
    
    ### What changes were proposed in this pull request?
    
    Retain `spark.sql.catalog.*` confs when resolving view.
    
    ### Why are the changes needed?
    
    Currently, if a view in default catalog ref a table in another catalog (e.g. jdbc), `org.apache.spark.sql.AnalysisException: Table or view not found: cat.t` will be thrown on accessing the view if the catalog has not been loaded yet.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes, bug fix.
    
    ### How was this patch tested?
    
    Add UT.
    
    Closes #33692 from pan3793/SPARK-36466.
    
    Authored-by: Cheng Pan <ch...@apache.org>
    Signed-off-by: Wenchen Fan <we...@databricks.com>
    (cherry picked from commit 7f56b73cad0e38498aa3b2bdf6a5c22388175dea)
    Signed-off-by: Wenchen Fan <we...@databricks.com>
---
 .../plans/logical/basicLogicalOperators.scala      |  2 +-
 .../spark/sql/execution/SQLViewTestSuite.scala     | 37 ++++++++++++++++++++++
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
index 4633a36..d9efdc1 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/basicLogicalOperators.scala
@@ -599,7 +599,7 @@ object View {
         "spark.sql.hive.convertMetastoreOrc",
         "spark.sql.hive.convertInsertingPartitionedTable",
         "spark.sql.hive.convertMetastoreCtas"
-      ).contains(key))
+      ).contains(key) || key.startsWith("spark.sql.catalog."))
     for ((k, v) <- configs ++ retainedConfigs) {
       sqlConf.settings.put(k, v)
     }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
index bc64f51..8383d44 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SQLViewTestSuite.scala
@@ -17,9 +17,12 @@
 
 package org.apache.spark.sql.execution
 
+import scala.collection.JavaConverters._
+
 import org.apache.spark.sql.{AnalysisException, QueryTest, Row}
 import org.apache.spark.sql.catalyst.TableIdentifier
 import org.apache.spark.sql.catalyst.plans.logical.Repartition
+import org.apache.spark.sql.connector.catalog._
 import org.apache.spark.sql.internal.SQLConf._
 import org.apache.spark.sql.test.{SharedSparkSession, SQLTestUtils}
 import org.apache.spark.sql.types.{IntegerType, StructField, StructType}
@@ -394,6 +397,20 @@ class GlobalTempViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
   }
 }
 
+class OneTableCatalog extends InMemoryCatalog {
+  override def loadTable(ident: Identifier): Table = {
+    if (ident.namespace.isEmpty && ident.name == "t") {
+      new InMemoryTable(
+        "t",
+        StructType.fromDDL("c1 INT"),
+        Array.empty,
+        Map.empty[String, String].asJava)
+    } else {
+      super.loadTable(ident)
+    }
+  }
+}
+
 class PersistedViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
   private def db: String = "default"
   override protected def viewTypeString: String = "VIEW"
@@ -493,4 +510,24 @@ class PersistedViewTestSuite extends SQLViewTestSuite with SharedSparkSession {
       }
     }
   }
+
+  test("SPARK-36466: Table in unloaded catalog referenced by view should load correctly") {
+    val viewName = "v"
+    val tableInOtherCatalog = "cat.t"
+    try {
+      spark.conf.set("spark.sql.catalog.cat", classOf[OneTableCatalog].getName)
+      withTable(tableInOtherCatalog) {
+        withView(viewName) {
+          createView(viewName, s"SELECT count(*) AS cnt FROM $tableInOtherCatalog")
+          checkViewOutput(viewName, Seq(Row(0)))
+          spark.sessionState.catalogManager.reset()
+          checkViewOutput(viewName, Seq(Row(0)))
+        }
+      }
+    } finally {
+      spark.sessionState.catalog.reset()
+      spark.sessionState.catalogManager.reset()
+      spark.sessionState.conf.clear()
+    }
+  }
 }

---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org