You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2021/11/17 05:03:28 UTC
[spark] branch branch-3.2 updated: [SPARK-37702][SQL][FOLLOWUP] Store referred temp functions for CacheTableAsSelect
This is an automated email from the ASF dual-hosted git repository.
wenchen pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new c3628197 [SPARK-37702][SQL][FOLLOWUP] Store referred temp functions for CacheTableAsSelect
c3628197 is described below
commit c36281976d7ef497793fe3fccec83e30eb872b23
Author: Linhong Liu <li...@databricks.com>
AuthorDate: Wed Nov 17 13:01:49 2021 +0800
[SPARK-37702][SQL][FOLLOWUP] Store referred temp functions for CacheTableAsSelect
### What changes were proposed in this pull request?
This is a follow-up of PR #34546 which uses the `AnalysisContext` to store referred
temporary functions when creating temp views. But the `CacheTableAsSelect` also
needs to update the temporary functions because it's a temp view under the hood.
### Why are the changes needed?
Followup of #34546
### Does this PR introduce _any_ user-facing change?
No
### How was this patch tested?
newly added ut
Closes #34603 from linhongliu-db/SPARK-37702-followup.
Authored-by: Linhong Liu <li...@databricks.com>
Signed-off-by: Wenchen Fan <we...@databricks.com>
(cherry picked from commit b6c472ee5a6a2384a4e7b795a2d3a97a06fe77c5)
Signed-off-by: Wenchen Fan <we...@databricks.com>
---
.../apache/spark/sql/catalyst/plans/logical/v2Commands.scala | 10 ++++++++--
.../spark/sql/execution/datasources/v2/CacheTableExec.scala | 6 ++++--
.../sql/execution/datasources/v2/DataSourceV2Strategy.scala | 3 ++-
.../src/test/scala/org/apache/spark/sql/CachedTableSuite.scala | 10 ++++++++++
4 files changed, 24 insertions(+), 5 deletions(-)
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
index 06ca759..09d2e4a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/logical/v2Commands.scala
@@ -999,7 +999,8 @@ case class CacheTableAsSelect(
originalText: String,
isLazy: Boolean,
options: Map[String, String],
- isAnalyzed: Boolean = false) extends AnalysisOnlyCommand {
+ isAnalyzed: Boolean = false,
+ referredTempFunctions: Seq[String] = Seq.empty) extends AnalysisOnlyCommand {
override protected def withNewChildrenInternal(
newChildren: IndexedSeq[LogicalPlan]): CacheTableAsSelect = {
assert(!isAnalyzed)
@@ -1008,7 +1009,12 @@ case class CacheTableAsSelect(
override def childrenToAnalyze: Seq[LogicalPlan] = plan :: Nil
- override def markAsAnalyzed(ac: AnalysisContext): LogicalPlan = copy(isAnalyzed = true)
+ override def markAsAnalyzed(ac: AnalysisContext): LogicalPlan = {
+ copy(
+ isAnalyzed = true,
+ // Collect the referred temporary functions from AnalysisContext
+ referredTempFunctions = ac.referredTempFunctionNames.toSeq)
+ }
}
/**
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala
index aec32e4..8c14b5e 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/CacheTableExec.scala
@@ -88,7 +88,8 @@ case class CacheTableAsSelectExec(
query: LogicalPlan,
originalText: String,
override val isLazy: Boolean,
- override val options: Map[String, String]) extends BaseCacheTableExec {
+ override val options: Map[String, String],
+ referredTempFunctions: Seq[String]) extends BaseCacheTableExec {
override lazy val relationName: String = tempViewName
override lazy val planToCache: LogicalPlan = {
@@ -102,7 +103,8 @@ case class CacheTableAsSelectExec(
allowExisting = false,
replace = false,
viewType = LocalTempView,
- isAnalyzed = true
+ isAnalyzed = true,
+ referredTempFunctions = referredTempFunctions
).run(session)
dataFrameForCachedPlan.logicalPlan
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
index 1a50c32..2312550 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/v2/DataSourceV2Strategy.scala
@@ -411,7 +411,8 @@ class DataSourceV2Strategy(session: SparkSession) extends Strategy with Predicat
CacheTableExec(r.table, r.multipartIdentifier, r.isLazy, r.options) :: Nil
case r: CacheTableAsSelect =>
- CacheTableAsSelectExec(r.tempViewName, r.plan, r.originalText, r.isLazy, r.options) :: Nil
+ CacheTableAsSelectExec(
+ r.tempViewName, r.plan, r.originalText, r.isLazy, r.options, r.referredTempFunctions) :: Nil
case r: UncacheTable =>
def isTempView(table: LogicalPlan): Boolean = table match {
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
index db717d2..bad2815 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala
@@ -1643,4 +1643,14 @@ class CachedTableSuite extends QueryTest with SQLTestUtils
}
}
}
+
+ test("SPARK-37702: cache table with temporary function ") {
+ withUserDefinedFunction("udf" -> true) {
+ spark.udf.register("udf", (id: Int) => id + 1)
+ withTempView("cached_t") {
+ sql("CACHE TABLE cached_t as SELECT udf(id) FROM VALUES (1), (2) t(id)")
+ checkAnswer(sql("SELECT * FROM cached_t"), Row(2) :: Row(3) :: Nil)
+ }
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org