You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2022/01/05 08:00:28 UTC
[spark] branch branch-3.1 updated: Revert "[SPARK-37779][SQL] Make ColumnarToRowExec plan canonicalizable after (de)serialization"
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 94a69ff Revert "[SPARK-37779][SQL] Make ColumnarToRowExec plan canonicalizable after (de)serialization"
94a69ff is described below
commit 94a69ffe0d26369004dcca8d9cb30057d2dcbf96
Author: Dongjoon Hyun <do...@apache.org>
AuthorDate: Tue Jan 4 23:59:16 2022 -0800
Revert "[SPARK-37779][SQL] Make ColumnarToRowExec plan canonicalizable after (de)serialization"
This reverts commit e17ab6e995a213e442e91df168e87fb724672613.
---
.../org/apache/spark/sql/execution/Columnar.scala | 3 +--
.../apache/spark/sql/execution/SparkPlanSuite.scala | 19 -------------------
2 files changed, 1 insertion(+), 21 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
index e2bdf4e..ccb525d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/Columnar.scala
@@ -64,8 +64,7 @@ trait ColumnarToRowTransition extends UnaryExecNode
* [[MapPartitionsInRWithArrowExec]]. Eventually this should replace those implementations.
*/
case class ColumnarToRowExec(child: SparkPlan) extends ColumnarToRowTransition with CodegenSupport {
- // supportsColumnar requires to be only called on driver side, see also SPARK-37779.
- assert(TaskContext.get != null || child.supportsColumnar)
+ assert(child.supportsColumnar)
override def output: Seq[Attribute] = child.output
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
index dacf8fe..56fff11 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/SparkPlanSuite.scala
@@ -88,23 +88,4 @@ class SparkPlanSuite extends QueryTest with SharedSparkSession {
test("SPARK-30780 empty LocalTableScan should use RDD without partitions") {
assert(LocalTableScanExec(Nil, Nil).execute().getNumPartitions == 0)
}
-
- test("SPARK-37779: ColumnarToRowExec should be canonicalizable after being (de)serialized") {
- withSQLConf(SQLConf.USE_V1_SOURCE_LIST.key -> "parquet") {
- withTempPath { path =>
- spark.range(1).write.parquet(path.getAbsolutePath)
- val df = spark.read.parquet(path.getAbsolutePath)
- val columnarToRowExec =
- df.queryExecution.executedPlan.collectFirst { case p: ColumnarToRowExec => p }.get
- try {
- spark.range(1).foreach { _ =>
- columnarToRowExec.canonicalized
- ()
- }
- } catch {
- case e: Throwable => fail("ColumnarToRowExec was not canonicalizable", e)
- }
- }
- }
- }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org