You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by li...@apache.org on 2018/09/15 23:20:50 UTC
spark git commit: [SPARK-25426][SQL] Remove the duplicate fallback
logic in UnsafeProjection
Repository: spark
Updated Branches:
refs/heads/master be454a7ce -> 5ebef33c8
[SPARK-25426][SQL] Remove the duplicate fallback logic in UnsafeProjection
## What changes were proposed in this pull request?
This pr removed the duplicate fallback logic in `UnsafeProjection`.
This pr comes from #22355.
## How was this patch tested?
Added tests in `CodeGeneratorWithInterpretedFallbackSuite`.
Closes #22417 from maropu/SPARK-25426.
Authored-by: Takeshi Yamamuro <ya...@apache.org>
Signed-off-by: gatorsmile <ga...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5ebef33c
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5ebef33c
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5ebef33c
Branch: refs/heads/master
Commit: 5ebef33c85a66cdc29db2eff2343600602bbe94e
Parents: be454a7
Author: Takeshi Yamamuro <ya...@apache.org>
Authored: Sat Sep 15 16:20:45 2018 -0700
Committer: gatorsmile <ga...@gmail.com>
Committed: Sat Sep 15 16:20:45 2018 -0700
----------------------------------------------------------------------
.../sql/catalyst/expressions/Projection.scala | 25 ++------------------
.../sql/execution/basicPhysicalOperators.scala | 3 +--
2 files changed, 3 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5ebef33c/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala
index 226a4dd..5f24170 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Projection.scala
@@ -17,10 +17,9 @@
package org.apache.spark.sql.catalyst.expressions
-import scala.util.control.NonFatal
-
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.expressions.codegen.{GenerateSafeProjection, GenerateUnsafeProjection}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types.{DataType, StructType}
/**
@@ -117,7 +116,7 @@ object UnsafeProjection
extends CodeGeneratorWithInterpretedFallback[Seq[Expression], UnsafeProjection] {
override protected def createCodeGeneratedObject(in: Seq[Expression]): UnsafeProjection = {
- GenerateUnsafeProjection.generate(in)
+ GenerateUnsafeProjection.generate(in, SQLConf.get.subexpressionEliminationEnabled)
}
override protected def createInterpretedObject(in: Seq[Expression]): UnsafeProjection = {
@@ -168,26 +167,6 @@ object UnsafeProjection
def create(exprs: Seq[Expression], inputSchema: Seq[Attribute]): UnsafeProjection = {
create(toBoundExprs(exprs, inputSchema))
}
-
- /**
- * Same as other create()'s but allowing enabling/disabling subexpression elimination.
- * The param `subexpressionEliminationEnabled` doesn't guarantee to work. For example,
- * when fallbacking to interpreted execution, it is not supported.
- */
- def create(
- exprs: Seq[Expression],
- inputSchema: Seq[Attribute],
- subexpressionEliminationEnabled: Boolean): UnsafeProjection = {
- val unsafeExprs = toUnsafeExprs(toBoundExprs(exprs, inputSchema))
- try {
- GenerateUnsafeProjection.generate(unsafeExprs, subexpressionEliminationEnabled)
- } catch {
- case NonFatal(_) =>
- // We should have already seen the error message in `CodeGenerator`
- logWarning("Expr codegen error and falling back to interpreter mode")
- InterpretedUnsafeProjection.createProjection(unsafeExprs)
- }
- }
}
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/5ebef33c/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
index 9434ceb..222a1b8 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/basicPhysicalOperators.scala
@@ -68,8 +68,7 @@ case class ProjectExec(projectList: Seq[NamedExpression], child: SparkPlan)
protected override def doExecute(): RDD[InternalRow] = {
child.execute().mapPartitionsWithIndexInternal { (index, iter) =>
- val project = UnsafeProjection.create(projectList, child.output,
- subexpressionEliminationEnabled)
+ val project = UnsafeProjection.create(projectList, child.output)
project.initialize(index)
iter.map(project)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org