You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2018/01/25 08:40:53 UTC
spark git commit: [SPARK-23208][SQL] Fix code generation for complex
create array (related) expressions
Repository: spark
Updated Branches:
refs/heads/master 45b4bbfdd -> e29b08add
[SPARK-23208][SQL] Fix code generation for complex create array (related) expressions
## What changes were proposed in this pull request?
The `GenArrayData.genCodeToCreateArrayData` produces illegal java code when code splitting is enabled. This is used in `CreateArray` and `CreateMap` expressions for complex object arrays.
This issue is caused by a typo.
## How was this patch tested?
Added a regression test in `complexTypesSuite`.
Author: Herman van Hovell <hv...@databricks.com>
Closes #20391 from hvanhovell/SPARK-23208.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e29b08ad
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e29b08ad
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e29b08ad
Branch: refs/heads/master
Commit: e29b08add92462a6505fef966629e74ba30e994e
Parents: 45b4bbf
Author: Herman van Hovell <hv...@databricks.com>
Authored: Thu Jan 25 16:40:41 2018 +0800
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Jan 25 16:40:41 2018 +0800
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/complexTypeCreator.scala | 2 +-
.../spark/sql/catalyst/optimizer/complexTypesSuite.scala | 8 +++++++-
2 files changed, 8 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e29b08ad/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index 3dc2ee0..047b80a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -111,7 +111,7 @@ private [sql] object GenArrayData {
val assignmentString = ctx.splitExpressionsWithCurrentInputs(
expressions = assignments,
funcName = "apply",
- extraArguments = ("Object[]", arrayDataName) :: Nil)
+ extraArguments = ("Object[]", arrayName) :: Nil)
(s"Object[] $arrayName = new Object[$numElements];",
assignmentString,
http://git-wip-us.apache.org/repos/asf/spark/blob/e29b08ad/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
index 0d11958..de544ac 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/optimizer/complexTypesSuite.scala
@@ -24,6 +24,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen.CodegenContext
import org.apache.spark.sql.catalyst.plans.PlanTest
import org.apache.spark.sql.catalyst.plans.logical.{LocalRelation, LogicalPlan, Range}
import org.apache.spark.sql.catalyst.rules.RuleExecutor
+import org.apache.spark.sql.catalyst.util.GenericArrayData
import org.apache.spark.sql.types._
/**
@@ -31,7 +32,7 @@ import org.apache.spark.sql.types._
* i.e. {{{create_named_struct(square, `x` * `x`).square}}} can be simplified to {{{`x` * `x`}}}.
* sam applies to create_array and create_map
*/
-class ComplexTypesSuite extends PlanTest{
+class ComplexTypesSuite extends PlanTest with ExpressionEvalHelper {
object Optimizer extends RuleExecutor[LogicalPlan] {
val batches =
@@ -171,6 +172,11 @@ class ComplexTypesSuite extends PlanTest{
assert(ctx.inlinedMutableStates.length == 0)
}
+ test("SPARK-23208: Test code splitting for create array related methods") {
+ val inputs = (1 to 2500).map(x => Literal(s"l_$x"))
+ checkEvaluation(CreateArray(inputs), new GenericArrayData(inputs.map(_.eval())))
+ }
+
test("simplify map ops") {
val rel = relation
.select(
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org