You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by we...@apache.org on 2017/11/16 16:56:26 UTC
spark git commit: [SPARK-22499][SQL] Fix 64KB JVM bytecode limit
problem with least and greatest
Repository: spark
Updated Branches:
refs/heads/master 03f2b7bff -> ed885e7a6
[SPARK-22499][SQL] Fix 64KB JVM bytecode limit problem with least and greatest
## What changes were proposed in this pull request?
This PR changes `least` and `greatest` code generation to place generated code for expression for arguments into separated methods if these size could be large.
This PR resolved two cases:
* `least` with a lot of argument
* `greatest` with a lot of argument
## How was this patch tested?
Added a new test case into `ArithmeticExpressionsSuite`
Author: Kazuaki Ishizaki <is...@jp.ibm.com>
Closes #19729 from kiszk/SPARK-22499.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ed885e7a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ed885e7a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ed885e7a
Branch: refs/heads/master
Commit: ed885e7a6504c439ffb6730e6963efbd050d43dd
Parents: 03f2b7b
Author: Kazuaki Ishizaki <is...@jp.ibm.com>
Authored: Thu Nov 16 17:56:21 2017 +0100
Committer: Wenchen Fan <we...@databricks.com>
Committed: Thu Nov 16 17:56:21 2017 +0100
----------------------------------------------------------------------
.../sql/catalyst/expressions/arithmetic.scala | 24 ++++++++++----------
.../expressions/ArithmeticExpressionSuite.scala | 9 ++++++++
2 files changed, 21 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ed885e7a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 7559852..72d5889 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -602,8 +602,8 @@ case class Least(children: Seq[Expression]) extends Expression {
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val evalChildren = children.map(_.genCode(ctx))
- val first = evalChildren(0)
- val rest = evalChildren.drop(1)
+ ctx.addMutableState("boolean", ev.isNull, "")
+ ctx.addMutableState(ctx.javaType(dataType), ev.value, "")
def updateEval(eval: ExprCode): String = {
s"""
${eval.code}
@@ -614,11 +614,11 @@ case class Least(children: Seq[Expression]) extends Expression {
}
"""
}
+ val codes = ctx.splitExpressions(ctx.INPUT_ROW, evalChildren.map(updateEval))
ev.copy(code = s"""
- ${first.code}
- boolean ${ev.isNull} = ${first.isNull};
- ${ctx.javaType(dataType)} ${ev.value} = ${first.value};
- ${rest.map(updateEval).mkString("\n")}""")
+ ${ev.isNull} = true;
+ ${ev.value} = ${ctx.defaultValue(dataType)};
+ $codes""")
}
}
@@ -668,8 +668,8 @@ case class Greatest(children: Seq[Expression]) extends Expression {
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val evalChildren = children.map(_.genCode(ctx))
- val first = evalChildren(0)
- val rest = evalChildren.drop(1)
+ ctx.addMutableState("boolean", ev.isNull, "")
+ ctx.addMutableState(ctx.javaType(dataType), ev.value, "")
def updateEval(eval: ExprCode): String = {
s"""
${eval.code}
@@ -680,10 +680,10 @@ case class Greatest(children: Seq[Expression]) extends Expression {
}
"""
}
+ val codes = ctx.splitExpressions(ctx.INPUT_ROW, evalChildren.map(updateEval))
ev.copy(code = s"""
- ${first.code}
- boolean ${ev.isNull} = ${first.isNull};
- ${ctx.javaType(dataType)} ${ev.value} = ${first.value};
- ${rest.map(updateEval).mkString("\n")}""")
+ ${ev.isNull} = true;
+ ${ev.value} = ${ctx.defaultValue(dataType)};
+ $codes""")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/ed885e7a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
index 0310537..fb759eb 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ArithmeticExpressionSuite.scala
@@ -334,4 +334,13 @@ class ArithmeticExpressionSuite extends SparkFunSuite with ExpressionEvalHelper
checkConsistencyBetweenInterpretedAndCodegen(Greatest, dt, 2)
}
}
+
+ test("SPARK-22499: Least and greatest should not generate codes beyond 64KB") {
+ val N = 3000
+ val strings = (1 to N).map(x => "s" * x)
+ val inputsExpr = strings.map(Literal.create(_, StringType))
+
+ checkEvaluation(Least(inputsExpr), "s" * 1, EmptyRow)
+ checkEvaluation(Greatest(inputsExpr), "s" * N, EmptyRow)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org