You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2018/03/06 12:55:18 UTC
spark git commit: [SPARK-23594][SQL] GetExternalRowField should
support interpreted execution
Repository: spark
Updated Branches:
refs/heads/master ad640a5af -> e8a259d66
[SPARK-23594][SQL] GetExternalRowField should support interpreted execution
## What changes were proposed in this pull request?
This pr added interpreted execution for `GetExternalRowField`.
## How was this patch tested?
Added tests in `ObjectExpressionsSuite`.
Author: Takeshi Yamamuro <ya...@apache.org>
Closes #20746 from maropu/SPARK-23594.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e8a259d6
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e8a259d6
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e8a259d6
Branch: refs/heads/master
Commit: e8a259d66dda0d4c76f3af8933676bade8a7451d
Parents: ad640a5
Author: Takeshi Yamamuro <ya...@apache.org>
Authored: Tue Mar 6 13:55:13 2018 +0100
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Tue Mar 6 13:55:13 2018 +0100
----------------------------------------------------------------------
.../catalyst/expressions/objects/objects.scala | 14 +++++++++++---
.../expressions/ObjectExpressionsSuite.scala | 20 ++++++++++++++++++++
2 files changed, 31 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e8a259d6/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index d832fe0..97e3ff8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -1358,11 +1358,19 @@ case class GetExternalRowField(
override def dataType: DataType = ObjectType(classOf[Object])
- override def eval(input: InternalRow): Any =
- throw new UnsupportedOperationException("Only code-generated evaluation is supported")
-
private val errMsg = s"The ${index}th field '$fieldName' of input row cannot be null."
+ override def eval(input: InternalRow): Any = {
+ val inputRow = child.eval(input).asInstanceOf[Row]
+ if (inputRow == null) {
+ throw new RuntimeException("The input external row cannot be null.")
+ }
+ if (inputRow.isNullAt(index)) {
+ throw new RuntimeException(errMsg)
+ }
+ inputRow.get(index)
+ }
+
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
// Use unnamed reference that doesn't create a local field here to reduce the number of fields
// because errMsgField is used only when the field is null.
http://git-wip-us.apache.org/repos/asf/spark/blob/e8a259d6/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
index d535578..0f376c4 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ObjectExpressionsSuite.scala
@@ -18,6 +18,7 @@
package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.SparkFunSuite
+import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.encoders.ExpressionEncoder
import org.apache.spark.sql.catalyst.expressions.objects._
@@ -84,4 +85,23 @@ class ObjectExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper {
checkEvaluation(wrapObject, expected, InternalRow.fromSeq(Seq(input)))
}
}
+
+ test("SPARK-23594 GetExternalRowField should support interpreted execution") {
+ val inputObject = BoundReference(0, ObjectType(classOf[Row]), nullable = true)
+ val getRowField = GetExternalRowField(inputObject, index = 0, fieldName = "c0")
+ Seq((Row(1), 1), (Row(3), 3)).foreach { case (input, expected) =>
+ checkEvaluation(getRowField, expected, InternalRow.fromSeq(Seq(input)))
+ }
+
+ // If an input row or a field are null, a runtime exception will be thrown
+ val errMsg1 = intercept[RuntimeException] {
+ evaluate(getRowField, InternalRow.fromSeq(Seq(null)))
+ }.getMessage
+ assert(errMsg1 === "The input external row cannot be null.")
+
+ val errMsg2 = intercept[RuntimeException] {
+ evaluate(getRowField, InternalRow.fromSeq(Seq(Row(null))))
+ }.getMessage
+ assert(errMsg2 === "The 0th field 'c0' of input row cannot be null.")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org