You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ue...@apache.org on 2018/09/04 04:28:43 UTC
spark git commit: [SPARK-25308][SQL] ArrayContains function may
return a error in the code generation phase.
Repository: spark
Updated Branches:
refs/heads/master 546683c21 -> 8e2169696
[SPARK-25308][SQL] ArrayContains function may return a error in the code generation phase.
## What changes were proposed in this pull request?
Invoking ArrayContains function with non nullable array type throws the following error in the code generation phase. Below is the error snippet.
```SQL
Code generation of array_contains([1,2,3], 1) failed:
java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: Expression "isNull_0" is not an rvalue
java.util.concurrent.ExecutionException: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: failed to compile: org.codehaus.commons.compiler.CompileException: File 'generated.java', Line 40, Column 11: Expression "isNull_0" is not an rvalue
at com.google.common.util.concurrent.AbstractFuture$Sync.getValue(AbstractFuture.java:306)
at com.google.common.util.concurrent.AbstractFuture$Sync.get(AbstractFuture.java:293)
at com.google.common.util.concurrent.AbstractFuture.get(AbstractFuture.java:116)
at com.google.common.util.concurrent.Uninterruptibles.getUninterruptibly(Uninterruptibles.java:135)
at com.google.common.cache.LocalCache$Segment.getAndRecordStats(LocalCache.java:2410)
at com.google.common.cache.LocalCache$Segment.loadSync(LocalCache.java:2380)
at com.google.common.cache.LocalCache$Segment.lockedGetOrLoad(LocalCache.java:2342)
at com.google.common.cache.LocalCache$Segment.get(LocalCache.java:2257)
at com.google.common.cache.LocalCache.get(LocalCache.java:4000)
at com.google.common.cache.LocalCache.getOrLoad(LocalCache.java:4004)
at com.google.common.cache.LocalCache$LocalLoadingCache.get(LocalCache.java:4874)
at org.apache.spark.sql.catalyst.expressions.codegen.CodeGenerator$.compile(CodeGenerator.scala:1305)
```
## How was this patch tested?
Added test in CollectionExpressionSuite.
Closes #22315 from dilipbiswal/SPARK-25308.
Authored-by: Dilip Biswal <db...@us.ibm.com>
Signed-off-by: Takuya UESHIN <ue...@databricks.com>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8e216969
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8e216969
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8e216969
Branch: refs/heads/master
Commit: 8e2169696f4cd52e5e3f51626a512d25215cffa4
Parents: 546683c
Author: Dilip Biswal <db...@us.ibm.com>
Authored: Tue Sep 4 13:28:36 2018 +0900
Committer: Takuya UESHIN <ue...@databricks.com>
Committed: Tue Sep 4 13:28:36 2018 +0900
----------------------------------------------------------------------
.../expressions/collectionOperations.scala | 32 ++++++++++++++------
.../CollectionExpressionsSuite.scala | 3 ++
2 files changed, 25 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/8e216969/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
index cf9796e..17c683c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala
@@ -1464,17 +1464,29 @@ case class ArrayContains(left: Expression, right: Expression)
nullSafeCodeGen(ctx, ev, (arr, value) => {
val i = ctx.freshName("i")
val getValue = CodeGenerator.getValue(arr, right.dataType, i)
- s"""
- for (int $i = 0; $i < $arr.numElements(); $i ++) {
- if ($arr.isNullAt($i)) {
- ${ev.isNull} = true;
- } else if (${ctx.genEqual(right.dataType, value, getValue)}) {
- ${ev.isNull} = false;
- ${ev.value} = true;
- break;
- }
+ val loopBodyCode = if (nullable) {
+ s"""
+ |if ($arr.isNullAt($i)) {
+ | ${ev.isNull} = true;
+ |} else if (${ctx.genEqual(right.dataType, value, getValue)}) {
+ | ${ev.isNull} = false;
+ | ${ev.value} = true;
+ | break;
+ |}
+ """.stripMargin
+ } else {
+ s"""
+ |if (${ctx.genEqual(right.dataType, value, getValue)}) {
+ | ${ev.value} = true;
+ | break;
+ |}
+ """.stripMargin
}
- """
+ s"""
+ |for (int $i = 0; $i < $arr.numElements(); $i ++) {
+ | $loopBodyCode
+ |}
+ """.stripMargin
})
}
http://git-wip-us.apache.org/repos/asf/spark/blob/8e216969/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
index 7b345aa..a9fc3e9 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CollectionExpressionsSuite.scala
@@ -383,10 +383,13 @@ class CollectionExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper
val a3 = Literal.create(null, ArrayType(StringType))
val a4 = Literal.create(Seq(create_row(1)), ArrayType(StructType(Seq(
StructField("a", IntegerType, true)))))
+ // Explicitly mark the array type not nullable (spark-25308)
+ val a5 = Literal.create(Seq(1, 2, 3), ArrayType(IntegerType, containsNull = false))
checkEvaluation(ArrayContains(a0, Literal(1)), true)
checkEvaluation(ArrayContains(a0, Literal(0)), false)
checkEvaluation(ArrayContains(a0, Literal.create(null, IntegerType)), null)
+ checkEvaluation(ArrayContains(a5, Literal(1)), true)
checkEvaluation(ArrayContains(a1, Literal("")), true)
checkEvaluation(ArrayContains(a1, Literal("a")), null)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org