You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by hv...@apache.org on 2016/11/21 13:50:41 UTC
spark git commit: [SPARK-18398][SQL] Fix nullabilities of MapObjects
and ExternalMapToCatalyst.
Repository: spark
Updated Branches:
refs/heads/master e811fbf9e -> 9f262ae16
[SPARK-18398][SQL] Fix nullabilities of MapObjects and ExternalMapToCatalyst.
## What changes were proposed in this pull request?
The nullabilities of `MapObject` can be made more strict by relying on `inputObject.nullable` and `lambdaFunction.nullable`.
Also `ExternalMapToCatalyst.dataType` can be made more strict by relying on `valueConverter.nullable`.
## How was this patch tested?
Existing tests.
Author: Takuya UESHIN <ue...@happy-camper.st>
Closes #15840 from ueshin/issues/SPARK-18398.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9f262ae1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9f262ae1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9f262ae1
Branch: refs/heads/master
Commit: 9f262ae163b6dca6526665b3ad12b3b2ea8fb873
Parents: e811fbf
Author: Takuya UESHIN <ue...@happy-camper.st>
Authored: Mon Nov 21 05:50:35 2016 -0800
Committer: Herman van Hovell <hv...@databricks.com>
Committed: Mon Nov 21 05:50:35 2016 -0800
----------------------------------------------------------------------
.../spark/sql/catalyst/expressions/objects/objects.scala | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/9f262ae1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 0b36091..5c27179 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -461,14 +461,15 @@ case class MapObjects private(
lambdaFunction: Expression,
inputData: Expression) extends Expression with NonSQLExpression {
- override def nullable: Boolean = true
+ override def nullable: Boolean = inputData.nullable
override def children: Seq[Expression] = lambdaFunction :: inputData :: Nil
override def eval(input: InternalRow): Any =
throw new UnsupportedOperationException("Only code-generated evaluation is supported")
- override def dataType: DataType = ArrayType(lambdaFunction.dataType)
+ override def dataType: DataType =
+ ArrayType(lambdaFunction.dataType, containsNull = lambdaFunction.nullable)
override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
val elementJavaType = ctx.javaType(loopVarDataType)
@@ -642,7 +643,8 @@ case class ExternalMapToCatalyst private(
override def foldable: Boolean = false
- override def dataType: MapType = MapType(keyConverter.dataType, valueConverter.dataType)
+ override def dataType: MapType = MapType(
+ keyConverter.dataType, valueConverter.dataType, valueContainsNull = valueConverter.nullable)
override def eval(input: InternalRow): Any =
throw new UnsupportedOperationException("Only code-generated evaluation is supported")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org