You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by gu...@apache.org on 2018/08/29 01:20:39 UTC
spark git commit: [MINOR] Avoid code duplication for nullable in
Higher Order function
Repository: spark
Updated Branches:
refs/heads/master bbbf81469 -> 32c8a3d7b
[MINOR] Avoid code duplication for nullable in Higher Order function
## What changes were proposed in this pull request?
Most of `HigherOrderFunction`s have the same `nullable` definition, ie. they are nullable when one of their arguments is nullable. The PR refactors it in order to avoid code duplication.
## How was this patch tested?
NA
Closes #22243 from mgaido91/MINOR_nullable_hof.
Authored-by: Marco Gaido <ma...@gmail.com>
Signed-off-by: hyukjinkwon <gu...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/32c8a3d7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/32c8a3d7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/32c8a3d7
Branch: refs/heads/master
Commit: 32c8a3d7beac4b47a75f5ec3c69b13ebc57de0c7
Parents: bbbf814
Author: Marco Gaido <ma...@gmail.com>
Authored: Wed Aug 29 09:20:32 2018 +0800
Committer: hyukjinkwon <gu...@apache.org>
Committed: Wed Aug 29 09:20:32 2018 +0800
----------------------------------------------------------------------
.../expressions/higherOrderFunctions.scala | 18 ++----------------
1 file changed, 2 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/32c8a3d7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
index 9f2e84a..2bb6b20 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala
@@ -90,6 +90,8 @@ object LambdaFunction {
*/
trait HigherOrderFunction extends Expression with ExpectsInputTypes {
+ override def nullable: Boolean = arguments.exists(_.nullable)
+
override def children: Seq[Expression] = arguments ++ functions
/**
@@ -217,8 +219,6 @@ case class ArrayTransform(
function: Expression)
extends ArrayBasedSimpleHigherOrderFunction with CodegenFallback {
- override def nullable: Boolean = argument.nullable
-
override def dataType: ArrayType = ArrayType(function.dataType, function.nullable)
override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction): ArrayTransform = {
@@ -287,8 +287,6 @@ case class MapFilter(
copy(function = f(function, (keyType, false) :: (valueType, valueContainsNull) :: Nil))
}
- override def nullable: Boolean = argument.nullable
-
override def nullSafeEval(inputRow: InternalRow, argumentValue: Any): Any = {
val m = argumentValue.asInstanceOf[MapData]
val f = functionForEval
@@ -328,8 +326,6 @@ case class ArrayFilter(
function: Expression)
extends ArrayBasedSimpleHigherOrderFunction with CodegenFallback {
- override def nullable: Boolean = argument.nullable
-
override def dataType: DataType = argument.dataType
override def functionType: AbstractDataType = BooleanType
@@ -375,8 +371,6 @@ case class ArrayExists(
function: Expression)
extends ArrayBasedSimpleHigherOrderFunction with CodegenFallback {
- override def nullable: Boolean = argument.nullable
-
override def dataType: DataType = BooleanType
override def functionType: AbstractDataType = BooleanType
@@ -516,8 +510,6 @@ case class TransformKeys(
function: Expression)
extends MapBasedSimpleHigherOrderFunction with CodegenFallback {
- override def nullable: Boolean = argument.nullable
-
@transient lazy val MapType(keyType, valueType, valueContainsNull) = argument.dataType
override def dataType: DataType = MapType(function.dataType, valueType, valueContainsNull)
@@ -568,8 +560,6 @@ case class TransformValues(
function: Expression)
extends MapBasedSimpleHigherOrderFunction with CodegenFallback {
- override def nullable: Boolean = argument.nullable
-
@transient lazy val MapType(keyType, valueType, valueContainsNull) = argument.dataType
override def dataType: DataType = MapType(keyType, function.dataType, function.nullable)
@@ -638,8 +628,6 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression)
override def functionTypes: Seq[AbstractDataType] = AnyDataType :: Nil
- override def nullable: Boolean = left.nullable || right.nullable
-
override def dataType: DataType = MapType(keyType, function.dataType, function.nullable)
override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction): MapZipWith = {
@@ -810,8 +798,6 @@ case class ZipWith(left: Expression, right: Expression, function: Expression)
override def functionTypes: Seq[AbstractDataType] = AnyDataType :: Nil
- override def nullable: Boolean = left.nullable || right.nullable
-
override def dataType: ArrayType = ArrayType(function.dataType, function.nullable)
override def bind(f: (Expression, Seq[(DataType, Boolean)]) => LambdaFunction): ZipWith = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org