You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/18 00:49:35 UTC
spark git commit: [SPARK-9080][SQL] add isNaN predicate expression
Repository: spark
Updated Branches:
refs/heads/master b2aa490bb -> 15fc2ffe5
[SPARK-9080][SQL] add isNaN predicate expression
JIRA: https://issues.apache.org/jira/browse/SPARK-9080
cc rxin
Author: Yijie Shen <he...@gmail.com>
Closes #7464 from yijieshen/isNaN and squashes the following commits:
11ae039 [Yijie Shen] add isNaN in functions
666718e [Yijie Shen] add isNaN predicate expression
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/15fc2ffe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/15fc2ffe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/15fc2ffe
Branch: refs/heads/master
Commit: 15fc2ffe5530c43c64cfc37f2d1ce83f04ce3bd9
Parents: b2aa490
Author: Yijie Shen <he...@gmail.com>
Authored: Fri Jul 17 15:49:31 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Fri Jul 17 15:49:31 2015 -0700
----------------------------------------------------------------------
.../catalyst/analysis/FunctionRegistry.scala | 1 +
.../sql/catalyst/expressions/predicates.scala | 50 ++++++++++++++++++++
.../catalyst/expressions/PredicateSuite.scala | 12 ++++-
.../scala/org/apache/spark/sql/Column.scala | 8 ++++
.../scala/org/apache/spark/sql/functions.scala | 10 +++-
.../spark/sql/ColumnExpressionSuite.scala | 21 ++++++++
6 files changed, 100 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/15fc2ffe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index a451817..7bb2579 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -79,6 +79,7 @@ object FunctionRegistry {
expression[Explode]("explode"),
expression[Greatest]("greatest"),
expression[If]("if"),
+ expression[IsNaN]("isnan"),
expression[IsNull]("isnull"),
expression[IsNotNull]("isnotnull"),
expression[Least]("least"),
http://git-wip-us.apache.org/repos/asf/spark/blob/15fc2ffe/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 7a6fb2b..2751c8e 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -120,6 +120,56 @@ case class InSet(child: Expression, hset: Set[Any])
}
}
+/**
+ * Evaluates to `true` if it's NaN or null
+ */
+case class IsNaN(child: Expression) extends UnaryExpression
+ with Predicate with ImplicitCastInputTypes {
+
+ override def inputTypes: Seq[AbstractDataType] = Seq(TypeCollection(DoubleType, FloatType))
+
+ override def nullable: Boolean = false
+
+ override def eval(input: InternalRow): Any = {
+ val value = child.eval(input)
+ if (value == null) {
+ true
+ } else {
+ child.dataType match {
+ case DoubleType => value.asInstanceOf[Double].isNaN
+ case FloatType => value.asInstanceOf[Float].isNaN
+ }
+ }
+ }
+
+ override def genCode(ctx: CodeGenContext, ev: GeneratedExpressionCode): String = {
+ val eval = child.gen(ctx)
+ child.dataType match {
+ case FloatType =>
+ s"""
+ ${eval.code}
+ boolean ${ev.isNull} = false;
+ ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)};
+ if (${eval.isNull}) {
+ ${ev.primitive} = true;
+ } else {
+ ${ev.primitive} = Float.isNaN(${eval.primitive});
+ }
+ """
+ case DoubleType =>
+ s"""
+ ${eval.code}
+ boolean ${ev.isNull} = false;
+ ${ctx.javaType(dataType)} ${ev.primitive} = ${ctx.defaultValue(dataType)};
+ if (${eval.isNull}) {
+ ${ev.primitive} = true;
+ } else {
+ ${ev.primitive} = Double.isNaN(${eval.primitive});
+ }
+ """
+ }
+ }
+}
case class And(left: Expression, right: Expression) extends BinaryOperator with Predicate {
http://git-wip-us.apache.org/repos/asf/spark/blob/15fc2ffe/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
index 188ecef..052abc5 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/PredicateSuite.scala
@@ -21,7 +21,7 @@ import scala.collection.immutable.HashSet
import org.apache.spark.SparkFunSuite
import org.apache.spark.sql.catalyst.dsl.expressions._
-import org.apache.spark.sql.types.{Decimal, IntegerType, BooleanType}
+import org.apache.spark.sql.types.{Decimal, DoubleType, IntegerType, BooleanType}
class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
@@ -116,6 +116,16 @@ class PredicateSuite extends SparkFunSuite with ExpressionEvalHelper {
true)
}
+ test("IsNaN") {
+ checkEvaluation(IsNaN(Literal(Double.NaN)), true)
+ checkEvaluation(IsNaN(Literal(Float.NaN)), true)
+ checkEvaluation(IsNaN(Literal(math.log(-3))), true)
+ checkEvaluation(IsNaN(Literal.create(null, DoubleType)), true)
+ checkEvaluation(IsNaN(Literal(Double.PositiveInfinity)), false)
+ checkEvaluation(IsNaN(Literal(Float.MaxValue)), false)
+ checkEvaluation(IsNaN(Literal(5.5f)), false)
+ }
+
test("INSET") {
val hS = HashSet[Any]() + 1 + 2
val nS = HashSet[Any]() + 1 + 2 + null
http://git-wip-us.apache.org/repos/asf/spark/blob/15fc2ffe/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index 1025026..221cd04 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -401,6 +401,14 @@ class Column(protected[sql] val expr: Expression) extends Logging {
}
/**
+ * True if the current expression is NaN or null
+ *
+ * @group expr_ops
+ * @since 1.5.0
+ */
+ def isNaN: Column = IsNaN(expr)
+
+ /**
* True if the current expression is null.
*
* @group expr_ops
http://git-wip-us.apache.org/repos/asf/spark/blob/15fc2ffe/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index fe511c2..b56fd9a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -620,7 +620,15 @@ object functions {
def explode(e: Column): Column = Explode(e.expr)
/**
- * Converts a string exprsesion to lower case.
+ * Return true if the column is NaN or null
+ *
+ * @group normal_funcs
+ * @since 1.5.0
+ */
+ def isNaN(e: Column): Column = IsNaN(e.expr)
+
+ /**
+ * Converts a string expression to lower case.
*
* @group normal_funcs
* @since 1.3.0
http://git-wip-us.apache.org/repos/asf/spark/blob/15fc2ffe/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index 88bb743..8f15479 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -201,6 +201,27 @@ class ColumnExpressionSuite extends QueryTest {
Row(false, true))
}
+ test("isNaN") {
+ val testData = ctx.createDataFrame(ctx.sparkContext.parallelize(
+ Row(Double.NaN, Float.NaN) ::
+ Row(math.log(-1), math.log(-3).toFloat) ::
+ Row(null, null) ::
+ Row(Double.MaxValue, Float.MinValue):: Nil),
+ StructType(Seq(StructField("a", DoubleType), StructField("b", FloatType))))
+
+ checkAnswer(
+ testData.select($"a".isNaN, $"b".isNaN),
+ Row(true, true) :: Row(true, true) :: Row(true, true) :: Row(false, false) :: Nil)
+
+ checkAnswer(
+ testData.select(isNaN($"a"), isNaN($"b")),
+ Row(true, true) :: Row(true, true) :: Row(true, true) :: Row(false, false) :: Nil)
+
+ checkAnswer(
+ ctx.sql("select isnan(15), isnan('invalid')"),
+ Row(false, true))
+ }
+
test("===") {
checkAnswer(
testData2.filter($"a" === 1),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org