You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/04/29 09:09:28 UTC
spark git commit: [SPARK-7188] added python support for math
DataFrame functions
Repository: spark
Updated Branches:
refs/heads/master 8dee2746b -> fe917f5ec
[SPARK-7188] added python support for math DataFrame functions
Adds support for the math functions for DataFrames in PySpark.
rxin I love Davies.
Author: Burak Yavuz <br...@gmail.com>
Closes #5750 from brkyvz/python-math-udfs and squashes the following commits:
7c4f563 [Burak Yavuz] removed is_math
3c4adde [Burak Yavuz] cleanup imports
d5dca3f [Burak Yavuz] moved math functions to mathfunctions
25e6534 [Burak Yavuz] addressed comments v2.0
d3f7e0f [Burak Yavuz] addressed comments and added tests
7b7d7c4 [Burak Yavuz] remove tests for removed methods
33c2c15 [Burak Yavuz] fixed python style
3ee0c05 [Burak Yavuz] added python functions
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fe917f5e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fe917f5e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fe917f5e
Branch: refs/heads/master
Commit: fe917f5ec9be8c8424416f7b5423ddb4318e03a0
Parents: 8dee274
Author: Burak Yavuz <br...@gmail.com>
Authored: Wed Apr 29 00:09:24 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Apr 29 00:09:24 2015 -0700
----------------------------------------------------------------------
python/pyspark/sql/functions.py | 2 +-
python/pyspark/sql/mathfunctions.py | 101 +++++
python/pyspark/sql/tests.py | 29 ++
.../catalyst/expressions/mathfuncs/binary.scala | 12 +-
.../catalyst/expressions/mathfuncs/unary.scala | 122 ++----
.../expressions/ExpressionEvaluationSuite.scala | 12 -
.../org/apache/spark/sql/mathfunctions.scala | 409 ++++++-------------
.../apache/spark/sql/MathExpressionsSuite.scala | 12 -
8 files changed, 276 insertions(+), 423 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 7b86655..555c2fa 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -54,7 +54,7 @@ _functions = {
'upper': 'Converts a string expression to upper case.',
'lower': 'Converts a string expression to upper case.',
'sqrt': 'Computes the square root of the specified float value.',
- 'abs': 'Computes the absolutle value.',
+ 'abs': 'Computes the absolute value.',
'max': 'Aggregate function: returns the maximum value of the expression in a group.',
'min': 'Aggregate function: returns the minimum value of the expression in a group.',
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/python/pyspark/sql/mathfunctions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/mathfunctions.py b/python/pyspark/sql/mathfunctions.py
new file mode 100644
index 0000000..7dbcab8
--- /dev/null
+++ b/python/pyspark/sql/mathfunctions.py
@@ -0,0 +1,101 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+A collection of builtin math functions
+"""
+
+from pyspark import SparkContext
+from pyspark.sql.dataframe import Column
+
+__all__ = []
+
+
+def _create_unary_mathfunction(name, doc=""):
+ """ Create a unary mathfunction by name"""
+ def _(col):
+ sc = SparkContext._active_spark_context
+ jc = getattr(sc._jvm.mathfunctions, name)(col._jc if isinstance(col, Column) else col)
+ return Column(jc)
+ _.__name__ = name
+ _.__doc__ = doc
+ return _
+
+
+def _create_binary_mathfunction(name, doc=""):
+ """ Create a binary mathfunction by name"""
+ def _(col1, col2):
+ sc = SparkContext._active_spark_context
+ # users might write ints for simplicity. This would throw an error on the JVM side.
+ if type(col1) is int:
+ col1 = col1 * 1.0
+ if type(col2) is int:
+ col2 = col2 * 1.0
+ jc = getattr(sc._jvm.mathfunctions, name)(col1._jc if isinstance(col1, Column) else col1,
+ col2._jc if isinstance(col2, Column) else col2)
+ return Column(jc)
+ _.__name__ = name
+ _.__doc__ = doc
+ return _
+
+
+# math functions are found under another object therefore, they need to be handled separately
+_mathfunctions = {
+ 'acos': 'Computes the cosine inverse of the given value; the returned angle is in the range' +
+ '0.0 through pi.',
+ 'asin': 'Computes the sine inverse of the given value; the returned angle is in the range' +
+ '-pi/2 through pi/2.',
+ 'atan': 'Computes the tangent inverse of the given value.',
+ 'cbrt': 'Computes the cube-root of the given value.',
+ 'ceil': 'Computes the ceiling of the given value.',
+ 'cos': 'Computes the cosine of the given value.',
+ 'cosh': 'Computes the hyperbolic cosine of the given value.',
+ 'exp': 'Computes the exponential of the given value.',
+ 'expm1': 'Computes the exponential of the given value minus one.',
+ 'floor': 'Computes the floor of the given value.',
+ 'log': 'Computes the natural logarithm of the given value.',
+ 'log10': 'Computes the logarithm of the given value in Base 10.',
+ 'log1p': 'Computes the natural logarithm of the given value plus one.',
+ 'rint': 'Returns the double value that is closest in value to the argument and' +
+ ' is equal to a mathematical integer.',
+ 'signum': 'Computes the signum of the given value.',
+ 'sin': 'Computes the sine of the given value.',
+ 'sinh': 'Computes the hyperbolic sine of the given value.',
+ 'tan': 'Computes the tangent of the given value.',
+ 'tanh': 'Computes the hyperbolic tangent of the given value.',
+ 'toDeg': 'Converts an angle measured in radians to an approximately equivalent angle ' +
+ 'measured in degrees.',
+ 'toRad': 'Converts an angle measured in degrees to an approximately equivalent angle ' +
+ 'measured in radians.'
+}
+
+# math functions that take two arguments as input
+_binary_mathfunctions = {
+ 'atan2': 'Returns the angle theta from the conversion of rectangular coordinates (x, y) to' +
+ 'polar coordinates (r, theta).',
+ 'hypot': 'Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.',
+ 'pow': 'Returns the value of the first argument raised to the power of the second argument.'
+}
+
+for _name, _doc in _mathfunctions.items():
+ globals()[_name] = _create_unary_mathfunction(_name, _doc)
+for _name, _doc in _binary_mathfunctions.items():
+ globals()[_name] = _create_binary_mathfunction(_name, _doc)
+del _name, _doc
+__all__ += _mathfunctions.keys()
+__all__ += _binary_mathfunctions.keys()
+__all__.sort()
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/python/pyspark/sql/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/tests.py b/python/pyspark/sql/tests.py
index fe43c37..2ffd18e 100644
--- a/python/pyspark/sql/tests.py
+++ b/python/pyspark/sql/tests.py
@@ -387,6 +387,35 @@ class SQLTests(ReusedPySparkTestCase):
self.assertTrue(95 < g.agg(functions.approxCountDistinct(df.key)).first()[0])
self.assertEqual(100, g.agg(functions.countDistinct(df.value)).first()[0])
+ def test_math_functions(self):
+ df = self.sc.parallelize([Row(a=i, b=2 * i) for i in range(10)]).toDF()
+ from pyspark.sql import mathfunctions as functions
+ import math
+
+ def get_values(l):
+ return [j[0] for j in l]
+
+ def assert_close(a, b):
+ c = get_values(b)
+ diff = [abs(v - c[k]) < 1e-6 for k, v in enumerate(a)]
+ return sum(diff) == len(a)
+ assert_close([math.cos(i) for i in range(10)],
+ df.select(functions.cos(df.a)).collect())
+ assert_close([math.cos(i) for i in range(10)],
+ df.select(functions.cos("a")).collect())
+ assert_close([math.sin(i) for i in range(10)],
+ df.select(functions.sin(df.a)).collect())
+ assert_close([math.sin(i) for i in range(10)],
+ df.select(functions.sin(df['a'])).collect())
+ assert_close([math.pow(i, 2 * i) for i in range(10)],
+ df.select(functions.pow(df.a, df.b)).collect())
+ assert_close([math.pow(i, 2) for i in range(10)],
+ df.select(functions.pow(df.a, 2)).collect())
+ assert_close([math.pow(i, 2) for i in range(10)],
+ df.select(functions.pow(df.a, 2.0)).collect())
+ assert_close([math.hypot(i, 2 * i) for i in range(10)],
+ df.select(functions.hypot(df.a, df.b)).collect())
+
def test_save_and_load(self):
df = self.df
tmpPath = tempfile.mkdtemp()
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
index 5b4d912..fcc06d3 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/binary.scala
@@ -65,12 +65,6 @@ abstract class BinaryMathExpression(f: (Double, Double) => Double, name: String)
}
}
-case class Pow(left: Expression, right: Expression) extends BinaryMathExpression(math.pow, "POWER")
-
-case class Hypot(
- left: Expression,
- right: Expression) extends BinaryMathExpression(math.hypot, "HYPOT")
-
case class Atan2(
left: Expression,
right: Expression) extends BinaryMathExpression(math.atan2, "ATAN2") {
@@ -91,3 +85,9 @@ case class Atan2(
}
}
}
+
+case class Hypot(
+ left: Expression,
+ right: Expression) extends BinaryMathExpression(math.hypot, "HYPOT")
+
+case class Pow(left: Expression, right: Expression) extends BinaryMathExpression(math.pow, "POWER")
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
index 96cb77d..dc68469 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/mathfuncs/unary.scala
@@ -25,27 +25,16 @@ import org.apache.spark.sql.types._
* input format, therefore these functions extend `ExpectsInputTypes`.
* @param name The short name of the function
*/
-abstract class MathematicalExpression(name: String)
+abstract class MathematicalExpression(f: Double => Double, name: String)
extends UnaryExpression with Serializable with ExpectsInputTypes {
self: Product =>
type EvaluatedType = Any
+ override def expectedChildTypes: Seq[DataType] = Seq(DoubleType)
override def dataType: DataType = DoubleType
override def foldable: Boolean = child.foldable
override def nullable: Boolean = true
override def toString: String = s"$name($child)"
-}
-
-/**
- * A unary expression specifically for math functions that take a `Double` as input and return
- * a `Double`.
- * @param f The math function.
- * @param name The short name of the function
- */
-abstract class MathematicalExpressionForDouble(f: Double => Double, name: String)
- extends MathematicalExpression(name) { self: Product =>
-
- override def expectedChildTypes: Seq[DataType] = Seq(DoubleType)
override def eval(input: Row): Any = {
val evalE = child.eval(input)
@@ -58,111 +47,46 @@ abstract class MathematicalExpressionForDouble(f: Double => Double, name: String
}
}
-/**
- * A unary expression specifically for math functions that take an `Int` as input and return
- * an `Int`.
- * @param f The math function.
- * @param name The short name of the function
- */
-abstract class MathematicalExpressionForInt(f: Int => Int, name: String)
- extends MathematicalExpression(name) { self: Product =>
+case class Acos(child: Expression) extends MathematicalExpression(math.acos, "ACOS")
- override def dataType: DataType = IntegerType
- override def expectedChildTypes: Seq[DataType] = Seq(IntegerType)
+case class Asin(child: Expression) extends MathematicalExpression(math.asin, "ASIN")
- override def eval(input: Row): Any = {
- val evalE = child.eval(input)
- if (evalE == null) null else f(evalE.asInstanceOf[Int])
- }
-}
+case class Atan(child: Expression) extends MathematicalExpression(math.atan, "ATAN")
-/**
- * A unary expression specifically for math functions that take a `Float` as input and return
- * a `Float`.
- * @param f The math function.
- * @param name The short name of the function
- */
-abstract class MathematicalExpressionForFloat(f: Float => Float, name: String)
- extends MathematicalExpression(name) { self: Product =>
+case class Cbrt(child: Expression) extends MathematicalExpression(math.cbrt, "CBRT")
- override def dataType: DataType = FloatType
- override def expectedChildTypes: Seq[DataType] = Seq(FloatType)
+case class Ceil(child: Expression) extends MathematicalExpression(math.ceil, "CEIL")
- override def eval(input: Row): Any = {
- val evalE = child.eval(input)
- if (evalE == null) {
- null
- } else {
- val result = f(evalE.asInstanceOf[Float])
- if (result.isNaN) null else result
- }
- }
-}
-
-/**
- * A unary expression specifically for math functions that take a `Long` as input and return
- * a `Long`.
- * @param f The math function.
- * @param name The short name of the function
- */
-abstract class MathematicalExpressionForLong(f: Long => Long, name: String)
- extends MathematicalExpression(name) { self: Product =>
-
- override def dataType: DataType = LongType
- override def expectedChildTypes: Seq[DataType] = Seq(LongType)
-
- override def eval(input: Row): Any = {
- val evalE = child.eval(input)
- if (evalE == null) null else f(evalE.asInstanceOf[Long])
- }
-}
-
-case class Sin(child: Expression) extends MathematicalExpressionForDouble(math.sin, "SIN")
-
-case class Asin(child: Expression) extends MathematicalExpressionForDouble(math.asin, "ASIN")
-
-case class Sinh(child: Expression) extends MathematicalExpressionForDouble(math.sinh, "SINH")
-
-case class Cos(child: Expression) extends MathematicalExpressionForDouble(math.cos, "COS")
+case class Cos(child: Expression) extends MathematicalExpression(math.cos, "COS")
-case class Acos(child: Expression) extends MathematicalExpressionForDouble(math.acos, "ACOS")
+case class Cosh(child: Expression) extends MathematicalExpression(math.cosh, "COSH")
-case class Cosh(child: Expression) extends MathematicalExpressionForDouble(math.cosh, "COSH")
+case class Exp(child: Expression) extends MathematicalExpression(math.exp, "EXP")
-case class Tan(child: Expression) extends MathematicalExpressionForDouble(math.tan, "TAN")
+case class Expm1(child: Expression) extends MathematicalExpression(math.expm1, "EXPM1")
-case class Atan(child: Expression) extends MathematicalExpressionForDouble(math.atan, "ATAN")
+case class Floor(child: Expression) extends MathematicalExpression(math.floor, "FLOOR")
-case class Tanh(child: Expression) extends MathematicalExpressionForDouble(math.tanh, "TANH")
+case class Log(child: Expression) extends MathematicalExpression(math.log, "LOG")
-case class Ceil(child: Expression) extends MathematicalExpressionForDouble(math.ceil, "CEIL")
+case class Log10(child: Expression) extends MathematicalExpression(math.log10, "LOG10")
-case class Floor(child: Expression) extends MathematicalExpressionForDouble(math.floor, "FLOOR")
+case class Log1p(child: Expression) extends MathematicalExpression(math.log1p, "LOG1P")
-case class Rint(child: Expression) extends MathematicalExpressionForDouble(math.rint, "ROUND")
+case class Rint(child: Expression) extends MathematicalExpression(math.rint, "ROUND")
-case class Cbrt(child: Expression) extends MathematicalExpressionForDouble(math.cbrt, "CBRT")
+case class Signum(child: Expression) extends MathematicalExpression(math.signum, "SIGNUM")
-case class Signum(child: Expression) extends MathematicalExpressionForDouble(math.signum, "SIGNUM")
+case class Sin(child: Expression) extends MathematicalExpression(math.sin, "SIN")
-case class ISignum(child: Expression) extends MathematicalExpressionForInt(math.signum, "ISIGNUM")
+case class Sinh(child: Expression) extends MathematicalExpression(math.sinh, "SINH")
-case class FSignum(child: Expression) extends MathematicalExpressionForFloat(math.signum, "FSIGNUM")
+case class Tan(child: Expression) extends MathematicalExpression(math.tan, "TAN")
-case class LSignum(child: Expression) extends MathematicalExpressionForLong(math.signum, "LSIGNUM")
+case class Tanh(child: Expression) extends MathematicalExpression(math.tanh, "TANH")
case class ToDegrees(child: Expression)
- extends MathematicalExpressionForDouble(math.toDegrees, "DEGREES")
+ extends MathematicalExpression(math.toDegrees, "DEGREES")
case class ToRadians(child: Expression)
- extends MathematicalExpressionForDouble(math.toRadians, "RADIANS")
-
-case class Log(child: Expression) extends MathematicalExpressionForDouble(math.log, "LOG")
-
-case class Log10(child: Expression) extends MathematicalExpressionForDouble(math.log10, "LOG10")
-
-case class Log1p(child: Expression) extends MathematicalExpressionForDouble(math.log1p, "LOG1P")
-
-case class Exp(child: Expression) extends MathematicalExpressionForDouble(math.exp, "EXP")
-
-case class Expm1(child: Expression) extends MathematicalExpressionForDouble(math.expm1, "EXPM1")
+ extends MathematicalExpression(math.toRadians, "RADIANS")
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
index 5390ce4..fa71001 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvaluationSuite.scala
@@ -1253,18 +1253,6 @@ class ExpressionEvaluationSuite extends ExpressionEvaluationBaseSuite {
unaryMathFunctionEvaluation[Double](Signum, math.signum)
}
- test("isignum") {
- unaryMathFunctionEvaluation[Int](ISignum, math.signum, (-5 to 5))
- }
-
- test("fsignum") {
- unaryMathFunctionEvaluation[Float](FSignum, math.signum, (-5 to 5).map(_.toFloat))
- }
-
- test("lsignum") {
- unaryMathFunctionEvaluation[Long](LSignum, math.signum, (5 to 5).map(_.toLong))
- }
-
test("log") {
unaryMathFunctionEvaluation(Log, math.log, (0 to 20).map(_ * 0.1))
unaryMathFunctionEvaluation(Log, math.log, (-5 to -1).map(_ * 0.1), true)
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala b/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
index 84f62bf..d901542 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/mathfunctions.scala
@@ -29,9 +29,6 @@ import org.apache.spark.sql.functions.lit
* Mathematical Functions available for [[DataFrame]].
*
* @groupname double_funcs Functions that require DoubleType as an input
- * @groupname int_funcs Functions that require IntegerType as an input
- * @groupname float_funcs Functions that require FloatType as an input
- * @groupname long_funcs Functions that require LongType as an input
*/
@Experimental
// scalastyle:off
@@ -41,522 +38,348 @@ object mathfunctions {
private[this] implicit def toColumn(expr: Expression): Column = Column(expr)
/**
- * Computes the sine of the given value.
- *
- * @group double_funcs
+ * Computes the cosine inverse of the given value; the returned angle is in the range
+ * 0.0 through pi.
*/
- def sin(e: Column): Column = Sin(e.expr)
+ def acos(e: Column): Column = Acos(e.expr)
/**
- * Computes the sine of the given column.
- *
- * @group double_funcs
+ * Computes the cosine inverse of the given column; the returned angle is in the range
+ * 0.0 through pi.
*/
- def sin(columnName: String): Column = sin(Column(columnName))
+ def acos(columnName: String): Column = acos(Column(columnName))
/**
* Computes the sine inverse of the given value; the returned angle is in the range
* -pi/2 through pi/2.
- *
- * @group double_funcs
*/
def asin(e: Column): Column = Asin(e.expr)
/**
* Computes the sine inverse of the given column; the returned angle is in the range
* -pi/2 through pi/2.
- *
- * @group double_funcs
*/
def asin(columnName: String): Column = asin(Column(columnName))
/**
- * Computes the hyperbolic sine of the given value.
- *
- * @group double_funcs
- */
- def sinh(e: Column): Column = Sinh(e.expr)
-
- /**
- * Computes the hyperbolic sine of the given column.
- *
- * @group double_funcs
+ * Computes the tangent inverse of the given value.
*/
- def sinh(columnName: String): Column = sinh(Column(columnName))
+ def atan(e: Column): Column = Atan(e.expr)
/**
- * Computes the cosine of the given value.
- *
- * @group double_funcs
+ * Computes the tangent inverse of the given column.
*/
- def cos(e: Column): Column = Cos(e.expr)
+ def atan(columnName: String): Column = atan(Column(columnName))
/**
- * Computes the cosine of the given column.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def cos(columnName: String): Column = cos(Column(columnName))
+ def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
/**
- * Computes the cosine inverse of the given value; the returned angle is in the range
- * 0.0 through pi.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def acos(e: Column): Column = Acos(e.expr)
+ def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
/**
- * Computes the cosine inverse of the given column; the returned angle is in the range
- * 0.0 through pi.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def acos(columnName: String): Column = acos(Column(columnName))
+ def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
/**
- * Computes the hyperbolic cosine of the given value.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def cosh(e: Column): Column = Cosh(e.expr)
+ def atan2(leftName: String, rightName: String): Column =
+ atan2(Column(leftName), Column(rightName))
/**
- * Computes the hyperbolic cosine of the given column.
- *
- * @group double_funcs
- */
- def cosh(columnName: String): Column = cosh(Column(columnName))
-
- /**
- * Computes the tangent of the given value.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def tan(e: Column): Column = Tan(e.expr)
+ def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
/**
- * Computes the tangent of the given column.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).=
*/
- def tan(columnName: String): Column = tan(Column(columnName))
+ def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
/**
- * Computes the tangent inverse of the given value.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def atan(e: Column): Column = Atan(e.expr)
+ def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
/**
- * Computes the tangent inverse of the given column.
- *
- * @group double_funcs
+ * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
+ * polar coordinates (r, theta).
*/
- def atan(columnName: String): Column = atan(Column(columnName))
+ def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
/**
- * Computes the hyperbolic tangent of the given value.
- *
- * @group double_funcs
+ * Computes the cube-root of the given value.
*/
- def tanh(e: Column): Column = Tanh(e.expr)
+ def cbrt(e: Column): Column = Cbrt(e.expr)
/**
- * Computes the hyperbolic tangent of the given column.
- *
- * @group double_funcs
+ * Computes the cube-root of the given column.
*/
- def tanh(columnName: String): Column = tanh(Column(columnName))
+ def cbrt(columnName: String): Column = cbrt(Column(columnName))
/**
- * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
- *
- * @group double_funcs
+ * Computes the ceiling of the given value.
*/
- def toDeg(e: Column): Column = ToDegrees(e.expr)
+ def ceil(e: Column): Column = Ceil(e.expr)
/**
- * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
- *
- * @group double_funcs
+ * Computes the ceiling of the given column.
*/
- def toDeg(columnName: String): Column = toDeg(Column(columnName))
+ def ceil(columnName: String): Column = ceil(Column(columnName))
/**
- * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
- *
- * @group double_funcs
+ * Computes the cosine of the given value.
*/
- def toRad(e: Column): Column = ToRadians(e.expr)
+ def cos(e: Column): Column = Cos(e.expr)
/**
- * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
- *
- * @group double_funcs
+ * Computes the cosine of the given column.
*/
- def toRad(columnName: String): Column = toRad(Column(columnName))
+ def cos(columnName: String): Column = cos(Column(columnName))
/**
- * Computes the ceiling of the given value.
- *
- * @group double_funcs
+ * Computes the hyperbolic cosine of the given value.
*/
- def ceil(e: Column): Column = Ceil(e.expr)
+ def cosh(e: Column): Column = Cosh(e.expr)
/**
- * Computes the ceiling of the given column.
- *
- * @group double_funcs
+ * Computes the hyperbolic cosine of the given column.
*/
- def ceil(columnName: String): Column = ceil(Column(columnName))
+ def cosh(columnName: String): Column = cosh(Column(columnName))
/**
- * Computes the floor of the given value.
- *
- * @group double_funcs
+ * Computes the exponential of the given value.
*/
- def floor(e: Column): Column = Floor(e.expr)
+ def exp(e: Column): Column = Exp(e.expr)
/**
- * Computes the floor of the given column.
- *
- * @group double_funcs
+ * Computes the exponential of the given column.
*/
- def floor(columnName: String): Column = floor(Column(columnName))
+ def exp(columnName: String): Column = exp(Column(columnName))
/**
- * Returns the double value that is closest in value to the argument and
- * is equal to a mathematical integer.
- *
- * @group double_funcs
+ * Computes the exponential of the given value minus one.
*/
- def rint(e: Column): Column = Rint(e.expr)
+ def expm1(e: Column): Column = Expm1(e.expr)
/**
- * Returns the double value that is closest in value to the argument and
- * is equal to a mathematical integer.
- *
- * @group double_funcs
+ * Computes the exponential of the given column.
*/
- def rint(columnName: String): Column = rint(Column(columnName))
+ def expm1(columnName: String): Column = expm1(Column(columnName))
/**
- * Computes the cube-root of the given value.
- *
- * @group double_funcs
+ * Computes the floor of the given value.
*/
- def cbrt(e: Column): Column = Cbrt(e.expr)
+ def floor(e: Column): Column = Floor(e.expr)
/**
- * Computes the cube-root of the given column.
- *
- * @group double_funcs
+ * Computes the floor of the given column.
*/
- def cbrt(columnName: String): Column = cbrt(Column(columnName))
+ def floor(columnName: String): Column = floor(Column(columnName))
/**
- * Computes the signum of the given value.
- *
- * @group double_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def signum(e: Column): Column = Signum(e.expr)
+ def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
/**
- * Computes the signum of the given column.
- *
- * @group double_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def signum(columnName: String): Column = signum(Column(columnName))
+ def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
/**
- * Computes the signum of the given value. For IntegerType.
- *
- * @group int_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def isignum(e: Column): Column = ISignum(e.expr)
+ def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
/**
- * Computes the signum of the given column. For IntegerType.
- *
- * @group int_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def isignum(columnName: String): Column = isignum(Column(columnName))
+ def hypot(leftName: String, rightName: String): Column =
+ hypot(Column(leftName), Column(rightName))
/**
- * Computes the signum of the given value. For FloatType.
- *
- * @group float_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def fsignum(e: Column): Column = FSignum(e.expr)
+ def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
/**
- * Computes the signum of the given column. For FloatType.
- *
- * @group float_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def fsignum(columnName: String): Column = fsignum(Column(columnName))
+ def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
/**
- * Computes the signum of the given value. For LongType.
- *
- * @group long_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def lsignum(e: Column): Column = LSignum(e.expr)
+ def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
/**
- * Computes the signum of the given column. For FloatType.
- *
- * @group long_funcs
+ * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
*/
- def lsignum(columnName: String): Column = lsignum(Column(columnName))
+ def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
/**
* Computes the natural logarithm of the given value.
- *
- * @group double_funcs
*/
def log(e: Column): Column = Log(e.expr)
/**
* Computes the natural logarithm of the given column.
- *
- * @group double_funcs
*/
def log(columnName: String): Column = log(Column(columnName))
/**
* Computes the logarithm of the given value in Base 10.
- *
- * @group double_funcs
*/
def log10(e: Column): Column = Log10(e.expr)
/**
* Computes the logarithm of the given value in Base 10.
- *
- * @group double_funcs
*/
def log10(columnName: String): Column = log10(Column(columnName))
/**
* Computes the natural logarithm of the given value plus one.
- *
- * @group double_funcs
*/
def log1p(e: Column): Column = Log1p(e.expr)
/**
* Computes the natural logarithm of the given column plus one.
- *
- * @group double_funcs
*/
def log1p(columnName: String): Column = log1p(Column(columnName))
/**
- * Computes the exponential of the given value.
- *
- * @group double_funcs
- */
- def exp(e: Column): Column = Exp(e.expr)
-
- /**
- * Computes the exponential of the given column.
- *
- * @group double_funcs
- */
- def exp(columnName: String): Column = exp(Column(columnName))
-
- /**
- * Computes the exponential of the given value minus one.
- *
- * @group double_funcs
- */
- def expm1(e: Column): Column = Expm1(e.expr)
-
- /**
- * Computes the exponential of the given column.
- *
- * @group double_funcs
- */
- def expm1(columnName: String): Column = expm1(Column(columnName))
-
- /**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(l: Column, r: Column): Column = Pow(l.expr, r.expr)
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(l: Column, rightName: String): Column = pow(l, Column(rightName))
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(leftName: String, r: Column): Column = pow(Column(leftName), r)
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName))
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(l: Column, r: Double): Column = pow(l, lit(r).expr)
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(leftName: String, r: Double): Column = pow(Column(leftName), r)
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(l: Double, r: Column): Column = pow(lit(l).expr, r)
/**
* Returns the value of the first argument raised to the power of the second argument.
- *
- * @group double_funcs
*/
def pow(l: Double, rightName: String): Column = pow(l, Column(rightName))
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Returns the double value that is closest in value to the argument and
+ * is equal to a mathematical integer.
*/
- def hypot(l: Column, r: Column): Column = Hypot(l.expr, r.expr)
+ def rint(e: Column): Column = Rint(e.expr)
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Returns the double value that is closest in value to the argument and
+ * is equal to a mathematical integer.
*/
- def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName))
+ def rint(columnName: String): Column = rint(Column(columnName))
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Computes the signum of the given value.
*/
- def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r)
+ def signum(e: Column): Column = Signum(e.expr)
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Computes the signum of the given column.
*/
- def hypot(leftName: String, rightName: String): Column =
- hypot(Column(leftName), Column(rightName))
+ def signum(columnName: String): Column = signum(Column(columnName))
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Computes the sine of the given value.
*/
- def hypot(l: Column, r: Double): Column = hypot(l, lit(r).expr)
+ def sin(e: Column): Column = Sin(e.expr)
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Computes the sine of the given column.
*/
- def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r)
+ def sin(columnName: String): Column = sin(Column(columnName))
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Computes the hyperbolic sine of the given value.
*/
- def hypot(l: Double, r: Column): Column = hypot(lit(l).expr, r)
+ def sinh(e: Column): Column = Sinh(e.expr)
/**
- * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow.
- *
- * @group double_funcs
+ * Computes the hyperbolic sine of the given column.
*/
- def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName))
-
+ def sinh(columnName: String): Column = sinh(Column(columnName))
+
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Computes the tangent of the given value.
*/
- def atan2(l: Column, r: Column): Column = Atan2(l.expr, r.expr)
+ def tan(e: Column): Column = Tan(e.expr)
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Computes the tangent of the given column.
*/
- def atan2(l: Column, rightName: String): Column = atan2(l, Column(rightName))
-
+ def tan(columnName: String): Column = tan(Column(columnName))
+
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Computes the hyperbolic tangent of the given value.
*/
- def atan2(leftName: String, r: Column): Column = atan2(Column(leftName), r)
+ def tanh(e: Column): Column = Tanh(e.expr)
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Computes the hyperbolic tangent of the given column.
*/
- def atan2(leftName: String, rightName: String): Column =
- atan2(Column(leftName), Column(rightName))
+ def tanh(columnName: String): Column = tanh(Column(columnName))
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
*/
- def atan2(l: Column, r: Double): Column = atan2(l, lit(r).expr)
+ def toDeg(e: Column): Column = ToDegrees(e.expr)
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Converts an angle measured in radians to an approximately equivalent angle measured in degrees.
*/
- def atan2(leftName: String, r: Double): Column = atan2(Column(leftName), r)
+ def toDeg(columnName: String): Column = toDeg(Column(columnName))
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
*/
- def atan2(l: Double, r: Column): Column = atan2(lit(l).expr, r)
+ def toRad(e: Column): Column = ToRadians(e.expr)
/**
- * Returns the angle theta from the conversion of rectangular coordinates (x, y) to
- * polar coordinates (r, theta).
- *
- * @group double_funcs
+ * Converts an angle measured in degrees to an approximately equivalent angle measured in radians.
*/
- def atan2(l: Double, rightName: String): Column = atan2(l, Column(rightName))
+ def toRad(columnName: String): Column = toRad(Column(columnName))
}
http://git-wip-us.apache.org/repos/asf/spark/blob/fe917f5e/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
index 561553c..9e19bb7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/MathExpressionsSuite.scala
@@ -194,18 +194,6 @@ class MathExpressionsSuite extends QueryTest {
testOneToOneMathFunction[Double](signum, math.signum)
}
- test("isignum") {
- testOneToOneMathFunction[Int](isignum, math.signum)
- }
-
- test("fsignum") {
- testOneToOneMathFunction[Float](fsignum, math.signum)
- }
-
- test("lsignum") {
- testOneToOneMathFunction[Long](lsignum, math.signum)
- }
-
test("pow") {
testTwoToOneMathFunction(pow, pow, math.pow)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org