You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/06/19 03:41:18 UTC
spark git commit: [SPARK-8218][SQL] Binary log math function update.
Repository: spark
Updated Branches:
refs/heads/master 207a98ca5 -> dc4131389
[SPARK-8218][SQL] Binary log math function update.
Some minor updates based on after merging #6725.
Author: Reynold Xin <rx...@databricks.com>
Closes #6871 from rxin/log and squashes the following commits:
ab51542 [Reynold Xin] Use JVM log
76fc8de [Reynold Xin] Fixed arg.
a7c1522 [Reynold Xin] [SPARK-8218][SQL] Binary log math function update.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/dc413138
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/dc413138
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/dc413138
Branch: refs/heads/master
Commit: dc413138995b45a7a957acae007dc11622110310
Parents: 207a98c
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu Jun 18 18:41:15 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Jun 18 18:41:15 2015 -0700
----------------------------------------------------------------------
python/pyspark/sql/functions.py | 13 +++++++++----
.../apache/spark/sql/catalyst/expressions/math.scala | 4 ++++
2 files changed, 13 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/dc413138/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 177fc19..acdb01d 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -404,18 +404,23 @@ def when(condition, value):
return Column(jc)
-@since(1.4)
-def log(col, base=math.e):
+@since(1.5)
+def log(arg1, arg2=None):
"""Returns the first argument-based logarithm of the second argument.
- >>> df.select(log(df.age, 10.0).alias('ten')).map(lambda l: str(l.ten)[:7]).collect()
+ If there is only one argument, then this takes the natural logarithm of the argument.
+
+ >>> df.select(log(10.0, df.age).alias('ten')).map(lambda l: str(l.ten)[:7]).collect()
['0.30102', '0.69897']
>>> df.select(log(df.age).alias('e')).map(lambda l: str(l.e)[:7]).collect()
['0.69314', '1.60943']
"""
sc = SparkContext._active_spark_context
- jc = sc._jvm.functions.log(base, _to_java_column(col))
+ if arg2 is None:
+ jc = sc._jvm.functions.log(_to_java_column(arg1))
+ else:
+ jc = sc._jvm.functions.log(arg1, _to_java_column(arg2))
return Column(jc)
http://git-wip-us.apache.org/repos/asf/spark/blob/dc413138/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/math.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/math.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/math.scala
index 3b83c6d..f79bf4a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/math.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/math.scala
@@ -260,6 +260,10 @@ case class Pow(left: Expression, right: Expression)
case class Logarithm(left: Expression, right: Expression)
extends BinaryMathExpression((c1, c2) => math.log(c2) / math.log(c1), "LOG") {
+
+ /**
+ * Natural log, i.e. using e as the base.
+ */
def this(child: Expression) = {
this(EulerNumber(), child)
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org