You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2016/01/05 19:23:39 UTC

spark git commit: [SPARK-12480][FOLLOW-UP] use a single column vararg for hash

Repository: spark
Updated Branches:
  refs/heads/master 9a6ba7e2c -> 76768337b


[SPARK-12480][FOLLOW-UP] use a single column vararg for hash

address comments in #10435

This makes the API easier to use if user programmatically generate the call to hash, and they will get analysis exception if the arguments of hash is empty.

Author: Wenchen Fan <we...@databricks.com>

Closes #10588 from cloud-fan/hash.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/76768337
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/76768337
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/76768337

Branch: refs/heads/master
Commit: 76768337beec6842660db7522ad15c25ee66d346
Parents: 9a6ba7e
Author: Wenchen Fan <we...@databricks.com>
Authored: Tue Jan 5 10:23:36 2016 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Jan 5 10:23:36 2016 -0800

----------------------------------------------------------------------
 python/pyspark/sql/functions.py                         | 12 ++++++++++++
 .../apache/spark/sql/catalyst/expressions/misc.scala    |  2 +-
 .../catalyst/analysis/ExpressionTypeCheckingSuite.scala |  1 +
 .../src/main/scala/org/apache/spark/sql/functions.scala |  4 ++--
 4 files changed, 16 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/76768337/python/pyspark/sql/functions.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/functions.py b/python/pyspark/sql/functions.py
index 7c15e38..b0390cb 100644
--- a/python/pyspark/sql/functions.py
+++ b/python/pyspark/sql/functions.py
@@ -1018,6 +1018,18 @@ def sha2(col, numBits):
     return Column(jc)
 
 
+@since(2.0)
+def hash(*cols):
+    """Calculates the hash code of given columns, and returns the result as a int column.
+
+    >>> sqlContext.createDataFrame([('ABC',)], ['a']).select(hash('a').alias('hash')).collect()
+    [Row(hash=1358996357)]
+    """
+    sc = SparkContext._active_spark_context
+    jc = sc._jvm.functions.hash(_to_seq(sc, cols, _to_java_column))
+    return Column(jc)
+
+
 # ---------------------- String/Binary functions ------------------------------
 
 _string_functions = {

http://git-wip-us.apache.org/repos/asf/spark/blob/76768337/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
index 8834924..6697d46 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/misc.scala
@@ -200,7 +200,7 @@ case class Murmur3Hash(children: Seq[Expression], seed: Int) extends Expression
 
   override def checkInputDataTypes(): TypeCheckResult = {
     if (children.isEmpty) {
-      TypeCheckResult.TypeCheckFailure("arguments of function hash cannot be empty")
+      TypeCheckResult.TypeCheckFailure("function hash requires at least one argument")
     } else {
       TypeCheckResult.TypeCheckSuccess
     }

http://git-wip-us.apache.org/repos/asf/spark/blob/76768337/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
index 915c585..f3df716 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/analysis/ExpressionTypeCheckingSuite.scala
@@ -163,6 +163,7 @@ class ExpressionTypeCheckingSuite extends SparkFunSuite {
     assertError(Coalesce(Seq('intField, 'booleanField)),
       "input to function coalesce should all be the same type")
     assertError(Coalesce(Nil), "input to function coalesce cannot be empty")
+    assertError(new Murmur3Hash(Nil), "function hash requires at least one argument")
     assertError(Explode('intField),
       "input to function explode should be array or map type")
   }

http://git-wip-us.apache.org/repos/asf/spark/blob/76768337/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
index e223e32..1c96f64 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/functions.scala
@@ -1820,8 +1820,8 @@ object functions extends LegacyFunctions {
    * @since 2.0
    */
   @scala.annotation.varargs
-  def hash(col: Column, cols: Column*): Column = withExpr {
-    new Murmur3Hash((col +: cols).map(_.expr))
+  def hash(cols: Column*): Column = withExpr {
+    new Murmur3Hash(cols.map(_.expr))
   }
 
   //////////////////////////////////////////////////////////////////////////////////////////////


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org