You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/12/01 22:29:02 UTC
spark git commit: [SPARK-4650][SQL] Supporting multi column support
in countDistinct function like count(distinct c1, c2..) in Spark SQL
Repository: spark
Updated Branches:
refs/heads/master b57365a1e -> 6a9ff19dc
[SPARK-4650][SQL] Supporting multi column support in countDistinct function like count(distinct c1,c2..) in Spark SQL
Supporting multi column support in countDistinct function like count(distinct c1,c2..) in Spark SQL
Author: ravipesala <ra...@huawei.com>
Author: Michael Armbrust <mi...@databricks.com>
Closes #3511 from ravipesala/countdistinct and squashes the following commits:
cc4dbb1 [ravipesala] style
070e12a [ravipesala] Supporting multi column support in count(distinct c1,c2..) in Spark SQL
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6a9ff19d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6a9ff19d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6a9ff19d
Branch: refs/heads/master
Commit: 6a9ff19dc06745144d5b311d4f87073c81d53a8f
Parents: b57365a
Author: ravipesala <ra...@huawei.com>
Authored: Mon Dec 1 13:26:44 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Mon Dec 1 13:28:04 2014 -0800
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/catalyst/SqlParser.scala | 3 ++-
.../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 7 +++++++
2 files changed, 9 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/6a9ff19d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
index a9ff10f..a2bcd73 100755
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/SqlParser.scala
@@ -277,7 +277,8 @@ class SqlParser extends AbstractSparkSQLParser {
| SUM ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ { case exp => SumDistinct(exp) }
| COUNT ~ "(" ~> "*" <~ ")" ^^ { case _ => Count(Literal(1)) }
| COUNT ~ "(" ~> expression <~ ")" ^^ { case exp => Count(exp) }
- | COUNT ~> "(" ~> DISTINCT ~> expression <~ ")" ^^ { case exp => CountDistinct(exp :: Nil) }
+ | COUNT ~> "(" ~> DISTINCT ~> repsep(expression, ",") <~ ")" ^^
+ { case exps => CountDistinct(exps) }
| APPROXIMATE ~ COUNT ~ "(" ~ DISTINCT ~> expression <~ ")" ^^
{ case exp => ApproxCountDistinct(exp) }
| APPROXIMATE ~> "(" ~> floatLit ~ ")" ~ COUNT ~ "(" ~ DISTINCT ~ expression <~ ")" ^^
http://git-wip-us.apache.org/repos/asf/spark/blob/6a9ff19d/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 84ee305..f83e647 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -992,4 +992,11 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
"nulldata2 on nulldata1.value <=> nulldata2.value"),
(1 to 2).map(i => Seq(i)))
}
+
+ test("Multi-column COUNT(DISTINCT ...)") {
+ val data = TestData(1,"val_1") :: TestData(2,"val_2") :: Nil
+ val rdd = sparkContext.parallelize((0 to 1).map(i => data(i)))
+ rdd.registerTempTable("distinctData")
+ checkAnswer(sql("SELECT COUNT(DISTINCT key,value) FROM distinctData"), 2)
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org