You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by da...@apache.org on 2015/06/28 17:29:13 UTC
spark git commit: [SPARK-8686] [SQL] DataFrame should support `where`
with expression represented by String
Repository: spark
Updated Branches:
refs/heads/master 77da5be6f -> ec7843819
[SPARK-8686] [SQL] DataFrame should support `where` with expression represented by String
DataFrame supports `filter` function with two types of argument, `Column` and `String`. But `where` doesn't.
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Closes #7063 from sarutak/SPARK-8686 and squashes the following commits:
180f9a4 [Kousuke Saruta] Added test
d61aec4 [Kousuke Saruta] Add "where" method with String argument to DataFrame
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ec784381
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ec784381
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ec784381
Branch: refs/heads/master
Commit: ec784381967506f8db4d6a357c0b72df25a0aa1b
Parents: 77da5be
Author: Kousuke Saruta <sa...@oss.nttdata.co.jp>
Authored: Sun Jun 28 08:29:07 2015 -0700
Committer: Davies Liu <da...@databricks.com>
Committed: Sun Jun 28 08:29:07 2015 -0700
----------------------------------------------------------------------
.../src/main/scala/org/apache/spark/sql/DataFrame.scala | 12 ++++++++++++
.../scala/org/apache/spark/sql/DataFrameSuite.scala | 6 ++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ec784381/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
index 0db4df3..d75d883 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala
@@ -715,6 +715,18 @@ class DataFrame private[sql](
def where(condition: Column): DataFrame = filter(condition)
/**
+ * Filters rows using the given SQL expression.
+ * {{{
+ * peopleDf.where("age > 15")
+ * }}}
+ * @group dfops
+ * @since 1.5.0
+ */
+ def where(conditionExpr: String): DataFrame = {
+ filter(Column(new SqlParser().parseExpression(conditionExpr)))
+ }
+
+ /**
* Groups the [[DataFrame]] using the specified columns, so we can run aggregation on them.
* See [[GroupedData]] for all the available aggregate functions.
*
http://git-wip-us.apache.org/repos/asf/spark/blob/ec784381/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 47443a9..d06b9c5 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -160,6 +160,12 @@ class DataFrameSuite extends QueryTest {
testData.collect().filter(_.getInt(0) > 90).toSeq)
}
+ test("filterExpr using where") {
+ checkAnswer(
+ testData.where("key > 50"),
+ testData.collect().filter(_.getInt(0) > 50).toSeq)
+ }
+
test("repartition") {
checkAnswer(
testData.select('key).repartition(10).select('key),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org