You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/06/19 08:13:08 UTC
spark git commit: [SPARK-8348][SQL] Add in operator to DataFrame
Column
Repository: spark
Updated Branches:
refs/heads/master a71cbbdea -> 754929b15
[SPARK-8348][SQL] Add in operator to DataFrame Column
I have added it for only Scala.
TODO: we should also support `in` operator in Python.
Author: Yu ISHIKAWA <yu...@gmail.com>
Closes #6824 from yu-iskw/SPARK-8348 and squashes the following commits:
e76d02f [Yu ISHIKAWA] Not use infix notation
6f744ac [Yu ISHIKAWA] Fit the test cases because these used the old test data set.
00077d3 [Yu ISHIKAWA] [SPARK-8348][SQL] Add in operator to DataFrame Column
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/754929b1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/754929b1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/754929b1
Branch: refs/heads/master
Commit: 754929b153aba3a8f8fbafa1581957da4ccc18be
Parents: a71cbbd
Author: Yu ISHIKAWA <yu...@gmail.com>
Authored: Thu Jun 18 23:13:05 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Jun 18 23:13:05 2015 -0700
----------------------------------------------------------------------
.../main/scala/org/apache/spark/sql/Column.scala | 2 +-
.../apache/spark/sql/ColumnExpressionSuite.scala | 16 ++++++++++++++++
2 files changed, 17 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/754929b1/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
index d3efa83..b4e008a 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala
@@ -621,7 +621,7 @@ class Column(protected[sql] val expr: Expression) extends Logging {
* @since 1.3.0
*/
@scala.annotation.varargs
- def in(list: Column*): Column = In(expr, list.map(_.expr))
+ def in(list: Any*): Column = In(expr, list.map(lit(_).expr))
/**
* SQL like expression.
http://git-wip-us.apache.org/repos/asf/spark/blob/754929b1/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
index 5a08578..88bb743 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ColumnExpressionSuite.scala
@@ -296,6 +296,22 @@ class ColumnExpressionSuite extends QueryTest {
checkAnswer(testData.filter($"a".between($"b", $"c")), expectAnswer)
}
+ test("in") {
+ val df = Seq((1, "x"), (2, "y"), (3, "z")).toDF("a", "b")
+ checkAnswer(df.filter($"a".in(1, 2)),
+ df.collect().toSeq.filter(r => r.getInt(0) == 1 || r.getInt(0) == 2))
+ checkAnswer(df.filter($"a".in(3, 2)),
+ df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 2))
+ checkAnswer(df.filter($"a".in(3, 1)),
+ df.collect().toSeq.filter(r => r.getInt(0) == 3 || r.getInt(0) == 1))
+ checkAnswer(df.filter($"b".in("y", "x")),
+ df.collect().toSeq.filter(r => r.getString(1) == "y" || r.getString(1) == "x"))
+ checkAnswer(df.filter($"b".in("z", "x")),
+ df.collect().toSeq.filter(r => r.getString(1) == "z" || r.getString(1) == "x"))
+ checkAnswer(df.filter($"b".in("z", "y")),
+ df.collect().toSeq.filter(r => r.getString(1) == "z" || r.getString(1) == "y"))
+ }
+
val booleanData = ctx.createDataFrame(ctx.sparkContext.parallelize(
Row(false, false) ::
Row(false, true) ::
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org