You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/12/16 07:25:11 UTC
spark git commit: [SPARK-12314][SQL] isnull operator not pushed down
for JDBC datasource.
Repository: spark
Updated Branches:
refs/heads/master 0f6936b5f -> 7f443a687
[SPARK-12314][SQL] isnull operator not pushed down for JDBC datasource.
https://issues.apache.org/jira/browse/SPARK-12314
`IsNull` filter is not being pushed down for JDBC datasource.
It looks it is SQL standard according to [SQL-92](http://www.contrib.andrew.cmu.edu/~shadow/sql/sql1992.txt), SQL:1999, [SQL:2003](http://www.wiscorp.com/sql_2003_standard.zip) and [SQL:201x](http://www.wiscorp.com/sql20nn.zip) and I believe most databases support this.
In this PR, I simply added the case for `IsNull` filter to produce a proper filter string.
Author: hyukjinkwon <gu...@gmail.com>
This patch had conflicts when merged, resolved by
Committer: Reynold Xin <rx...@databricks.com>
Closes #10286 from HyukjinKwon/SPARK-12314.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/7f443a68
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/7f443a68
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/7f443a68
Branch: refs/heads/master
Commit: 7f443a6879fa33ca8adb682bd85df2d56fb5fcda
Parents: 0f6936b
Author: hyukjinkwon <gu...@gmail.com>
Authored: Tue Dec 15 22:25:08 2015 -0800
Committer: Reynold Xin <rx...@databricks.com>
Committed: Tue Dec 15 22:25:08 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 1 +
sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 1 +
2 files changed, 2 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/7f443a68/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index c18a2d2..3271b46 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -286,6 +286,7 @@ private[sql] class JDBCRDD(
case GreaterThan(attr, value) => s"$attr > ${compileValue(value)}"
case LessThanOrEqual(attr, value) => s"$attr <= ${compileValue(value)}"
case GreaterThanOrEqual(attr, value) => s"$attr >= ${compileValue(value)}"
+ case IsNull(attr) => s"$attr IS NULL"
case _ => null
}
http://git-wip-us.apache.org/repos/asf/spark/blob/7f443a68/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index aca1443..0305667 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -182,6 +182,7 @@ class JDBCSuite extends SparkFunSuite with BeforeAndAfter with SharedSQLContext
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE NAME = 'fred'")).collect().size === 1)
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE NAME > 'fred'")).collect().size === 2)
assert(stripSparkFilter(sql("SELECT * FROM foobar WHERE NAME != 'fred'")).collect().size === 2)
+ assert(stripSparkFilter(sql("SELECT * FROM nulltypes WHERE A IS NULL")).collect().size === 1)
}
test("SELECT * WHERE (quoted strings)") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org