You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/12/02 00:33:12 UTC
spark git commit: [SPARK-11788][SQL] surround timestamp/date value
with quotes in JDBC data source
Repository: spark
Updated Branches:
refs/heads/master 47a0abc34 -> 5a8b5fdd6
[SPARK-11788][SQL] surround timestamp/date value with quotes in JDBC data source
When query the Timestamp or Date column like the following
val filtered = jdbcdf.where($"TIMESTAMP_COLUMN" >= beg && $"TIMESTAMP_COLUMN" < end)
The generated SQL query is "TIMESTAMP_COLUMN >= 2015-01-01 00:00:00.0"
It should have quote around the Timestamp/Date value such as "TIMESTAMP_COLUMN >= '2015-01-01 00:00:00.0'"
Author: Huaxin Gao <hu...@oc0558782468.ibm.com>
Closes #9872 from huaxingao/spark-11788.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5a8b5fdd
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5a8b5fdd
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5a8b5fdd
Branch: refs/heads/master
Commit: 5a8b5fdd6ffa58f015cdadf3f2c6df78e0a388ad
Parents: 47a0abc
Author: Huaxin Gao <hu...@oc0558782468.ibm.com>
Authored: Tue Dec 1 15:32:57 2015 -0800
Committer: Yin Huai <yh...@databricks.com>
Committed: Tue Dec 1 15:32:57 2015 -0800
----------------------------------------------------------------------
.../spark/sql/execution/datasources/jdbc/JDBCRDD.scala | 4 +++-
.../test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala | 11 +++++++++++
2 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/5a8b5fdd/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
index 57a8a04..392d3ed 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/jdbc/JDBCRDD.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.execution.datasources.jdbc
-import java.sql.{Connection, DriverManager, ResultSet, ResultSetMetaData, SQLException}
+import java.sql.{Connection, Date, DriverManager, ResultSet, ResultSetMetaData, SQLException, Timestamp}
import java.util.Properties
import scala.util.control.NonFatal
@@ -267,6 +267,8 @@ private[sql] class JDBCRDD(
*/
private def compileValue(value: Any): Any = value match {
case stringValue: String => s"'${escapeSql(stringValue)}'"
+ case timestampValue: Timestamp => "'" + timestampValue + "'"
+ case dateValue: Date => "'" + dateValue + "'"
case _ => value
}
http://git-wip-us.apache.org/repos/asf/spark/blob/5a8b5fdd/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index d530b1a..8c24aa3 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -484,4 +484,15 @@ class JDBCSuite extends SparkFunSuite with BeforeAndAfter with SharedSQLContext
assert(h2.getTableExistsQuery(table) == defaultQuery)
assert(derby.getTableExistsQuery(table) == defaultQuery)
}
+
+ test("Test DataFrame.where for Date and Timestamp") {
+ // Regression test for bug SPARK-11788
+ val timestamp = java.sql.Timestamp.valueOf("2001-02-20 11:22:33.543543");
+ val date = java.sql.Date.valueOf("1995-01-01")
+ val jdbcDf = sqlContext.read.jdbc(urlWithUserAndPass, "TEST.TIMETYPES", new Properties)
+ val rows = jdbcDf.where($"B" > date && $"C" > timestamp).collect()
+ assert(rows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+ assert(rows(0).getAs[java.sql.Timestamp](2)
+ === java.sql.Timestamp.valueOf("2002-02-20 11:22:33.543543"))
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org