You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/05/15 05:49:28 UTC
spark git commit: [SPARK-7098][SQL] Make the WHERE clause with
timestamp show consistent result
Repository: spark
Updated Branches:
refs/heads/master 6d0633e3e -> f9705d461
[SPARK-7098][SQL] Make the WHERE clause with timestamp show consistent result
JIRA: https://issues.apache.org/jira/browse/SPARK-7098
The WHERE clause with timstamp shows inconsistent results. This pr fixes it.
Author: Liang-Chi Hsieh <vi...@gmail.com>
Closes #5682 from viirya/consistent_timestamp and squashes the following commits:
171445a [Liang-Chi Hsieh] Merge remote-tracking branch 'upstream/master' into consistent_timestamp
4e98520 [Liang-Chi Hsieh] Make the WHERE clause with timestamp show consistent result.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f9705d46
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f9705d46
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f9705d46
Branch: refs/heads/master
Commit: f9705d461350c6fccf8022e933ea909f40c53576
Parents: 6d0633e
Author: Liang-Chi Hsieh <vi...@gmail.com>
Authored: Thu May 14 20:49:21 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu May 14 20:49:21 2015 -0700
----------------------------------------------------------------------
.../apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala | 6 +++---
.../src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 4 ++++
sql/core/src/test/scala/org/apache/spark/sql/TestData.scala | 2 +-
3 files changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f9705d46/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
index 168a4e3..fe0d3f2 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala
@@ -251,10 +251,10 @@ trait HiveTypeCoercion {
p.makeCopy(Array(Cast(p.left, StringType), p.right))
case p: BinaryComparison if p.left.dataType == StringType &&
p.right.dataType == TimestampType =>
- p.makeCopy(Array(p.left, Cast(p.right, StringType)))
+ p.makeCopy(Array(Cast(p.left, TimestampType), p.right))
case p: BinaryComparison if p.left.dataType == TimestampType &&
p.right.dataType == StringType =>
- p.makeCopy(Array(Cast(p.left, StringType), p.right))
+ p.makeCopy(Array(p.left, Cast(p.right, TimestampType)))
case p: BinaryComparison if p.left.dataType == TimestampType &&
p.right.dataType == DateType =>
p.makeCopy(Array(Cast(p.left, StringType), Cast(p.right, StringType)))
@@ -274,7 +274,7 @@ trait HiveTypeCoercion {
i.makeCopy(Array(Cast(a, StringType), b))
case i @ In(a, b) if a.dataType == TimestampType &&
b.forall(_.dataType == StringType) =>
- i.makeCopy(Array(Cast(a, StringType), b))
+ i.makeCopy(Array(a, b.map(Cast(_, TimestampType))))
case i @ In(a, b) if a.dataType == DateType &&
b.forall(_.dataType == TimestampType) =>
i.makeCopy(Array(Cast(a, StringType), b.map(Cast(_, StringType))))
http://git-wip-us.apache.org/repos/asf/spark/blob/f9705d46/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
index 8cdbe07..479ad9f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala
@@ -298,6 +298,10 @@ class SQLQuerySuite extends QueryTest with BeforeAndAfterAll {
test("SPARK-3173 Timestamp support in the parser") {
checkAnswer(sql(
+ "SELECT time FROM timestamps WHERE time='1969-12-31 16:00:00.0'"),
+ Row(java.sql.Timestamp.valueOf("1969-12-31 16:00:00")))
+
+ checkAnswer(sql(
"SELECT time FROM timestamps WHERE time=CAST('1969-12-31 16:00:00.001' AS TIMESTAMP)"),
Row(java.sql.Timestamp.valueOf("1969-12-31 16:00:00.001")))
http://git-wip-us.apache.org/repos/asf/spark/blob/f9705d46/sql/core/src/test/scala/org/apache/spark/sql/TestData.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/TestData.scala b/sql/core/src/test/scala/org/apache/spark/sql/TestData.scala
index 446771a..8fbc2d2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/TestData.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/TestData.scala
@@ -175,7 +175,7 @@ object TestData {
"4, D4, true, 2147483644" :: Nil)
case class TimestampField(time: Timestamp)
- val timestamps = TestSQLContext.sparkContext.parallelize((1 to 3).map { i =>
+ val timestamps = TestSQLContext.sparkContext.parallelize((0 to 3).map { i =>
TimestampField(new Timestamp(i))
})
timestamps.toDF().registerTempTable("timestamps")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org