You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/07/18 20:25:27 UTC
spark git commit: [SPARK-9167][SQL] use UTC Calendar in `stringToDate`
Repository: spark
Updated Branches:
refs/heads/master 1b4ff0553 -> 692378c01
[SPARK-9167][SQL] use UTC Calendar in `stringToDate`
fix 2 bugs introduced in https://github.com/apache/spark/pull/7353
1. we should use UTC Calendar when cast string to date . Before #7353 , we use `DateTimeUtils.fromJavaDate(Date.valueOf(s.toString))` to cast string to date, and `fromJavaDate` will call `millisToDays` to avoid the time zone issue. Now we use `DateTimeUtils.stringToDate(s)`, we should create a Calendar with UTC in the begging.
2. we should not change the default time zone in test cases. The `threadLocalLocalTimeZone` and `threadLocalTimestampFormat` in `DateTimeUtils` will only be evaluated once for each thread, so we can't set the default time zone back anymore.
Author: Wenchen Fan <cl...@outlook.com>
Closes #7488 from cloud-fan/datetime and squashes the following commits:
9cd6005 [Wenchen Fan] address comments
21ef293 [Wenchen Fan] fix 2 bugs in datetime
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/692378c0
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/692378c0
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/692378c0
Branch: refs/heads/master
Commit: 692378c01d949dfe2b2a884add153cd5f8054b5a
Parents: 1b4ff05
Author: Wenchen Fan <cl...@outlook.com>
Authored: Sat Jul 18 11:25:16 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Sat Jul 18 11:25:16 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/util/DateTimeUtils.scala | 9 +++++----
.../sql/catalyst/expressions/CastSuite.scala | 3 ---
.../sql/catalyst/util/DateTimeUtilsSuite.scala | 19 ++++++++++---------
3 files changed, 15 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/692378c0/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
index f33e34b..45e45ae 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/util/DateTimeUtils.scala
@@ -65,8 +65,8 @@ object DateTimeUtils {
def millisToDays(millisUtc: Long): Int = {
// SPARK-6785: use Math.floor so negative number of days (dates before 1970)
// will correctly work as input for function toJavaDate(Int)
- val millisLocal = millisUtc.toDouble + threadLocalLocalTimeZone.get().getOffset(millisUtc)
- Math.floor(millisLocal / MILLIS_PER_DAY).toInt
+ val millisLocal = millisUtc + threadLocalLocalTimeZone.get().getOffset(millisUtc)
+ Math.floor(millisLocal.toDouble / MILLIS_PER_DAY).toInt
}
// reverse of millisToDays
@@ -375,8 +375,9 @@ object DateTimeUtils {
segments(2) < 1 || segments(2) > 31) {
return None
}
- val c = Calendar.getInstance()
+ val c = Calendar.getInstance(TimeZone.getTimeZone("GMT"))
c.set(segments(0), segments(1) - 1, segments(2), 0, 0, 0)
- Some((c.getTimeInMillis / 1000 / 3600 / 24).toInt)
+ c.set(Calendar.MILLISECOND, 0)
+ Some((c.getTimeInMillis / MILLIS_PER_DAY).toInt)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/692378c0/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index ef8bcd4..ccf448e 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -281,8 +281,6 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
val nts = sts + ".1"
val ts = Timestamp.valueOf(nts)
- val defaultTimeZone = TimeZone.getDefault
- TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"))
var c = Calendar.getInstance()
c.set(2015, 2, 8, 2, 30, 0)
checkEvaluation(cast(cast(new Timestamp(c.getTimeInMillis), StringType), TimestampType),
@@ -291,7 +289,6 @@ class CastSuite extends SparkFunSuite with ExpressionEvalHelper {
c.set(2015, 10, 1, 2, 30, 0)
checkEvaluation(cast(cast(new Timestamp(c.getTimeInMillis), StringType), TimestampType),
c.getTimeInMillis * 1000)
- TimeZone.setDefault(defaultTimeZone)
checkEvaluation(cast("abdef", StringType), "abdef")
checkEvaluation(cast("abdef", DecimalType.Unlimited), null)
http://git-wip-us.apache.org/repos/asf/spark/blob/692378c0/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
index 5c3a621..04c5f09 100644
--- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
+++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/util/DateTimeUtilsSuite.scala
@@ -90,34 +90,35 @@ class DateTimeUtilsSuite extends SparkFunSuite {
}
test("string to date") {
- val millisPerDay = 1000L * 3600L * 24L
+ import DateTimeUtils.millisToDays
+
var c = Calendar.getInstance()
c.set(2015, 0, 28, 0, 0, 0)
c.set(Calendar.MILLISECOND, 0)
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-01-28")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
c.set(2015, 0, 1, 0, 0, 0)
c.set(Calendar.MILLISECOND, 0)
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
c = Calendar.getInstance()
c.set(2015, 2, 1, 0, 0, 0)
c.set(Calendar.MILLISECOND, 0)
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
c = Calendar.getInstance()
c.set(2015, 2, 18, 0, 0, 0)
c.set(Calendar.MILLISECOND, 0)
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18 ")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18 123142")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18T123123")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18T")).get ===
- c.getTimeInMillis / millisPerDay)
+ millisToDays(c.getTimeInMillis))
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015-03-18X")).isEmpty)
assert(DateTimeUtils.stringToDate(UTF8String.fromString("2015/03/18")).isEmpty)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org