You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2019/07/09 02:57:59 UTC
[spark] branch branch-2.3 updated: [SPARK-28308][CORE]
CalendarInterval sub-second part should be padded before parsing
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 8d7a724 [SPARK-28308][CORE] CalendarInterval sub-second part should be padded before parsing
8d7a724 is described below
commit 8d7a724e9bb634897f71b124db9057ee367b7c21
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Mon Jul 8 19:40:41 2019 -0700
[SPARK-28308][CORE] CalendarInterval sub-second part should be padded before parsing
The sub-second part of the interval should be padded before parsing. Currently, Spark gives a correct value only when there is 9 digits below `.`.
```
spark-sql> select interval '0 0:0:0.123456789' day to second;
interval 123 milliseconds 456 microseconds
spark-sql> select interval '0 0:0:0.12345678' day to second;
interval 12 milliseconds 345 microseconds
spark-sql> select interval '0 0:0:0.1234' day to second;
interval 1 microseconds
```
Pass the Jenkins with the fixed test cases.
Closes #25079 from dongjoon-hyun/SPARK-28308.
Authored-by: Dongjoon Hyun <dh...@apple.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
(cherry picked from commit a5ff9221fc23cc758db228493d451f542591eff7)
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
(cherry picked from commit 072e0eb8881be7df7d5c81efa472b448b9d67e95)
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java | 3 ++-
.../java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java | 3 ++-
.../scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 4 +++-
3 files changed, 7 insertions(+), 3 deletions(-)
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
index e36efa3..1818fef 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/CalendarInterval.java
@@ -177,7 +177,8 @@ public final class CalendarInterval implements Serializable {
long minutes = toLongWithRange("minute", m.group(4), 0, 59);
long seconds = toLongWithRange("second", m.group(5), 0, 59);
// Hive allow nanosecond precision interval
- long nanos = toLongWithRange("nanosecond", m.group(7), 0L, 999999999L);
+ String nanoStr = m.group(7) == null ? null : (m.group(7) + "000000000").substring(0, 9);
+ long nanos = toLongWithRange("nanosecond", nanoStr, 0L, 999999999L);
result = new CalendarInterval(0, sign * (
days * MICROS_PER_DAY + hours * MICROS_PER_HOUR + minutes * MICROS_PER_MINUTE +
seconds * MICROS_PER_SECOND + nanos / 1000L));
diff --git a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
index 994af8f..c125ba5 100644
--- a/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
+++ b/common/unsafe/src/test/java/org/apache/spark/unsafe/types/CalendarIntervalSuite.java
@@ -162,7 +162,8 @@ public class CalendarIntervalSuite {
assertEquals(fromDayTimeString(input), i);
input = "10 0:12:0.888";
- i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE);
+ i = new CalendarInterval(0, 10 * MICROS_PER_DAY + 12 * MICROS_PER_MINUTE +
+ 888 * MICROS_PER_MILLI);
assertEquals(fromDayTimeString(input), i);
input = "-3 0:0:0";
diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index d11270e..d81606e 100644
--- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -1176,11 +1176,13 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils with TestHiveSingleton {
}
test("Convert hive interval term into Literal of CalendarIntervalType") {
+ checkAnswer(sql("select interval '0 0:0:0.1' day to second"),
+ Row(CalendarInterval.fromString("interval 100 milliseconds")))
checkAnswer(sql("select interval '10-9' year to month"),
Row(CalendarInterval.fromString("interval 10 years 9 months")))
checkAnswer(sql("select interval '20 15:40:32.99899999' day to second"),
Row(CalendarInterval.fromString("interval 2 weeks 6 days 15 hours 40 minutes " +
- "32 seconds 99 milliseconds 899 microseconds")))
+ "32 seconds 998 milliseconds 999 microseconds")))
checkAnswer(sql("select interval '30' year"),
Row(CalendarInterval.fromString("interval 30 years")))
checkAnswer(sql("select interval '25' month"),
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org