You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@drill.apache.org by cg...@apache.org on 2021/09/09 02:45:09 UTC
[drill] branch master updated: DRILL-7989: Use the UTC formatter in
the JSON reader (#2299)
This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new dc8cbb5 DRILL-7989: Use the UTC formatter in the JSON reader (#2299)
dc8cbb5 is described below
commit dc8cbb55e09e8d95ff6704f6985babd3bb99d434
Author: luoc <lu...@apache.org>
AuthorDate: Thu Sep 9 10:45:02 2021 +0800
DRILL-7989: Use the UTC formatter in the JSON reader (#2299)
* DRILL-7989: Use the UTC formatter in the JSON reader
* Add the offset to local instant
* Remove the unused import
---
.../drill/exec/vector/complex/fn/VectorOutput.java | 22 ++++++++++++++++------
.../java/org/apache/drill/TestFrameworkTest.java | 12 ++++++++++--
.../exec/store/json/TestJsonRecordReader.java | 5 +++++
.../src/test/resources/jsoninput/input2.json | 15 ++++++++++++---
4 files changed, 43 insertions(+), 11 deletions(-)
diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/VectorOutput.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/VectorOutput.java
index 835a873..7a22fb1 100644
--- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/VectorOutput.java
+++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/VectorOutput.java
@@ -18,10 +18,12 @@
package org.apache.drill.exec.vector.complex.fn;
import java.io.IOException;
+import java.time.Instant;
import java.time.LocalDate;
import java.time.LocalTime;
import java.time.OffsetDateTime;
import java.time.OffsetTime;
+import java.time.ZoneId;
import java.time.ZoneOffset;
import org.apache.drill.common.exceptions.UserException;
@@ -248,9 +250,13 @@ abstract class VectorOutput {
ts.writeTimeStamp(dt.getMillis());
break;
case VALUE_STRING:
- OffsetDateTime originalDateTime = OffsetDateTime.parse(parser.getValueAsString(), DateUtility.isoFormatTimeStamp);
- OffsetDateTime utcDateTime = OffsetDateTime.of(originalDateTime.toLocalDateTime(), ZoneOffset.UTC); // strips the time zone from the original
- ts.writeTimeStamp(utcDateTime.toInstant().toEpochMilli());
+ // Note mongo use the UTC format to specify the date value by default.
+ // See the mongo specs and the Drill handler (in new JSON loader) :
+ // 1. https://docs.mongodb.com/manual/reference/mongodb-extended-json
+ // 2. org.apache.drill.exec.store.easy.json.values.UtcTimestampValueListener
+ Instant instant = Instant.parse(parser.getValueAsString());
+ long offset = ZoneId.systemDefault().getRules().getOffset(instant).getTotalSeconds() * 1000;
+ ts.writeTimeStamp(instant.toEpochMilli() + offset);
break;
default:
throw UserException.unsupportedError()
@@ -351,9 +357,13 @@ abstract class VectorOutput {
ts.writeTimeStamp(dt.getMillis());
break;
case VALUE_STRING:
- OffsetDateTime originalDateTime = OffsetDateTime.parse(parser.getValueAsString(), DateUtility.isoFormatTimeStamp);
- OffsetDateTime utcDateTime = OffsetDateTime.of(originalDateTime.toLocalDateTime(), ZoneOffset.UTC); // strips the time zone from the original
- ts.writeTimeStamp(utcDateTime.toInstant().toEpochMilli());
+ // Note mongo use the UTC format to specify the date value by default.
+ // See the mongo specs and the Drill handler (in new JSON loader) :
+ // 1. https://docs.mongodb.com/manual/reference/mongodb-extended-json
+ // 2. org.apache.drill.exec.store.easy.json.values.UtcTimestampValueListener
+ Instant instant = Instant.parse(parser.getValueAsString());
+ long offset = ZoneId.systemDefault().getRules().getOffset(instant).getTotalSeconds() * 1000;
+ ts.writeTimeStamp(instant.toEpochMilli() + offset);
break;
default:
throw UserException.unsupportedError()
diff --git a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
index c1f4103..5ff50b9 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/TestFrameworkTest.java
@@ -24,6 +24,9 @@ import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertTrue;
import java.math.BigDecimal;
+import java.time.Instant;
+import java.time.LocalDateTime;
+import java.time.ZoneId;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -206,10 +209,14 @@ public class TestFrameworkTest extends BaseTestQuery {
@Test
public void testBaselineValsVerificationWithComplexAndNulls() throws Exception {
+ LocalDateTime localDT = LocalDateTime.of(2019, 9, 30, 20, 47, 43, 123);
+ Instant instant = localDT.atZone(ZoneId.systemDefault()).toInstant();
+ long ts = instant.toEpochMilli() + instant.getNano();
+ ts = ts + ZoneId.systemDefault().getRules().getOffset(instant).getTotalSeconds() * 1000;
testBuilder()
.sqlQuery("select * from cp.`jsoninput/input2.json` limit 1")
.ordered()
- .baselineColumns("integer", "float", "x", "z", "l", "rl")
+ .baselineColumns("integer", "float", "x", "z", "l", "rl", "`date`")
.baselineValues(2010l,
17.4,
mapOf("y", "kevin",
@@ -219,7 +226,8 @@ public class TestFrameworkTest extends BaseTestQuery {
mapOf("pink", "purple")),
listOf(4l, 2l),
listOf(listOf(2l, 1l),
- listOf(4l, 6l)))
+ listOf(4l, 6l)),
+ LocalDateTime.ofInstant(Instant.ofEpochMilli(ts), ZoneId.systemDefault()))
.build().run();
}
diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
index 719c3a3..bf83ae2 100644
--- a/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
+++ b/exec/java-exec/src/test/java/org/apache/drill/exec/store/json/TestJsonRecordReader.java
@@ -44,6 +44,11 @@ public class TestJsonRecordReader extends BaseTestQuery {
}
@Test
+ public void testDateJsonInput() throws Exception {
+ test("select `date`, AGE(`date`, CAST('2019-09-30 20:47:43' as timestamp)) from cp.`jsoninput/input2.json` limit 10 ");
+ }
+
+ @Test
public void testContainingArray() throws Exception {
test("select * from cp.`store/json/listdoc.json`");
}
diff --git a/exec/java-exec/src/test/resources/jsoninput/input2.json b/exec/java-exec/src/test/resources/jsoninput/input2.json
index d37f686..e95442e 100644
--- a/exec/java-exec/src/test/resources/jsoninput/input2.json
+++ b/exec/java-exec/src/test/resources/jsoninput/input2.json
@@ -9,7 +9,10 @@
{"pink" : "purple" }
],
"l": [4,2],
- "rl": [ [2,1], [4,6] ]
+ "rl": [ [2,1], [4,6] ],
+ "date" : {
+ "$date" : "2019-09-30T20:47:43.123Z"
+ }
}
{ "integer" : -2002,
"float" : -1.2
@@ -24,7 +27,10 @@
{"pink" : "lilac" }
],
"l": [4,2],
- "rl": [ [2,1], [4,6] ]
+ "rl": [ [2,1], [4,6] ],
+ "date" : {
+ "$date" : "2019-09-30T20:47:43.10Z"
+ }
}
{ "integer" : 6005,
"float" : 1.2,
@@ -36,5 +42,8 @@
{"orange" : "stucco" }
],
"l": [4,2],
- "rl": [ [2,1], [4,6] ]
+ "rl": [ [2,1], [4,6] ],
+ "date" : {
+ "$date" : "2019-09-30T20:47:43.1Z"
+ }
}