You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2019/10/08 22:31:50 UTC
[orc] branch master updated: ORC-526: Make orc-tools convert
respect second fractions.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/master by this push:
new 5242bff ORC-526: Make orc-tools convert respect second fractions.
5242bff is described below
commit 5242bff6e50879ad5a919e70be023d27e0c16b2d
Author: Yukihiro Okada <ca...@gmail.com>
AuthorDate: Sat Aug 31 13:20:57 2019 +0900
ORC-526: Make orc-tools convert respect second fractions.
Fixes #425
Signed-off-by: Owen O'Malley <om...@apache.org>
---
.../org/apache/orc/tools/convert/CsvReader.java | 12 ++++++---
.../org/apache/orc/tools/convert/JsonReader.java | 10 ++++++--
.../apache/orc/tools/convert/TestCsvReader.java | 11 ++++----
.../apache/orc/tools/convert/TestJsonReader.java | 30 ++++++++++++++++++++++
4 files changed, 52 insertions(+), 11 deletions(-)
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java b/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
index 62c458d..3967a49 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/CsvReader.java
@@ -254,11 +254,15 @@ public class CsvReader implements RecordReader {
dateTimeFormatter.parseBest(values[offset],
ZonedDateTime.FROM, LocalDateTime.FROM);
if (temporalAccessor instanceof ZonedDateTime) {
- vector.set(row, new Timestamp(
- ((ZonedDateTime) temporalAccessor).toEpochSecond() * 1000L));
+ ZonedDateTime zonedDateTime = ((ZonedDateTime) temporalAccessor);
+ Timestamp timestamp = new Timestamp(zonedDateTime.toEpochSecond() * 1000L);
+ timestamp.setNanos(zonedDateTime.getNano());
+ vector.set(row, timestamp);
} else if (temporalAccessor instanceof LocalDateTime) {
- vector.set(row, new Timestamp(((LocalDateTime) temporalAccessor)
- .atZone(ZoneId.systemDefault()).toEpochSecond() * 1000L));
+ ZonedDateTime tz = ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+ Timestamp timestamp = new Timestamp(tz.toEpochSecond() * 1000L);
+ timestamp.setNanos(tz.getNano());
+ vector.set(row, timestamp);
} else {
column.noNulls = false;
column.isNull[row] = true;
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
index 7db3c67..c3720af 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
@@ -147,9 +147,15 @@ public class JsonReader implements RecordReader {
TemporalAccessor temporalAccessor = dateTimeFormatter.parseBest(value.getAsString(),
ZonedDateTime.FROM, LocalDateTime.FROM);
if (temporalAccessor instanceof ZonedDateTime) {
- vector.set(row, new Timestamp(((ZonedDateTime) temporalAccessor).toEpochSecond() * 1000L));
+ ZonedDateTime zonedDateTime = ((ZonedDateTime) temporalAccessor);
+ Timestamp timestamp = new Timestamp(zonedDateTime.toEpochSecond() * 1000L);
+ timestamp.setNanos(zonedDateTime.getNano());
+ vector.set(row, timestamp);
} else if (temporalAccessor instanceof LocalDateTime) {
- vector.set(row, new Timestamp(((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault()).toEpochSecond() * 1000L));
+ ZonedDateTime tz = ((LocalDateTime) temporalAccessor).atZone(ZoneId.systemDefault());
+ Timestamp timestamp = new Timestamp(tz.toEpochSecond() * 1000L);
+ timestamp.setNanos(tz.getNano());
+ vector.set(row, timestamp);
} else {
vect.noNulls = false;
vect.isNull[row] = true;
diff --git a/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java b/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
index 6752de6..06ed5d2 100644
--- a/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
+++ b/java/tools/src/test/org/apache/orc/tools/convert/TestCsvReader.java
@@ -183,10 +183,11 @@ public class TestCsvReader {
@Test
public void testCustomTimestampFormat() throws Exception {
- String tsFormat = "d[d] MMM yyyy HH:mm:ss";
+ String tsFormat = "d[d] MMM yyyy HH:mm:ss.SSSSSS";
+
StringReader input = new StringReader(
- "'21 Mar 2018 12:23:34'\n" +
- "'3 Feb 2018 18:04:51'\n"
+ "'21 Mar 2018 12:23:34.123456'\n" +
+ "'3 Feb 2018 18:04:51.456789'\n"
);
TypeDescription schema = TypeDescription.fromString(
"struct<a:timestamp>");
@@ -196,7 +197,7 @@ public class TestCsvReader {
assertEquals(true, reader.nextBatch(batch));
assertEquals(2, batch.size);
TimestampColumnVector cv = (TimestampColumnVector) batch.cols[0];
- assertEquals("2018-03-21 12:23:34.0", cv.asScratchTimestamp(0).toString());
- assertEquals("2018-02-03 18:04:51.0", cv.asScratchTimestamp(1).toString());
+ assertEquals("2018-03-21 12:23:34.123456", cv.asScratchTimestamp(0).toString());
+ assertEquals("2018-02-03 18:04:51.456789", cv.asScratchTimestamp(1).toString());
}
}
diff --git a/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java b/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java
new file mode 100644
index 0000000..10529d8
--- /dev/null
+++ b/java/tools/src/test/org/apache/orc/tools/convert/TestJsonReader.java
@@ -0,0 +1,30 @@
+package org.apache.orc.tools.convert;
+
+import org.apache.hadoop.hive.ql.exec.vector.TimestampColumnVector;
+import org.apache.hadoop.hive.ql.exec.vector.VectorizedRowBatch;
+import org.apache.orc.TypeDescription;
+import org.junit.Test;
+
+import java.io.StringReader;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestJsonReader {
+ @Test
+ public void testCustomTimestampFormat() throws Exception {
+ String tsFormat = "yyyy-MM-dd HH:mm:ss.SSSSSS";
+
+ String s = "{\"a\":\"2018-03-21 12:23:34.123456\"}\n" +
+ "{\"a\":\"2018-02-03 18:04:51.456789\"}\n";
+ StringReader input = new StringReader(s);
+ TypeDescription schema = TypeDescription.fromString(
+ "struct<a:timestamp>");
+ JsonReader reader = new JsonReader(input, null, 1, schema, tsFormat);
+ VectorizedRowBatch batch = schema.createRowBatch(2);
+ assertEquals(true, reader.nextBatch(batch));
+ assertEquals(2, batch.size);
+ TimestampColumnVector cv = (TimestampColumnVector) batch.cols[0];
+ assertEquals("2018-03-21 12:23:34.123456", cv.asScratchTimestamp(0).toString());
+ assertEquals("2018-02-03 18:04:51.456789", cv.asScratchTimestamp(1).toString());
+ }
+}