You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@orc.apache.org by om...@apache.org on 2019/08/30 16:35:49 UTC
[orc] branch master updated (ed5c91c -> 69e05e8)
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git.
discard ed5c91c Support timestampformat option in JsonReader
new 69e05e8 ORC-455. Support timestampformat option in JsonReader
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (ed5c91c)
\
N -- N -- N refs/heads/master (69e05e8)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
[orc] 01/01: ORC-455. Support timestampformat option in JsonReader
Posted by om...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
omalley pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/orc.git
commit 69e05e891ac38a1f1d83f5d25443a5e56a550739
Author: yuokada <ca...@gmail.com>
AuthorDate: Sat Jan 12 12:02:20 2019 +0900
ORC-455. Support timestampformat option in JsonReader
Signed-off-by: Owen O'Malley <om...@apache.org>
---
.../org/apache/orc/tools/convert/ConvertTool.java | 2 +-
.../org/apache/orc/tools/convert/JsonReader.java | 24 ++++++++++++----------
2 files changed, 14 insertions(+), 12 deletions(-)
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/ConvertTool.java b/java/tools/src/java/org/apache/orc/tools/convert/ConvertTool.java
index 02d7ee8..2e6ba10 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/ConvertTool.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/ConvertTool.java
@@ -146,7 +146,7 @@ public class ConvertTool {
}
case JSON: {
FSDataInputStream underlying = filesystem.open(path);
- return new JsonReader(getReader(underlying), underlying, size, schema);
+ return new JsonReader(getReader(underlying), underlying, size, schema, timestampFormat);
}
case CSV: {
FSDataInputStream underlying = filesystem.open(path);
diff --git a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
index 1c058fa..7db3c67 100644
--- a/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
+++ b/java/tools/src/java/org/apache/orc/tools/convert/JsonReader.java
@@ -58,8 +58,6 @@ import java.util.Map;
import java.util.zip.GZIPInputStream;
public class JsonReader implements RecordReader {
- private static final DateTimeFormatter DATE_TIME_FORMATTER = DateTimeFormatter.ofPattern(
- "yyyy[[-][/]]MM[[-][/]]dd[['T'][ ]]HH:mm:ss[ ][XXX][X]");
private final TypeDescription schema;
private final Iterator<JsonElement> parser;
@@ -67,6 +65,7 @@ public class JsonReader implements RecordReader {
private final long totalSize;
private final FSDataInputStream input;
private long rowNumber = 0;
+ private final DateTimeFormatter dateTimeFormatter;
interface JsonConverter {
void convert(JsonElement value, ColumnVector vect, int row);
@@ -138,14 +137,14 @@ public class JsonReader implements RecordReader {
}
}
- static class TimestampColumnConverter implements JsonConverter {
+ class TimestampColumnConverter implements JsonConverter {
public void convert(JsonElement value, ColumnVector vect, int row) {
if (value == null || value.isJsonNull()) {
vect.noNulls = false;
vect.isNull[row] = true;
} else {
TimestampColumnVector vector = (TimestampColumnVector) vect;
- TemporalAccessor temporalAccessor = DATE_TIME_FORMATTER.parseBest(value.getAsString(),
+ TemporalAccessor temporalAccessor = dateTimeFormatter.parseBest(value.getAsString(),
ZonedDateTime.FROM, LocalDateTime.FROM);
if (temporalAccessor instanceof ZonedDateTime) {
vector.set(row, new Timestamp(((ZonedDateTime) temporalAccessor).toEpochSecond() * 1000L));
@@ -171,7 +170,7 @@ public class JsonReader implements RecordReader {
}
}
- static class StructColumnConverter implements JsonConverter {
+ class StructColumnConverter implements JsonConverter {
private JsonConverter[] childrenConverters;
private List<String> fieldNames;
@@ -199,7 +198,7 @@ public class JsonReader implements RecordReader {
}
}
- static class ListColumnConverter implements JsonConverter {
+ class ListColumnConverter implements JsonConverter {
private JsonConverter childrenConverter;
public ListColumnConverter(TypeDescription schema) {
@@ -225,7 +224,7 @@ public class JsonReader implements RecordReader {
}
}
- static class MapColumnConverter implements JsonConverter {
+ class MapColumnConverter implements JsonConverter {
private JsonConverter keyConverter;
private JsonConverter valueConverter;
@@ -259,7 +258,7 @@ public class JsonReader implements RecordReader {
}
}
- static JsonConverter createConverter(TypeDescription schema) {
+ JsonConverter createConverter(TypeDescription schema) {
switch (schema.getCategory()) {
case BYTE:
case SHORT:
@@ -296,14 +295,16 @@ public class JsonReader implements RecordReader {
public JsonReader(Reader reader,
FSDataInputStream underlying,
long size,
- TypeDescription schema) throws IOException {
- this(new JsonStreamParser(reader), underlying, size, schema);
+ TypeDescription schema,
+ String timestampFormat) throws IOException {
+ this(new JsonStreamParser(reader), underlying, size, schema, timestampFormat);
}
public JsonReader(Iterator<JsonElement> parser,
FSDataInputStream underlying,
long size,
- TypeDescription schema) throws IOException {
+ TypeDescription schema,
+ String timestampFormat) throws IOException {
this.schema = schema;
if (schema.getCategory() != TypeDescription.Category.STRUCT) {
throw new IllegalArgumentException("Root must be struct - " + schema);
@@ -311,6 +312,7 @@ public class JsonReader implements RecordReader {
this.input = underlying;
this.totalSize = size;
this.parser = parser;
+ this.dateTimeFormatter = DateTimeFormatter.ofPattern(timestampFormat);
List<TypeDescription> fieldTypes = schema.getChildren();
converters = new JsonConverter[fieldTypes.size()];
for(int c = 0; c < converters.length; ++c) {