You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@arrow.apache.org by we...@apache.org on 2017/02/04 04:00:36 UTC
arrow git commit: ARROW-477: [Java] Add support for
second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer
Repository: arrow
Updated Branches:
refs/heads/master 720d422fa -> 08f38d979
ARROW-477: [Java] Add support for second/microsecond/nanosecond timestamps in-memory and in IPC/JSON layer
Changes include:
- add support for TimeStamp data type with second/microsecond/nanosecond time units
- add an additional readLong() method to timestamp readers to support reading raw long values
- add a simple test case for timestamp readers and writers
Author: Jingyuan Wang <ji...@live.com>
Closes #303 from alphalfalfa/arrow-477 and squashes the following commits:
0199574 [Jingyuan Wang] rename TimeStamp to TimeStampMilli
068e47f [Jingyuan Wang] use a test value that exhibits micro/nanosecond truncation when converting timestamps to JODA DateTime
bef2330 [Jingyuan Wang] fix a typo
9b4d7b4 [Jingyuan Wang] add support for timestamp data type with second/microsecond/nanosecond time units
Project: http://git-wip-us.apache.org/repos/asf/arrow/repo
Commit: http://git-wip-us.apache.org/repos/asf/arrow/commit/08f38d97
Tree: http://git-wip-us.apache.org/repos/asf/arrow/tree/08f38d97
Diff: http://git-wip-us.apache.org/repos/asf/arrow/diff/08f38d97
Branch: refs/heads/master
Commit: 08f38d97904e8d265dea09cdc67946119998e039
Parents: 720d422
Author: Jingyuan Wang <ji...@live.com>
Authored: Fri Feb 3 23:00:25 2017 -0500
Committer: Wes McKinney <we...@twosigma.com>
Committed: Fri Feb 3 23:00:25 2017 -0500
----------------------------------------------------------------------
.../src/main/codegen/data/ValueVectorTypes.tdd | 5 +-
.../main/codegen/templates/ComplexReaders.java | 10 +++
.../codegen/templates/FixedValueVectors.java | 31 ++++++-
.../codegen/templates/NullableValueVectors.java | 8 +-
.../arrow/vector/file/json/JsonFileReader.java | 18 +++-
.../arrow/vector/file/json/JsonFileWriter.java | 18 +++-
.../org/apache/arrow/vector/types/Types.java | 88 +++++++++++++++++---
.../complex/writer/TestComplexWriter.java | 85 +++++++++++++++++++
.../apache/arrow/vector/file/BaseFileTest.java | 10 +--
.../apache/arrow/vector/pojo/TestConvert.java | 2 +-
10 files changed, 250 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
index f7790bb..2181cfd 100644
--- a/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
+++ b/java/vector/src/main/codegen/data/ValueVectorTypes.tdd
@@ -71,7 +71,10 @@
{ class: "UInt8" },
{ class: "Float8", javaType: "double" , boxedType: "Double", fields: [{name: "value", type: "double"}], },
{ class: "Date", javaType: "long", friendlyType: "DateTime" },
- { class: "TimeStamp", javaType: "long", friendlyType: "DateTime" }
+ { class: "TimeStampSec", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+ { class: "TimeStampMilli", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+ { class: "TimeStampMicro", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
+ { class: "TimeStampNano", javaType: "long", boxedType: "Long", friendlyType: "DateTime" }
]
},
{
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/templates/ComplexReaders.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/ComplexReaders.java b/java/vector/src/main/codegen/templates/ComplexReaders.java
index 74a19a6..d537445 100644
--- a/java/vector/src/main/codegen/templates/ComplexReaders.java
+++ b/java/vector/src/main/codegen/templates/ComplexReaders.java
@@ -96,6 +96,16 @@ public class ${name}ReaderImpl extends AbstractFieldReader {
public ${friendlyType} read${safeType}(){
return vector.getAccessor().getObject(idx());
}
+
+ <#if minor.class == "TimeStampSec" ||
+ minor.class == "TimeStampMilli" ||
+ minor.class == "TimeStampMicro" ||
+ minor.class == "TimeStampNano">
+ @Override
+ public ${minor.boxedType} read${minor.boxedType}(){
+ return vector.getAccessor().get(idx());
+ }
+ </#if>
public void copyValue(FieldWriter w){
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/templates/FixedValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/FixedValueVectors.java b/java/vector/src/main/codegen/templates/FixedValueVectors.java
index be385d1..d5265f1 100644
--- a/java/vector/src/main/codegen/templates/FixedValueVectors.java
+++ b/java/vector/src/main/codegen/templates/FixedValueVectors.java
@@ -490,7 +490,16 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return date;
}
- <#elseif minor.class == "TimeStamp">
+ <#elseif minor.class == "TimeStampSec">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ long secs = java.util.concurrent.TimeUnit.SECONDS.toMillis(get(index));
+ org.joda.time.DateTime date = new org.joda.time.DateTime(secs, org.joda.time.DateTimeZone.UTC);
+ date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+ return date;
+ }
+
+ <#elseif minor.class == "TimeStampMilli">
@Override
public ${friendlyType} getObject(int index) {
org.joda.time.DateTime date = new org.joda.time.DateTime(get(index), org.joda.time.DateTimeZone.UTC);
@@ -498,6 +507,26 @@ public final class ${minor.class}Vector extends BaseDataValueVector implements F
return date;
}
+ <#elseif minor.class == "TimeStampMicro">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ // value is truncated when converting microseconds to milliseconds in order to use DateTime type
+ long micros = java.util.concurrent.TimeUnit.MICROSECONDS.toMillis(get(index));
+ org.joda.time.DateTime date = new org.joda.time.DateTime(micros, org.joda.time.DateTimeZone.UTC);
+ date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+ return date;
+ }
+
+ <#elseif minor.class == "TimeStampNano">
+ @Override
+ public ${friendlyType} getObject(int index) {
+ // value is truncated when converting nanoseconds to milliseconds in order to use DateTime type
+ long millis = java.util.concurrent.TimeUnit.NANOSECONDS.toMillis(get(index));
+ org.joda.time.DateTime date = new org.joda.time.DateTime(millis, org.joda.time.DateTimeZone.UTC);
+ date = date.withZoneRetainFields(org.joda.time.DateTimeZone.getDefault());
+ return date;
+ }
+
<#elseif minor.class == "IntervalYear">
@Override
public ${friendlyType} getObject(int index) {
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/codegen/templates/NullableValueVectors.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/codegen/templates/NullableValueVectors.java b/java/vector/src/main/codegen/templates/NullableValueVectors.java
index 6a9ce65..ce63710 100644
--- a/java/vector/src/main/codegen/templates/NullableValueVectors.java
+++ b/java/vector/src/main/codegen/templates/NullableValueVectors.java
@@ -102,8 +102,14 @@ public final class ${className} extends BaseDataValueVector implements <#if type
field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.SINGLE), null);
<#elseif minor.class == "Float8">
field = new Field(name, true, new FloatingPoint(org.apache.arrow.vector.types.FloatingPointPrecision.DOUBLE), null);
- <#elseif minor.class == "TimeStamp">
+ <#elseif minor.class == "TimeStampSec">
+ field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND), null);
+ <#elseif minor.class == "TimeStampMilli">
field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND), null);
+ <#elseif minor.class == "TimeStampMicro">
+ field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND), null);
+ <#elseif minor.class == "TimeStampNano">
+ field = new Field(name, true, new org.apache.arrow.vector.types.pojo.ArrowType.Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND), null);
<#elseif minor.class == "IntervalDay">
field = new Field(name, true, new Interval(org.apache.arrow.vector.types.IntervalUnit.DAY_TIME), null);
<#elseif minor.class == "IntervalYear">
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
index 152867c..71fe88e 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileReader.java
@@ -37,7 +37,10 @@ import org.apache.arrow.vector.Float4Vector;
import org.apache.arrow.vector.Float8Vector;
import org.apache.arrow.vector.IntVector;
import org.apache.arrow.vector.SmallIntVector;
-import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
import org.apache.arrow.vector.TinyIntVector;
import org.apache.arrow.vector.UInt1Vector;
import org.apache.arrow.vector.UInt2Vector;
@@ -199,9 +202,18 @@ public class JsonFileReader implements AutoCloseable {
case VARCHAR:
((VarCharVector)valueVector).getMutator().setSafe(i, parser.readValueAs(String.class).getBytes(UTF_8));
break;
- case TIMESTAMP:
- ((TimeStampVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+ case TIMESTAMPSEC:
+ ((TimeStampSecVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
break;
+ case TIMESTAMPMILLI:
+ ((TimeStampMilliVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+ break;
+ case TIMESTAMPMICRO:
+ ((TimeStampMicroVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+ break;
+ case TIMESTAMPNANO:
+ ((TimeStampNanoVector)valueVector).getMutator().set(i, parser.readValueAs(Long.class));
+ break;
default:
throw new UnsupportedOperationException("minor type: " + valueVector.getMinorType());
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
index 6ff3577..ddc8043 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/file/json/JsonFileWriter.java
@@ -24,7 +24,10 @@ import java.util.List;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.BufferBacked;
import org.apache.arrow.vector.FieldVector;
-import org.apache.arrow.vector.TimeStampVector;
+import org.apache.arrow.vector.TimeStampSecVector;
+import org.apache.arrow.vector.TimeStampMilliVector;
+import org.apache.arrow.vector.TimeStampMicroVector;
+import org.apache.arrow.vector.TimeStampNanoVector;
import org.apache.arrow.vector.ValueVector;
import org.apache.arrow.vector.ValueVector.Accessor;
import org.apache.arrow.vector.VectorSchemaRoot;
@@ -139,8 +142,17 @@ public class JsonFileWriter implements AutoCloseable {
private void writeValueToGenerator(ValueVector valueVector, int i) throws IOException {
switch (valueVector.getMinorType()) {
- case TIMESTAMP:
- generator.writeNumber(((TimeStampVector)valueVector).getAccessor().get(i));
+ case TIMESTAMPSEC:
+ generator.writeNumber(((TimeStampSecVector)valueVector).getAccessor().get(i));
+ break;
+ case TIMESTAMPMILLI:
+ generator.writeNumber(((TimeStampMilliVector)valueVector).getAccessor().get(i));
+ break;
+ case TIMESTAMPMICRO:
+ generator.writeNumber(((TimeStampMicroVector)valueVector).getAccessor().get(i));
+ break;
+ case TIMESTAMPNANO:
+ generator.writeNumber(((TimeStampNanoVector)valueVector).getAccessor().get(i));
break;
case BIT:
generator.writeNumber(((BitVector)valueVector).getAccessor().get(i));
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
----------------------------------------------------------------------
diff --git a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
index 2a2fb74..ab539d5 100644
--- a/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
+++ b/java/vector/src/main/java/org/apache/arrow/vector/types/Types.java
@@ -33,7 +33,10 @@ import org.apache.arrow.vector.NullableIntVector;
import org.apache.arrow.vector.NullableIntervalDayVector;
import org.apache.arrow.vector.NullableIntervalYearVector;
import org.apache.arrow.vector.NullableSmallIntVector;
-import org.apache.arrow.vector.NullableTimeStampVector;
+import org.apache.arrow.vector.NullableTimeStampSecVector;
+import org.apache.arrow.vector.NullableTimeStampMilliVector;
+import org.apache.arrow.vector.NullableTimeStampMicroVector;
+import org.apache.arrow.vector.NullableTimeStampNanoVector;
import org.apache.arrow.vector.NullableTimeVector;
import org.apache.arrow.vector.NullableTinyIntVector;
import org.apache.arrow.vector.NullableUInt1Vector;
@@ -58,7 +61,10 @@ import org.apache.arrow.vector.complex.impl.IntervalDayWriterImpl;
import org.apache.arrow.vector.complex.impl.IntervalYearWriterImpl;
import org.apache.arrow.vector.complex.impl.NullableMapWriter;
import org.apache.arrow.vector.complex.impl.SmallIntWriterImpl;
-import org.apache.arrow.vector.complex.impl.TimeStampWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampSecWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMilliWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampMicroWriterImpl;
+import org.apache.arrow.vector.complex.impl.TimeStampNanoWriterImpl;
import org.apache.arrow.vector.complex.impl.TimeWriterImpl;
import org.apache.arrow.vector.complex.impl.TinyIntWriterImpl;
import org.apache.arrow.vector.complex.impl.UInt1WriterImpl;
@@ -102,7 +108,10 @@ public class Types {
private static final Field UINT8_FIELD = new Field("", true, new Int(64, false), null);
private static final Field DATE_FIELD = new Field("", true, Date.INSTANCE, null);
private static final Field TIME_FIELD = new Field("", true, Time.INSTANCE, null);
- private static final Field TIMESTAMP_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
+ private static final Field TIMESTAMPSEC_FIELD = new Field("", true, new Timestamp(TimeUnit.SECOND), null);
+ private static final Field TIMESTAMPMILLI_FIELD = new Field("", true, new Timestamp(TimeUnit.MILLISECOND), null);
+ private static final Field TIMESTAMPMICRO_FIELD = new Field("", true, new Timestamp(TimeUnit.MICROSECOND), null);
+ private static final Field TIMESTAMPNANO_FIELD = new Field("", true, new Timestamp(TimeUnit.NANOSECOND), null);
private static final Field INTERVALDAY_FIELD = new Field("", true, new Interval(IntervalUnit.DAY_TIME), null);
private static final Field INTERVALYEAR_FIELD = new Field("", true, new Interval(IntervalUnit.YEAR_MONTH), null);
private static final Field FLOAT4_FIELD = new Field("", true, new FloatingPoint(FloatingPointPrecision.SINGLE), null);
@@ -241,21 +250,72 @@ public class Types {
return new TimeWriterImpl((NullableTimeVector) vector);
}
},
+ // time in second from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
+ TIMESTAMPSEC(new Timestamp(org.apache.arrow.vector.types.TimeUnit.SECOND)) {
+ @Override
+ public Field getField() {
+ return TIMESTAMPSEC_FIELD;
+ }
+
+ @Override
+ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableTimeStampSecVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new TimeStampSecWriterImpl((NullableTimeStampSecVector) vector);
+ }
+ },
// time in millis from the Unix epoch, 00:00:00.000 on 1 January 1970, UTC.
- TIMESTAMP(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
+ TIMESTAMPMILLI(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MILLISECOND)) {
@Override
public Field getField() {
- return TIMESTAMP_FIELD;
+ return TIMESTAMPMILLI_FIELD;
}
@Override
public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
- return new NullableTimeStampVector(name, allocator);
+ return new NullableTimeStampMilliVector(name, allocator);
}
@Override
public FieldWriter getNewFieldWriter(ValueVector vector) {
- return new TimeStampWriterImpl((NullableTimeStampVector) vector);
+ return new TimeStampMilliWriterImpl((NullableTimeStampMilliVector) vector);
+ }
+ },
+ // time in microsecond from the Unix epoch, 00:00:00.000000 on 1 January 1970, UTC.
+ TIMESTAMPMICRO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.MICROSECOND)) {
+ @Override
+ public Field getField() {
+ return TIMESTAMPMICRO_FIELD;
+ }
+
+ @Override
+ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableTimeStampMicroVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new TimeStampMicroWriterImpl((NullableTimeStampMicroVector) vector);
+ }
+ },
+ // time in nanosecond from the Unix epoch, 00:00:00.000000000 on 1 January 1970, UTC.
+ TIMESTAMPNANO(new Timestamp(org.apache.arrow.vector.types.TimeUnit.NANOSECOND)) {
+ @Override
+ public Field getField() {
+ return TIMESTAMPNANO_FIELD;
+ }
+
+ @Override
+ public FieldVector getNewVector(String name, BufferAllocator allocator, CallBack callBack, int... precisionScale) {
+ return new NullableTimeStampNanoVector(name, allocator);
+ }
+
+ @Override
+ public FieldWriter getNewFieldWriter(ValueVector vector) {
+ return new TimeStampNanoWriterImpl((NullableTimeStampNanoVector) vector);
}
},
INTERVALDAY(new Interval(IntervalUnit.DAY_TIME)) {
@@ -579,10 +639,18 @@ public class Types {
}
@Override public MinorType visit(Timestamp type) {
- if (type.getUnit() != TimeUnit.MILLISECOND) {
- throw new UnsupportedOperationException("Only milliseconds supported: " + type);
+ switch (type.getUnit()) {
+ case SECOND:
+ return MinorType.TIMESTAMPSEC;
+ case MILLISECOND:
+ return MinorType.TIMESTAMPMILLI;
+ case MICROSECOND:
+ return MinorType.TIMESTAMPMICRO;
+ case NANOSECOND:
+ return MinorType.TIMESTAMPNANO;
+ default:
+ throw new IllegalArgumentException("unknown unit: " + type);
}
- return MinorType.TIMESTAMP;
}
@Override
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
index 2c0c853..7a2d416 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/complex/writer/TestComplexWriter.java
@@ -43,12 +43,15 @@ import org.apache.arrow.vector.complex.reader.FieldReader;
import org.apache.arrow.vector.complex.writer.BaseWriter.ComplexWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
+import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.ArrowType.ArrowTypeID;
import org.apache.arrow.vector.types.pojo.ArrowType.Int;
import org.apache.arrow.vector.types.pojo.ArrowType.Union;
import org.apache.arrow.vector.types.pojo.ArrowType.Utf8;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.util.Text;
+import org.joda.time.DateTime;
+import org.joda.time.DateTimeZone;
import org.junit.Assert;
import org.junit.Test;
@@ -561,4 +564,86 @@ public class TestComplexWriter {
Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$"));
Assert.assertTrue(fieldNamesCaseSensitive.contains("list_field::$data$::bit_field"));
}
+
+ @Test
+ public void timeStampWriters() throws Exception {
+ // test values
+ final long expectedNanos = 981173106123456789L;
+ final long expectedMicros = 981173106123456L;
+ final long expectedMillis = 981173106123L;
+ final long expectedSecs = 981173106L;
+ final DateTime expectedSecDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 0).withZoneRetainFields(DateTimeZone.getDefault());
+ final DateTime expectedMilliDateTime = new DateTime(2001, 2, 3, 4, 5, 6, 123).withZoneRetainFields(DateTimeZone.getDefault());
+ final DateTime expectedMicroDateTime = expectedMilliDateTime;
+ final DateTime expectedNanoDateTime = expectedMilliDateTime;
+
+ // write
+ MapVector parent = new MapVector("parent", allocator, null);
+ ComplexWriter writer = new ComplexWriterImpl("root", parent);
+ MapWriter rootWriter = writer.rootAsMap();
+
+ TimeStampSecWriter timeStampSecWriter = rootWriter.timeStampSec("sec");
+ timeStampSecWriter.setPosition(0);
+ timeStampSecWriter.writeTimeStampSec(expectedSecs);
+
+ TimeStampMilliWriter timeStampWriter = rootWriter.timeStampMilli("milli");
+ timeStampWriter.setPosition(1);
+ timeStampWriter.writeTimeStampMilli(expectedMillis);
+
+ TimeStampMicroWriter timeStampMicroWriter = rootWriter.timeStampMicro("micro");
+ timeStampMicroWriter.setPosition(2);
+ timeStampMicroWriter.writeTimeStampMicro(expectedMicros);
+
+ TimeStampNanoWriter timeStampNanoWriter = rootWriter.timeStampNano("nano");
+ timeStampNanoWriter.setPosition(3);
+ timeStampNanoWriter.writeTimeStampNano(expectedNanos);
+
+ // schema
+ Field secField = parent.getField().getChildren().get(0).getChildren().get(0);
+ Assert.assertEquals("sec", secField.getName());
+ Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, secField.getType().getTypeID());
+
+ Field milliField = parent.getField().getChildren().get(0).getChildren().get(1);
+ Assert.assertEquals("milli", milliField.getName());
+ Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, milliField.getType().getTypeID());
+
+ Field microField = parent.getField().getChildren().get(0).getChildren().get(2);
+ Assert.assertEquals("micro", microField.getName());
+ Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, microField.getType().getTypeID());
+
+ Field nanoField = parent.getField().getChildren().get(0).getChildren().get(3);
+ Assert.assertEquals("nano", nanoField.getName());
+ Assert.assertEquals(ArrowType.Timestamp.TYPE_TYPE, nanoField.getType().getTypeID());
+
+ // read
+ MapReader rootReader = new SingleMapReaderImpl(parent).reader("root");
+
+ FieldReader secReader = rootReader.reader("sec");
+ secReader.setPosition(0);
+ DateTime secDateTime = secReader.readDateTime();
+ Assert.assertEquals(expectedSecDateTime, secDateTime);
+ long secLong = secReader.readLong();
+ Assert.assertEquals(expectedSecs, secLong);
+
+ FieldReader milliReader = rootReader.reader("milli");
+ milliReader.setPosition(1);
+ DateTime milliDateTime = milliReader.readDateTime();
+ Assert.assertEquals(expectedMilliDateTime, milliDateTime);
+ long milliLong = milliReader.readLong();
+ Assert.assertEquals(expectedMillis, milliLong);
+
+ FieldReader microReader = rootReader.reader("micro");
+ microReader.setPosition(2);
+ DateTime microDateTime = microReader.readDateTime();
+ Assert.assertEquals(expectedMicroDateTime, microDateTime);
+ long microLong = microReader.readLong();
+ Assert.assertEquals(expectedMicros, microLong);
+
+ FieldReader nanoReader = rootReader.reader("nano");
+ nanoReader.setPosition(3);
+ DateTime nanoDateTime = nanoReader.readDateTime();
+ Assert.assertEquals(expectedNanoDateTime, nanoDateTime);
+ long nanoLong = nanoReader.readLong();
+ Assert.assertEquals(expectedNanos, nanoLong);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
index 6e577b5..774bead 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/file/BaseFileTest.java
@@ -33,7 +33,7 @@ import org.apache.arrow.vector.complex.writer.BaseWriter.ListWriter;
import org.apache.arrow.vector.complex.writer.BaseWriter.MapWriter;
import org.apache.arrow.vector.complex.writer.BigIntWriter;
import org.apache.arrow.vector.complex.writer.IntWriter;
-import org.apache.arrow.vector.holders.NullableTimeStampHolder;
+import org.apache.arrow.vector.holders.NullableTimeStampMilliHolder;
import org.joda.time.DateTimeZone;
import org.junit.After;
import org.junit.Assert;
@@ -100,7 +100,7 @@ public class BaseFileTest {
listWriter.endList();
mapWriter.setPosition(i);
mapWriter.start();
- mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+ mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
mapWriter.end();
}
writer.setValueCount(count);
@@ -130,7 +130,7 @@ public class BaseFileTest {
}
Assert.assertEquals(Long.valueOf(i), root.getVector("bigInt").getAccessor().getObject(i));
Assert.assertEquals(i % 3, ((List<?>)root.getVector("list").getAccessor().getObject(i)).size());
- NullableTimeStampHolder h = new NullableTimeStampHolder();
+ NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
FieldReader mapReader = root.getVector("map").getReader();
mapReader.setPosition(i);
mapReader.reader("timestamp").read(h);
@@ -167,7 +167,7 @@ public class BaseFileTest {
Assert.assertEquals(i % 3, unionReader.size());
break;
case 3:
- NullableTimeStampHolder h = new NullableTimeStampHolder();
+ NullableTimeStampMilliHolder h = new NullableTimeStampMilliHolder();
unionReader.reader("timestamp").read(h);
Assert.assertEquals(i, h.value);
break;
@@ -209,7 +209,7 @@ public class BaseFileTest {
case 3:
mapWriter.setPosition(i);
mapWriter.start();
- mapWriter.timeStamp("timestamp").writeTimeStamp(i);
+ mapWriter.timeStampMilli("timestamp").writeTimeStampMilli(i);
mapWriter.end();
break;
}
http://git-wip-us.apache.org/repos/asf/arrow/blob/08f38d97/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
----------------------------------------------------------------------
diff --git a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
index 5a238bc..65823e2 100644
--- a/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
+++ b/java/vector/src/test/java/org/apache/arrow/vector/pojo/TestConvert.java
@@ -80,7 +80,7 @@ public class TestConvert {
childrenBuilder.add(new Field("child4", true, new List(), ImmutableList.<Field>of(
new Field("child4.1", true, Utf8.INSTANCE, null)
)));
- childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMP.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
+ childrenBuilder.add(new Field("child5", true, new Union(UnionMode.Sparse, new int[] { MinorType.TIMESTAMPMILLI.ordinal(), MinorType.FLOAT8.ordinal() } ), ImmutableList.<Field>of(
new Field("child5.1", true, new Timestamp(TimeUnit.MILLISECOND), null),
new Field("child5.2", true, new FloatingPoint(DOUBLE), ImmutableList.<Field>of())
)));