You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/12/12 21:09:55 UTC
[iceberg] branch master updated: Hive: Support timestamp with local
zone in Hive3 (#1897)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 4c0bf8a Hive: Support timestamp with local zone in Hive3 (#1897)
4c0bf8a is described below
commit 4c0bf8aec7b04f8b969fb34685775492c0a04fcc
Author: Marton Bod <ma...@gmail.com>
AuthorDate: Sat Dec 12 22:09:45 2020 +0100
Hive: Support timestamp with local zone in Hive3 (#1897)
---
build.gradle | 1 +
.../apache/iceberg/hive/HiveSchemaConverter.java | 4 ++
.../org/apache/iceberg/hive/HiveSchemaUtil.java | 4 ++
.../IcebergTimestampObjectInspectorHive3.java | 39 ++++------
...ebergTimestampWithZoneObjectInspectorHive3.java | 82 ++++++++++++++++++++++
.../iceberg/mr/hive/TestHiveSchemaUtilHive3.java | 50 +++++++++++++
.../TestIcebergTimestampObjectInspectorHive3.java | 42 ++---------
...ebergTimestampWithZoneObjectInspectorHive3.java | 76 ++++++++++++++++++++
.../objectinspector/IcebergObjectInspector.java | 12 ++--
.../IcebergTimestampObjectInspector.java | 51 +++-----------
...> IcebergTimestampWithZoneObjectInspector.java} | 54 ++++----------
.../TestIcebergObjectInspector.java | 12 ++--
.../TestIcebergTimestampObjectInspector.java | 10 +--
...stIcebergTimestampWithZoneObjectInspector.java} | 21 +++---
14 files changed, 294 insertions(+), 164 deletions(-)
diff --git a/build.gradle b/build.gradle
index 98e2e10..32475a3 100644
--- a/build.gradle
+++ b/build.gradle
@@ -563,6 +563,7 @@ if (jdkVersion == '8') {
test {
exclude '**/TestIcebergDateObjectInspector.class'
exclude '**/TestIcebergTimestampObjectInspector.class'
+ exclude '**/TestIcebergTimestampWithZoneObjectInspector.class'
}
dependencies {
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
index 88068c2..fec8c9e 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
@@ -99,6 +99,10 @@ class HiveSchemaConverter {
case INTERVAL_YEAR_MONTH:
case INTERVAL_DAY_TIME:
default:
+ // special case for Timestamp with Local TZ which is only available in Hive3
+ if ("TIMESTAMPLOCALTZ".equalsIgnoreCase(((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory().name())) {
+ return Types.TimestampType.withZone();
+ }
throw new IllegalArgumentException("Unsupported Hive type (" +
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory() +
") for Iceberg tables.");
diff --git a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
index 8f73df3..f98c4ec 100644
--- a/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
+++ b/hive-metastore/src/main/java/org/apache/iceberg/hive/HiveSchemaUtil.java
@@ -108,6 +108,10 @@ public final class HiveSchemaUtil {
case TIME:
return "string";
case TIMESTAMP:
+ Types.TimestampType timestampType = (Types.TimestampType) type;
+ if (MetastoreUtil.hive3PresentOnClasspath() && timestampType.shouldAdjustToUTC()) {
+ return "timestamp with local time zone";
+ }
return "timestamp";
case STRING:
case UUID:
diff --git a/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java b/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java
index cf57b2c..652cf13 100644
--- a/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java
+++ b/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspectorHive3.java
@@ -20,7 +20,6 @@
package org.apache.iceberg.mr.hive.serde.objectinspector;
import java.time.LocalDateTime;
-import java.time.OffsetDateTime;
import java.time.ZoneOffset;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
@@ -28,42 +27,34 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitive
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-public abstract class IcebergTimestampObjectInspectorHive3 extends AbstractPrimitiveJavaObjectInspector
- implements TimestampObjectInspector {
+public class IcebergTimestampObjectInspectorHive3 extends AbstractPrimitiveJavaObjectInspector
+ implements TimestampObjectInspector, WriteObjectInspector {
- private static final IcebergTimestampObjectInspectorHive3 INSTANCE_WITH_ZONE =
- new IcebergTimestampObjectInspectorHive3() {
- @Override
- LocalDateTime toLocalDateTime(Object o) {
- return ((OffsetDateTime) o).toLocalDateTime();
- }
- };
+ private static final IcebergTimestampObjectInspectorHive3 INSTANCE = new IcebergTimestampObjectInspectorHive3();
- private static final IcebergTimestampObjectInspectorHive3 INSTANCE_WITHOUT_ZONE =
- new IcebergTimestampObjectInspectorHive3() {
- @Override
- LocalDateTime toLocalDateTime(Object o) {
- return (LocalDateTime) o;
- }
- };
-
- public static IcebergTimestampObjectInspectorHive3 get(boolean adjustToUTC) {
- return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE;
+ public static IcebergTimestampObjectInspectorHive3 get() {
+ return INSTANCE;
}
private IcebergTimestampObjectInspectorHive3() {
super(TypeInfoFactory.timestampTypeInfo);
}
-
- abstract LocalDateTime toLocalDateTime(Object object);
+ @Override
+ public LocalDateTime convert(Object o) {
+ if (o == null) {
+ return null;
+ }
+ Timestamp timestamp = ((TimestampWritableV2) o).getTimestamp();
+ return LocalDateTime.ofEpochSecond(timestamp.toEpochSecond(), timestamp.getNanos(), ZoneOffset.UTC);
+ }
@Override
public Timestamp getPrimitiveJavaObject(Object o) {
if (o == null) {
return null;
}
- LocalDateTime time = toLocalDateTime(o);
+ LocalDateTime time = (LocalDateTime) o;
Timestamp timestamp = Timestamp.ofEpochMilli(time.toInstant(ZoneOffset.UTC).toEpochMilli());
timestamp.setNanos(time.getNano());
return timestamp;
@@ -86,8 +77,6 @@ public abstract class IcebergTimestampObjectInspectorHive3 extends AbstractPrimi
Timestamp copy = new Timestamp(ts);
copy.setNanos(ts.getNanos());
return copy;
- } else if (o instanceof OffsetDateTime) {
- return OffsetDateTime.of(((OffsetDateTime) o).toLocalDateTime(), ((OffsetDateTime) o).getOffset());
} else if (o instanceof LocalDateTime) {
return LocalDateTime.of(((LocalDateTime) o).toLocalDate(), ((LocalDateTime) o).toLocalTime());
} else {
diff --git a/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java b/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java
new file mode 100644
index 0000000..417864d
--- /dev/null
+++ b/hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspectorHive3.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
+import java.time.ZonedDateTime;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampLocalTZObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+
+public class IcebergTimestampWithZoneObjectInspectorHive3 extends AbstractPrimitiveJavaObjectInspector
+ implements TimestampLocalTZObjectInspector, WriteObjectInspector {
+
+ private static final IcebergTimestampWithZoneObjectInspectorHive3 INSTANCE =
+ new IcebergTimestampWithZoneObjectInspectorHive3();
+
+ public static IcebergTimestampWithZoneObjectInspectorHive3 get() {
+ return INSTANCE;
+ }
+
+ private IcebergTimestampWithZoneObjectInspectorHive3() {
+ super(TypeInfoFactory.timestampLocalTZTypeInfo);
+ }
+
+ @Override
+ public OffsetDateTime convert(Object o) {
+ if (o == null) {
+ return null;
+ }
+ ZonedDateTime zdt = ((TimestampLocalTZWritable) o).getTimestampTZ().getZonedDateTime();
+ return OffsetDateTime.of(zdt.toLocalDateTime(), zdt.getOffset());
+ }
+
+ @Override
+ public TimestampTZ getPrimitiveJavaObject(Object o) {
+ if (o == null) {
+ return null;
+ }
+ OffsetDateTime odt = (OffsetDateTime) o;
+ ZonedDateTime zdt = odt.atZoneSameInstant(ZoneOffset.UTC);
+ return new TimestampTZ(zdt);
+ }
+
+ @Override
+ public TimestampLocalTZWritable getPrimitiveWritableObject(Object o) {
+ TimestampTZ tsTz = getPrimitiveJavaObject(o);
+ return tsTz == null ? null : new TimestampLocalTZWritable(tsTz);
+ }
+
+ @Override
+ public Object copyObject(Object o) {
+ if (o instanceof TimestampTZ) {
+ TimestampTZ ts = (TimestampTZ) o;
+ return new TimestampTZ(ts.getZonedDateTime());
+ } else if (o instanceof OffsetDateTime) {
+ OffsetDateTime odt = (OffsetDateTime) o;
+ return OffsetDateTime.of(odt.toLocalDateTime(), odt.getOffset());
+ } else {
+ return o;
+ }
+ }
+}
diff --git a/hive3/src/test/java/org/apache/iceberg/mr/hive/TestHiveSchemaUtilHive3.java b/hive3/src/test/java/org/apache/iceberg/mr/hive/TestHiveSchemaUtilHive3.java
new file mode 100644
index 0000000..63eb736
--- /dev/null
+++ b/hive3/src/test/java/org/apache/iceberg/mr/hive/TestHiveSchemaUtilHive3.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive;
+
+import java.util.List;
+import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.serde.serdeConstants;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.hive.TestHiveSchemaUtil;
+import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
+
+import static org.apache.iceberg.types.Types.NestedField.optional;
+
+public class TestHiveSchemaUtilHive3 extends TestHiveSchemaUtil {
+
+ @Override
+ protected List<FieldSchema> getSupportedFieldSchemas() {
+ List<FieldSchema> fields = Lists.newArrayList(super.getSupportedFieldSchemas());
+ // timestamp local tz only present in Hive3
+ fields.add(new FieldSchema("c_timestamptz", serdeConstants.TIMESTAMPLOCALTZ_TYPE_NAME, ""));
+ return fields;
+ }
+
+ @Override
+ protected Schema getSchemaWithSupportedTypes() {
+ Schema schema = super.getSchemaWithSupportedTypes();
+ List<Types.NestedField> columns = Lists.newArrayList(schema.columns());
+ // timestamp local tz only present in Hive3
+ columns.add(optional(columns.size(), "c_timestamptz", Types.TimestampType.withZone()));
+ return new Schema(columns);
+ }
+}
diff --git a/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspectorHive3.java b/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspectorHive3.java
index e885689..e22f71f 100644
--- a/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspectorHive3.java
+++ b/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspectorHive3.java
@@ -21,14 +21,11 @@ package org.apache.iceberg.mr.hive.serde.objectinspector;
import java.time.Instant;
import java.time.LocalDateTime;
-import java.time.OffsetDateTime;
import java.time.ZoneId;
-import java.time.ZoneOffset;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.serde2.io.TimestampWritableV2;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.junit.Assert;
import org.junit.Test;
@@ -37,7 +34,7 @@ public class TestIcebergTimestampObjectInspectorHive3 {
@Test
public void testIcebergTimestampObjectInspector() {
- TimestampObjectInspector oi = IcebergTimestampObjectInspectorHive3.get(false);
+ IcebergTimestampObjectInspectorHive3 oi = IcebergTimestampObjectInspectorHive3.get();
Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, oi.getPrimitiveCategory());
@@ -51,10 +48,12 @@ public class TestIcebergTimestampObjectInspectorHive3 {
Assert.assertNull(oi.copyObject(null));
Assert.assertNull(oi.getPrimitiveJavaObject(null));
Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
long epochMilli = 1601471970000L;
- LocalDateTime local = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneId.of("UTC"));
+ LocalDateTime local = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneId.of("UTC")).plusNanos(34000);
Timestamp ts = Timestamp.ofEpochMilli(epochMilli);
+ ts.setNanos(34000);
Assert.assertEquals(ts, oi.getPrimitiveJavaObject(local));
Assert.assertEquals(new TimestampWritableV2(ts), oi.getPrimitiveWritableObject(local));
@@ -65,39 +64,8 @@ public class TestIcebergTimestampObjectInspectorHive3 {
Assert.assertNotSame(ts, copy);
Assert.assertFalse(oi.preferWritable());
- }
-
- @Test
- public void testIcebergTimestampObjectInspectorWithUTCAdjustment() {
- TimestampObjectInspector oi = IcebergTimestampObjectInspectorHive3.get(true);
-
- Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
- Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, oi.getPrimitiveCategory());
-
- Assert.assertEquals(TypeInfoFactory.timestampTypeInfo, oi.getTypeInfo());
- Assert.assertEquals(TypeInfoFactory.timestampTypeInfo.getTypeName(), oi.getTypeName());
-
- Assert.assertEquals(Timestamp.class, oi.getJavaPrimitiveClass());
- Assert.assertEquals(TimestampWritableV2.class, oi.getPrimitiveWritableClass());
-
- Assert.assertNull(oi.copyObject(null));
- Assert.assertNull(oi.getPrimitiveJavaObject(null));
- Assert.assertNull(oi.getPrimitiveWritableObject(null));
-
- long epochMilli = 1601471970000L;
- LocalDateTime local = LocalDateTime.ofInstant(Instant.ofEpochMilli(epochMilli), ZoneId.of("UTC"));
- OffsetDateTime offsetDateTime = OffsetDateTime.of(local, ZoneOffset.ofHours(4));
- Timestamp ts = Timestamp.ofEpochMilli(epochMilli);
-
- Assert.assertEquals(ts, oi.getPrimitiveJavaObject(offsetDateTime));
- Assert.assertEquals(new TimestampWritableV2(ts), oi.getPrimitiveWritableObject(offsetDateTime));
- Timestamp copy = (Timestamp) oi.copyObject(ts);
-
- Assert.assertEquals(ts, copy);
- Assert.assertNotSame(ts, copy);
-
- Assert.assertFalse(oi.preferWritable());
+ Assert.assertEquals(local, oi.convert(new TimestampWritableV2(ts)));
}
}
diff --git a/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampWithZoneObjectInspectorHive3.java b/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampWithZoneObjectInspectorHive3.java
new file mode 100644
index 0000000..9aacdb9
--- /dev/null
+++ b/hive3/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampWithZoneObjectInspectorHive3.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import org.apache.hadoop.hive.common.type.TimestampTZ;
+import org.apache.hadoop.hive.serde2.io.TimestampLocalTZWritable;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestIcebergTimestampWithZoneObjectInspectorHive3 {
+
+ @Test
+ public void testIcebergTimestampLocalTZObjectInspector() {
+ IcebergTimestampWithZoneObjectInspectorHive3 oi = IcebergTimestampWithZoneObjectInspectorHive3.get();
+
+ Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
+ Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMPLOCALTZ, oi.getPrimitiveCategory());
+
+ Assert.assertEquals(TypeInfoFactory.timestampLocalTZTypeInfo, oi.getTypeInfo());
+ Assert.assertEquals(TypeInfoFactory.timestampLocalTZTypeInfo.getTypeName(), oi.getTypeName());
+
+ Assert.assertEquals(TimestampTZ.class, oi.getJavaPrimitiveClass());
+ Assert.assertEquals(TimestampLocalTZWritable.class, oi.getPrimitiveWritableClass());
+
+ Assert.assertNull(oi.copyObject(null));
+ Assert.assertNull(oi.getPrimitiveJavaObject(null));
+ Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
+
+ LocalDateTime dateTimeAtUTC = LocalDateTime.of(2020, 12, 10, 15, 55, 20, 30000);
+ OffsetDateTime offsetDateTime = OffsetDateTime.of(dateTimeAtUTC.plusHours(4), ZoneOffset.ofHours(4));
+ TimestampTZ ts = new TimestampTZ(dateTimeAtUTC.atZone(ZoneId.of("UTC")));
+
+ Assert.assertEquals(ts, oi.getPrimitiveJavaObject(offsetDateTime));
+ Assert.assertEquals(new TimestampLocalTZWritable(ts), oi.getPrimitiveWritableObject(offsetDateTime));
+
+ // try with another offset as well
+ offsetDateTime = OffsetDateTime.of(dateTimeAtUTC.plusHours(11), ZoneOffset.ofHours(11));
+ Assert.assertEquals(ts, oi.getPrimitiveJavaObject(offsetDateTime));
+ Assert.assertEquals(new TimestampLocalTZWritable(ts), oi.getPrimitiveWritableObject(offsetDateTime));
+
+ TimestampTZ copy = (TimestampTZ) oi.copyObject(ts);
+
+ Assert.assertEquals(ts, copy);
+ Assert.assertNotSame(ts, copy);
+
+ Assert.assertFalse(oi.preferWritable());
+
+ Assert.assertEquals(OffsetDateTime.of(dateTimeAtUTC, ZoneOffset.UTC), oi.convert(new TimestampLocalTZWritable(ts)));
+ }
+
+}
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
index f7a0382..9175f3f 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
@@ -50,15 +50,19 @@ public final class IcebergObjectInspector extends TypeUtil.SchemaVisitor<ObjectI
"org.apache.iceberg.mr.hive.serde.objectinspector.IcebergTimestampObjectInspectorHive3" :
"org.apache.iceberg.mr.hive.serde.objectinspector.IcebergTimestampObjectInspector";
+ private static final String TIMESTAMPTZ_INSPECTOR_CLASS = MetastoreUtil.hive3PresentOnClasspath() ?
+ "org.apache.iceberg.mr.hive.serde.objectinspector.IcebergTimestampWithZoneObjectInspectorHive3" :
+ "org.apache.iceberg.mr.hive.serde.objectinspector.IcebergTimestampWithZoneObjectInspector";
+
public static final ObjectInspector TIMESTAMP_INSPECTOR = DynMethods.builder("get")
- .impl(TIMESTAMP_INSPECTOR_CLASS, boolean.class)
+ .impl(TIMESTAMP_INSPECTOR_CLASS)
.buildStatic()
- .invoke(false);
+ .invoke();
public static final ObjectInspector TIMESTAMP_INSPECTOR_WITH_TZ = DynMethods.builder("get")
- .impl(TIMESTAMP_INSPECTOR_CLASS, boolean.class)
+ .impl(TIMESTAMPTZ_INSPECTOR_CLASS)
.buildStatic()
- .invoke(true);
+ .invoke();
public static ObjectInspector create(@Nullable Schema schema) {
if (schema == null) {
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java
index 1f40aa8..39b8226 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java
@@ -21,55 +21,32 @@ package org.apache.iceberg.mr.hive.serde.objectinspector;
import java.sql.Timestamp;
import java.time.LocalDateTime;
-import java.time.OffsetDateTime;
-import java.time.ZoneId;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector
+public class IcebergTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements TimestampObjectInspector, WriteObjectInspector {
- private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = new IcebergTimestampObjectInspector() {
- @Override
- LocalDateTime toLocalDateTime(Object o) {
- return ((OffsetDateTime) o).toLocalDateTime();
- }
-
- @Override
- public OffsetDateTime convert(Object o) {
- return o == null ? null :
- OffsetDateTime.ofInstant(((TimestampWritable) o).getTimestamp().toInstant(), ZoneId.of("UTC"));
- }
- };
-
- private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = new IcebergTimestampObjectInspector() {
- @Override
- LocalDateTime toLocalDateTime(Object o) {
- return (LocalDateTime) o;
- }
-
- @Override
- public LocalDateTime convert(Object o) {
- return o == null ? null : ((TimestampWritable) o).getTimestamp().toLocalDateTime();
- }
- };
+ private static final IcebergTimestampObjectInspector INSTANCE = new IcebergTimestampObjectInspector();
- public static IcebergTimestampObjectInspector get(boolean adjustToUTC) {
- return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE;
+ public static IcebergTimestampObjectInspector get() {
+ return INSTANCE;
}
private IcebergTimestampObjectInspector() {
super(TypeInfoFactory.timestampTypeInfo);
}
-
- abstract LocalDateTime toLocalDateTime(Object object);
+ @Override
+ public LocalDateTime convert(Object o) {
+ return o == null ? null : ((TimestampWritable) o).getTimestamp().toLocalDateTime();
+ }
@Override
public Timestamp getPrimitiveJavaObject(Object o) {
- return o == null ? null : Timestamp.valueOf(toLocalDateTime(o));
+ return o == null ? null : Timestamp.valueOf((LocalDateTime) o);
}
@Override
@@ -80,22 +57,16 @@ public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJ
@Override
public Object copyObject(Object o) {
- if (o == null) {
- return null;
- }
-
if (o instanceof Timestamp) {
Timestamp ts = (Timestamp) o;
Timestamp copy = new Timestamp(ts.getTime());
copy.setNanos(ts.getNanos());
return copy;
- } else if (o instanceof OffsetDateTime) {
- return OffsetDateTime.of(((OffsetDateTime) o).toLocalDateTime(), ((OffsetDateTime) o).getOffset());
} else if (o instanceof LocalDateTime) {
- return LocalDateTime.of(((LocalDateTime) o).toLocalDate(), ((LocalDateTime) o).toLocalTime());
+ LocalDateTime ldt = (LocalDateTime) o;
+ return LocalDateTime.of(ldt.toLocalDate(), ldt.toLocalTime());
} else {
return o;
}
}
-
}
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspector.java
similarity index 54%
copy from mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java
copy to mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspector.java
index 1f40aa8..c9d23ac 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampObjectInspector.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimestampWithZoneObjectInspector.java
@@ -20,56 +20,35 @@
package org.apache.iceberg.mr.hive.serde.objectinspector;
import java.sql.Timestamp;
-import java.time.LocalDateTime;
import java.time.OffsetDateTime;
-import java.time.ZoneId;
+import java.time.ZoneOffset;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJavaObjectInspector
+public class IcebergTimestampWithZoneObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements TimestampObjectInspector, WriteObjectInspector {
- private static final IcebergTimestampObjectInspector INSTANCE_WITH_ZONE = new IcebergTimestampObjectInspector() {
- @Override
- LocalDateTime toLocalDateTime(Object o) {
- return ((OffsetDateTime) o).toLocalDateTime();
- }
-
- @Override
- public OffsetDateTime convert(Object o) {
- return o == null ? null :
- OffsetDateTime.ofInstant(((TimestampWritable) o).getTimestamp().toInstant(), ZoneId.of("UTC"));
- }
- };
-
- private static final IcebergTimestampObjectInspector INSTANCE_WITHOUT_ZONE = new IcebergTimestampObjectInspector() {
- @Override
- LocalDateTime toLocalDateTime(Object o) {
- return (LocalDateTime) o;
- }
-
- @Override
- public LocalDateTime convert(Object o) {
- return o == null ? null : ((TimestampWritable) o).getTimestamp().toLocalDateTime();
- }
- };
+ private static final IcebergTimestampWithZoneObjectInspector INSTANCE = new IcebergTimestampWithZoneObjectInspector();
- public static IcebergTimestampObjectInspector get(boolean adjustToUTC) {
- return adjustToUTC ? INSTANCE_WITH_ZONE : INSTANCE_WITHOUT_ZONE;
+ public static IcebergTimestampWithZoneObjectInspector get() {
+ return INSTANCE;
}
- private IcebergTimestampObjectInspector() {
+ private IcebergTimestampWithZoneObjectInspector() {
super(TypeInfoFactory.timestampTypeInfo);
}
-
- abstract LocalDateTime toLocalDateTime(Object object);
+ @Override
+ public OffsetDateTime convert(Object o) {
+ return o == null ? null :
+ OffsetDateTime.of(((TimestampWritable) o).getTimestamp().toLocalDateTime(), ZoneOffset.UTC);
+ }
@Override
public Timestamp getPrimitiveJavaObject(Object o) {
- return o == null ? null : Timestamp.valueOf(toLocalDateTime(o));
+ return o == null ? null : Timestamp.valueOf(((OffsetDateTime) o).toLocalDateTime());
}
@Override
@@ -80,19 +59,14 @@ public abstract class IcebergTimestampObjectInspector extends AbstractPrimitiveJ
@Override
public Object copyObject(Object o) {
- if (o == null) {
- return null;
- }
-
if (o instanceof Timestamp) {
Timestamp ts = (Timestamp) o;
Timestamp copy = new Timestamp(ts.getTime());
copy.setNanos(ts.getNanos());
return copy;
} else if (o instanceof OffsetDateTime) {
- return OffsetDateTime.of(((OffsetDateTime) o).toLocalDateTime(), ((OffsetDateTime) o).getOffset());
- } else if (o instanceof LocalDateTime) {
- return LocalDateTime.of(((LocalDateTime) o).toLocalDate(), ((LocalDateTime) o).toLocalTime());
+ OffsetDateTime odt = (OffsetDateTime) o;
+ return OffsetDateTime.of(odt.toLocalDateTime(), odt.getOffset());
} else {
return o;
}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
index 0c79b0b..f4178d7 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
@@ -154,10 +154,10 @@ public class TestIcebergObjectInspector {
Assert.assertEquals("timestamp_field", timestampField.getFieldName());
Assert.assertEquals("timestamp comment", timestampField.getFieldComment());
if (MetastoreUtil.hive3PresentOnClasspath()) {
- Assert.assertTrue(timestampField.getFieldObjectInspector().getClass().getName()
- .startsWith("org.apache.iceberg.mr.hive.serde.objectinspector.IcebergTimestampObjectInspectorHive3"));
+ Assert.assertEquals("IcebergTimestampObjectInspectorHive3",
+ timestampField.getFieldObjectInspector().getClass().getSimpleName());
} else {
- Assert.assertEquals(IcebergTimestampObjectInspector.get(false), timestampField.getFieldObjectInspector());
+ Assert.assertEquals(IcebergTimestampObjectInspector.get(), timestampField.getFieldObjectInspector());
}
// timestamp with tz
@@ -166,10 +166,10 @@ public class TestIcebergObjectInspector {
Assert.assertEquals("timestamptz_field", timestampTzField.getFieldName());
Assert.assertEquals("timestamptz comment", timestampTzField.getFieldComment());
if (MetastoreUtil.hive3PresentOnClasspath()) {
- Assert.assertTrue(timestampTzField.getFieldObjectInspector().getClass().getName()
- .startsWith("org.apache.iceberg.mr.hive.serde.objectinspector.IcebergTimestampObjectInspectorHive3"));
+ Assert.assertEquals("IcebergTimestampWithZoneObjectInspectorHive3",
+ timestampTzField.getFieldObjectInspector().getClass().getSimpleName());
} else {
- Assert.assertEquals(IcebergTimestampObjectInspector.get(true), timestampTzField.getFieldObjectInspector());
+ Assert.assertEquals(IcebergTimestampWithZoneObjectInspector.get(), timestampTzField.getFieldObjectInspector());
}
// UUID
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java
index 7c91e80..79629f0 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java
@@ -24,7 +24,6 @@ import java.time.LocalDateTime;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.junit.Assert;
import org.junit.Test;
@@ -33,7 +32,7 @@ public class TestIcebergTimestampObjectInspector {
@Test
public void testIcebergTimestampObjectInspector() {
- TimestampObjectInspector oi = IcebergTimestampObjectInspector.get(false);
+ IcebergTimestampObjectInspector oi = IcebergTimestampObjectInspector.get();
Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, oi.getPrimitiveCategory());
@@ -47,9 +46,10 @@ public class TestIcebergTimestampObjectInspector {
Assert.assertNull(oi.copyObject(null));
Assert.assertNull(oi.getPrimitiveJavaObject(null));
Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
- LocalDateTime local = LocalDateTime.of(2020, 1, 1, 0, 0);
- Timestamp ts = Timestamp.valueOf("2020-01-01 00:00:00");
+ LocalDateTime local = LocalDateTime.of(2020, 1, 1, 12, 55, 30, 5560000);
+ Timestamp ts = Timestamp.valueOf(local);
Assert.assertEquals(ts, oi.getPrimitiveJavaObject(local));
Assert.assertEquals(new TimestampWritable(ts), oi.getPrimitiveWritableObject(local));
@@ -60,6 +60,8 @@ public class TestIcebergTimestampObjectInspector {
Assert.assertNotSame(ts, copy);
Assert.assertFalse(oi.preferWritable());
+
+ Assert.assertEquals(local, oi.convert(new TimestampWritable(ts)));
}
}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampWithZoneObjectInspector.java
similarity index 73%
copy from mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java
copy to mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampWithZoneObjectInspector.java
index 7c91e80..af2fa40 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampObjectInspector.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimestampWithZoneObjectInspector.java
@@ -21,19 +21,20 @@ package org.apache.iceberg.mr.hive.serde.objectinspector;
import java.sql.Timestamp;
import java.time.LocalDateTime;
+import java.time.OffsetDateTime;
+import java.time.ZoneOffset;
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.primitive.TimestampObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.junit.Assert;
import org.junit.Test;
-public class TestIcebergTimestampObjectInspector {
+public class TestIcebergTimestampWithZoneObjectInspector {
@Test
- public void testIcebergTimestampObjectInspector() {
- TimestampObjectInspector oi = IcebergTimestampObjectInspector.get(false);
+ public void testIcebergTimestampObjectInspectorWithUTCAdjustment() {
+ IcebergTimestampWithZoneObjectInspector oi = IcebergTimestampWithZoneObjectInspector.get();
Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.TIMESTAMP, oi.getPrimitiveCategory());
@@ -47,12 +48,14 @@ public class TestIcebergTimestampObjectInspector {
Assert.assertNull(oi.copyObject(null));
Assert.assertNull(oi.getPrimitiveJavaObject(null));
Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
- LocalDateTime local = LocalDateTime.of(2020, 1, 1, 0, 0);
- Timestamp ts = Timestamp.valueOf("2020-01-01 00:00:00");
+ LocalDateTime local = LocalDateTime.of(2020, 1, 1, 16, 45, 33, 456000);
+ OffsetDateTime offsetDateTime = OffsetDateTime.of(local, ZoneOffset.ofHours(-5));
+ Timestamp ts = Timestamp.valueOf(offsetDateTime.toLocalDateTime());
- Assert.assertEquals(ts, oi.getPrimitiveJavaObject(local));
- Assert.assertEquals(new TimestampWritable(ts), oi.getPrimitiveWritableObject(local));
+ Assert.assertEquals(ts, oi.getPrimitiveJavaObject(offsetDateTime));
+ Assert.assertEquals(new TimestampWritable(ts), oi.getPrimitiveWritableObject(offsetDateTime));
Timestamp copy = (Timestamp) oi.copyObject(ts);
@@ -60,6 +63,8 @@ public class TestIcebergTimestampObjectInspector {
Assert.assertNotSame(ts, copy);
Assert.assertFalse(oi.preferWritable());
+
+ Assert.assertEquals(OffsetDateTime.of(local, ZoneOffset.UTC), oi.convert(new TimestampWritable(ts)));
}
}