You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2021/01/16 19:53:19 UTC
[iceberg] branch master updated: Hive: Add inspectors for UUID,
Fixed, and Time types (#2077)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 9ed3216 Hive: Add inspectors for UUID, Fixed, and Time types (#2077)
9ed3216 is described below
commit 9ed321672cb8518616194fe6eae18bdfee5247af
Author: László Pintér <47...@users.noreply.github.com>
AuthorDate: Sat Jan 16 20:53:07 2021 +0100
Hive: Add inspectors for UUID, Fixed, and Time types (#2077)
---
.../IcebergBinaryObjectInspector.java | 47 ++++-----------
...ector.java => IcebergFixedObjectInspector.java} | 52 ++++-------------
.../objectinspector/IcebergObjectInspector.java | 9 +--
.../IcebergTimeObjectInspector.java | 68 ++++++++++++++++++++++
.../IcebergUUIDObjectInspector.java | 68 ++++++++++++++++++++++
.../iceberg/mr/hive/HiveIcebergTestUtils.java | 23 ++++----
.../apache/iceberg/mr/hive/TestDeserializer.java | 23 --------
.../TestHiveIcebergStorageHandlerWithEngine.java | 11 +++-
.../TestIcebergBinaryObjectInspector.java | 32 +---------
.../TestIcebergFixedObjectInspector.java | 64 ++++++++++++++++++++
.../TestIcebergObjectInspector.java | 22 +++----
.../TestIcebergTimeObjectInspector.java | 65 +++++++++++++++++++++
.../TestIcebergUUIDObjectInspector.java | 64 ++++++++++++++++++++
13 files changed, 392 insertions(+), 156 deletions(-)
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java
index 5fa0181..61b30c7 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java
@@ -27,50 +27,22 @@ import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.BytesWritable;
import org.apache.iceberg.util.ByteBuffers;
-public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJavaObjectInspector
+public class IcebergBinaryObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements BinaryObjectInspector, WriteObjectInspector {
- private static final IcebergBinaryObjectInspector BYTE_ARRAY = new IcebergBinaryObjectInspector() {
- @Override
- byte[] toByteArray(Object o) {
- return (byte[]) o;
- }
-
- @Override
- public byte[] convert(Object o) {
- return (byte[]) o;
- }
- };
-
- private static final IcebergBinaryObjectInspector BYTE_BUFFER = new IcebergBinaryObjectInspector() {
- @Override
- byte[] toByteArray(Object o) {
- return ByteBuffers.toByteArray((ByteBuffer) o);
- }
-
- @Override
- public ByteBuffer convert(Object o) {
- return o == null ? null : ByteBuffer.wrap((byte[]) o);
- }
- };
+ private static final IcebergBinaryObjectInspector INSTANCE = new IcebergBinaryObjectInspector();
- public static IcebergBinaryObjectInspector byteArray() {
- return BYTE_ARRAY;
- }
-
- public static IcebergBinaryObjectInspector byteBuffer() {
- return BYTE_BUFFER;
+ public static IcebergBinaryObjectInspector get() {
+ return INSTANCE;
}
private IcebergBinaryObjectInspector() {
super(TypeInfoFactory.binaryTypeInfo);
}
- abstract byte[] toByteArray(Object object);
-
@Override
public byte[] getPrimitiveJavaObject(Object o) {
- return toByteArray(o);
+ return ByteBuffers.toByteArray((ByteBuffer) o);
}
@Override
@@ -87,12 +59,17 @@ public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJava
byte[] bytes = (byte[]) o;
return Arrays.copyOf(bytes, bytes.length);
} else if (o instanceof ByteBuffer) {
- ByteBuffer copy = ByteBuffer.wrap(((ByteBuffer) o).array(), ((ByteBuffer) o).arrayOffset(),
- ((ByteBuffer) o).limit());
+ ByteBuffer copy =
+ ByteBuffer.wrap(((ByteBuffer) o).array(), ((ByteBuffer) o).arrayOffset(), ((ByteBuffer) o).limit());
return copy;
} else {
return o;
}
}
+ @Override
+ public ByteBuffer convert(Object o) {
+ return o == null ? null : ByteBuffer.wrap((byte[]) o);
+ }
+
}
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergFixedObjectInspector.java
similarity index 58%
copy from mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java
copy to mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergFixedObjectInspector.java
index 5fa0181..be4ce2b 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergBinaryObjectInspector.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergFixedObjectInspector.java
@@ -19,58 +19,28 @@
package org.apache.iceberg.mr.hive.serde.objectinspector;
-import java.nio.ByteBuffer;
import java.util.Arrays;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.BinaryObjectInspector;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.hadoop.io.BytesWritable;
-import org.apache.iceberg.util.ByteBuffers;
-public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJavaObjectInspector
+public class IcebergFixedObjectInspector extends AbstractPrimitiveJavaObjectInspector
implements BinaryObjectInspector, WriteObjectInspector {
- private static final IcebergBinaryObjectInspector BYTE_ARRAY = new IcebergBinaryObjectInspector() {
- @Override
- byte[] toByteArray(Object o) {
- return (byte[]) o;
- }
-
- @Override
- public byte[] convert(Object o) {
- return (byte[]) o;
- }
- };
-
- private static final IcebergBinaryObjectInspector BYTE_BUFFER = new IcebergBinaryObjectInspector() {
- @Override
- byte[] toByteArray(Object o) {
- return ByteBuffers.toByteArray((ByteBuffer) o);
- }
-
- @Override
- public ByteBuffer convert(Object o) {
- return o == null ? null : ByteBuffer.wrap((byte[]) o);
- }
- };
+ private static final IcebergFixedObjectInspector INSTANCE = new IcebergFixedObjectInspector();
- public static IcebergBinaryObjectInspector byteArray() {
- return BYTE_ARRAY;
+ public static IcebergFixedObjectInspector get() {
+ return INSTANCE;
}
- public static IcebergBinaryObjectInspector byteBuffer() {
- return BYTE_BUFFER;
- }
-
- private IcebergBinaryObjectInspector() {
+ private IcebergFixedObjectInspector() {
super(TypeInfoFactory.binaryTypeInfo);
}
- abstract byte[] toByteArray(Object object);
-
@Override
public byte[] getPrimitiveJavaObject(Object o) {
- return toByteArray(o);
+ return (byte[]) o;
}
@Override
@@ -79,6 +49,11 @@ public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJava
}
@Override
+ public byte[] convert(Object o) {
+ return o == null ? null : (byte[]) o;
+ }
+
+ @Override
public Object copyObject(Object o) {
if (o == null) {
return null;
@@ -86,13 +61,8 @@ public abstract class IcebergBinaryObjectInspector extends AbstractPrimitiveJava
if (o instanceof byte[]) {
byte[] bytes = (byte[]) o;
return Arrays.copyOf(bytes, bytes.length);
- } else if (o instanceof ByteBuffer) {
- ByteBuffer copy = ByteBuffer.wrap(((ByteBuffer) o).array(), ((ByteBuffer) o).arrayOffset(),
- ((ByteBuffer) o).limit());
- return copy;
} else {
return o;
}
}
-
}
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
index 9175f3f..0399502 100644
--- a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergObjectInspector.java
@@ -98,7 +98,7 @@ public final class IcebergObjectInspector extends TypeUtil.SchemaVisitor<ObjectI
switch (primitiveType.typeId()) {
case BINARY:
- return IcebergBinaryObjectInspector.byteBuffer();
+ return IcebergBinaryObjectInspector.get();
case BOOLEAN:
primitiveTypeInfo = TypeInfoFactory.booleanTypeInfo;
break;
@@ -111,7 +111,7 @@ public final class IcebergObjectInspector extends TypeUtil.SchemaVisitor<ObjectI
primitiveTypeInfo = TypeInfoFactory.doubleTypeInfo;
break;
case FIXED:
- return IcebergBinaryObjectInspector.byteArray();
+ return IcebergFixedObjectInspector.get();
case FLOAT:
primitiveTypeInfo = TypeInfoFactory.floatTypeInfo;
break;
@@ -122,14 +122,15 @@ public final class IcebergObjectInspector extends TypeUtil.SchemaVisitor<ObjectI
primitiveTypeInfo = TypeInfoFactory.longTypeInfo;
break;
case STRING:
- case UUID:
primitiveTypeInfo = TypeInfoFactory.stringTypeInfo;
break;
+ case UUID:
+ return IcebergUUIDObjectInspector.get();
case TIMESTAMP:
boolean adjustToUTC = ((Types.TimestampType) primitiveType).shouldAdjustToUTC();
return adjustToUTC ? TIMESTAMP_INSPECTOR_WITH_TZ : TIMESTAMP_INSPECTOR;
-
case TIME:
+ return IcebergTimeObjectInspector.get();
default:
throw new IllegalArgumentException(primitiveType.typeId() + " type is not supported");
}
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimeObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimeObjectInspector.java
new file mode 100644
index 0000000..f5be1fe
--- /dev/null
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimeObjectInspector.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+
+public class IcebergTimeObjectInspector extends AbstractPrimitiveJavaObjectInspector
+ implements StringObjectInspector, WriteObjectInspector {
+
+ private static final IcebergTimeObjectInspector INSTANCE = new IcebergTimeObjectInspector();
+
+ private IcebergTimeObjectInspector() {
+ super(TypeInfoFactory.stringTypeInfo);
+ }
+
+ public static IcebergTimeObjectInspector get() {
+ return INSTANCE;
+ }
+
+ @Override
+ public String getPrimitiveJavaObject(Object o) {
+ return o == null ? null : o.toString();
+ }
+
+ @Override
+ public Text getPrimitiveWritableObject(Object o) {
+ String value = getPrimitiveJavaObject(o);
+ return value == null ? null : new Text(value);
+ }
+
+ @Override
+ public Object convert(Object o) {
+ return o == null ? null : o.toString();
+ }
+
+ @Override
+ public Object copyObject(Object o) {
+ if (o == null) {
+ return null;
+ }
+
+ if (o instanceof Text) {
+ return new Text((Text) o);
+ } else {
+ return o;
+ }
+ }
+}
diff --git a/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergUUIDObjectInspector.java b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergUUIDObjectInspector.java
new file mode 100644
index 0000000..0cb8464
--- /dev/null
+++ b/mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergUUIDObjectInspector.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.AbstractPrimitiveJavaObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.StringObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+
+public class IcebergUUIDObjectInspector extends AbstractPrimitiveJavaObjectInspector
+ implements StringObjectInspector, WriteObjectInspector {
+
+ private static final IcebergUUIDObjectInspector INSTANCE = new IcebergUUIDObjectInspector();
+
+ private IcebergUUIDObjectInspector() {
+ super(TypeInfoFactory.stringTypeInfo);
+ }
+
+ public static IcebergUUIDObjectInspector get() {
+ return INSTANCE;
+ }
+
+ @Override
+ public String getPrimitiveJavaObject(Object o) {
+ return o == null ? null : o.toString();
+ }
+
+ @Override
+ public Text getPrimitiveWritableObject(Object o) {
+ String value = getPrimitiveJavaObject(o);
+ return value == null ? null : new Text(value);
+ }
+
+ @Override
+ public String convert(Object o) {
+ return o == null ? null : o.toString();
+ }
+
+ @Override
+ public Object copyObject(Object o) {
+ if (o == null) {
+ return null;
+ }
+
+ if (o instanceof Text) {
+ return new Text((Text) o);
+ } else {
+ return o;
+ }
+ }
+}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java
index 9114400..7f497cd 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/HiveIcebergTestUtils.java
@@ -74,19 +74,20 @@ public class HiveIcebergTestUtils {
optional(4, "float_type", Types.FloatType.get()),
optional(5, "double_type", Types.DoubleType.get()),
optional(6, "date_type", Types.DateType.get()),
- // TimeType is not supported
- // required(7, "time_type", Types.TimeType.get()),
optional(7, "tstz", Types.TimestampType.withZone()),
optional(8, "ts", Types.TimestampType.withoutZone()),
optional(9, "string_type", Types.StringType.get()),
optional(10, "fixed_type", Types.FixedType.ofLength(3)),
optional(11, "binary_type", Types.BinaryType.get()),
- optional(12, "decimal_type", Types.DecimalType.of(38, 10)));
+ optional(12, "decimal_type", Types.DecimalType.of(38, 10)),
+ optional(13, "time_type", Types.TimeType.get()),
+ optional(14, "uuid_type", Types.UUIDType.get()));
public static final StandardStructObjectInspector FULL_SCHEMA_OBJECT_INSPECTOR =
ObjectInspectorFactory.getStandardStructObjectInspector(
Arrays.asList("boolean_type", "integer_type", "long_type", "float_type", "double_type",
- "date_type", "tstz", "ts", "string_type", "fixed_type", "binary_type", "decimal_type"),
+ "date_type", "tstz", "ts", "string_type", "fixed_type", "binary_type", "decimal_type",
+ "time_type", "uuid_type"),
Arrays.asList(
PrimitiveObjectInspectorFactory.writableBooleanObjectInspector,
PrimitiveObjectInspectorFactory.writableIntObjectInspector,
@@ -99,7 +100,9 @@ public class HiveIcebergTestUtils {
PrimitiveObjectInspectorFactory.writableStringObjectInspector,
PrimitiveObjectInspectorFactory.writableBinaryObjectInspector,
PrimitiveObjectInspectorFactory.writableBinaryObjectInspector,
- PrimitiveObjectInspectorFactory.writableHiveDecimalObjectInspector
+ PrimitiveObjectInspectorFactory.writableHiveDecimalObjectInspector,
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector,
+ PrimitiveObjectInspectorFactory.writableStringObjectInspector
));
private HiveIcebergTestUtils() {
@@ -118,8 +121,6 @@ public class HiveIcebergTestUtils {
record.set(3, 3.1f);
record.set(4, 4.2d);
record.set(5, LocalDate.of(2020, 1, 21));
- // TimeType is not supported
- // record.set(6, LocalTime.of(11, 33));
// Nano is not supported ?
record.set(6, OffsetDateTime.of(2017, 11, 22, 11, 30, 7, 0, ZoneOffset.ofHours(2)));
record.set(7, LocalDateTime.of(2019, 2, 22, 9, 44, 54));
@@ -127,6 +128,8 @@ public class HiveIcebergTestUtils {
record.set(9, new byte[]{0, 1, 2});
record.set(10, ByteBuffer.wrap(new byte[]{0, 1, 2, 3}));
record.set(11, new BigDecimal("0.0000000013"));
+ record.set(12, "11:33");
+ record.set(13, "73689599-d7fc-4dfb-b94e-106ff20284a5");
return record;
}
@@ -158,14 +161,14 @@ public class HiveIcebergTestUtils {
new FloatWritable(record.get(3, Float.class)),
new DoubleWritable(record.get(4, Double.class)),
new DateWritable((int) record.get(5, LocalDate.class).toEpochDay()),
- // TimeType is not supported
- // new Timestamp()
new TimestampWritable(Timestamp.from(record.get(6, OffsetDateTime.class).toInstant())),
new TimestampWritable(Timestamp.valueOf(record.get(7, LocalDateTime.class))),
new Text(record.get(8, String.class)),
new BytesWritable(record.get(9, byte[].class)),
new BytesWritable(ByteBuffers.toByteArray(record.get(10, ByteBuffer.class))),
- new HiveDecimalWritable(HiveDecimal.create(record.get(11, BigDecimal.class)))
+ new HiveDecimalWritable(HiveDecimal.create(record.get(11, BigDecimal.class))),
+ new Text(record.get(12, String.class)),
+ new Text(record.get(13, String.class))
);
}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestDeserializer.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestDeserializer.java
index d31a5d7..adad32a 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestDeserializer.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestDeserializer.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectIn
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.MapWritable;
import org.apache.hadoop.io.Text;
-import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.Schema;
import org.apache.iceberg.data.GenericRecord;
import org.apache.iceberg.data.Record;
@@ -196,26 +195,4 @@ public class TestDeserializer {
// Check null record as well
Assert.assertNull(deserializer.deserialize(null));
}
-
- @Test
- public void testUnsupportedType() {
- Schema unsupported = new Schema(
- optional(1, "time_type", Types.TimeType.get())
- );
- StandardStructObjectInspector objectInspector = ObjectInspectorFactory.getStandardStructObjectInspector(
- Arrays.asList("time_type"),
- Arrays.asList(
- PrimitiveObjectInspectorFactory.writableStringObjectInspector
- ));
-
- AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class,
- "type is not supported", () -> {
- new Deserializer.Builder()
- .schema(unsupported)
- .writerInspector((StructObjectInspector) IcebergObjectInspector.create(unsupported))
- .sourceInspector(objectInspector)
- .build();
- }
- );
- }
}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
index 7df3418..8679df7 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
@@ -81,7 +81,8 @@ public class TestHiveIcebergStorageHandlerWithEngine {
ImmutableList.of(Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(),
Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(),
Types.TimestampType.withoutZone(), Types.StringType.get(), Types.BinaryType.get(),
- Types.DecimalType.of(3, 1));
+ Types.DecimalType.of(3, 1), Types.UUIDType.get(), Types.FixedType.ofLength(5),
+ Types.TimeType.get());
@Parameters(name = "fileFormat={0}, engine={1}, catalog={2}")
public static Collection<Object[]> parameters() {
@@ -224,6 +225,10 @@ public class TestHiveIcebergStorageHandlerWithEngine {
public void testJoinTablesSupportedTypes() throws IOException {
for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
Type type = SUPPORTED_TYPES.get(i);
+ // TODO: remove this filter when issue #1881 is resolved
+ if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
+ continue;
+ }
String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
String columnName = type.typeId().toString().toLowerCase() + "_column";
@@ -243,6 +248,10 @@ public class TestHiveIcebergStorageHandlerWithEngine {
public void testSelectDistinctFromTable() throws IOException {
for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
Type type = SUPPORTED_TYPES.get(i);
+ // TODO: remove this filter when issue #1881 is resolved
+ if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
+ continue;
+ }
String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
String columnName = type.typeId().toString().toLowerCase() + "_column";
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergBinaryObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergBinaryObjectInspector.java
index 3209600..9d47d74 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergBinaryObjectInspector.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergBinaryObjectInspector.java
@@ -31,38 +31,8 @@ import org.junit.Test;
public class TestIcebergBinaryObjectInspector {
@Test
- public void testIcebergByteArrayObjectInspector() {
- BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteArray();
-
- Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
- Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory());
-
- Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo());
- Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName());
-
- Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass());
- Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass());
-
- Assert.assertNull(oi.copyObject(null));
- Assert.assertNull(oi.getPrimitiveJavaObject(null));
- Assert.assertNull(oi.getPrimitiveWritableObject(null));
-
- byte[] bytes = new byte[] {0, 1};
-
- Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(bytes));
- Assert.assertEquals(new BytesWritable(bytes), oi.getPrimitiveWritableObject(bytes));
-
- byte[] copy = (byte[]) oi.copyObject(bytes);
-
- Assert.assertArrayEquals(bytes, copy);
- Assert.assertNotSame(bytes, copy);
-
- Assert.assertFalse(oi.preferWritable());
- }
-
- @Test
public void testIcebergByteBufferObjectInspector() {
- BinaryObjectInspector oi = IcebergBinaryObjectInspector.byteBuffer();
+ BinaryObjectInspector oi = IcebergBinaryObjectInspector.get();
Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory());
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergFixedObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergFixedObjectInspector.java
new file mode 100644
index 0000000..82bfa5e
--- /dev/null
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergFixedObjectInspector.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.BytesWritable;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestIcebergFixedObjectInspector {
+
+ @Test
+ public void testIcebergFixedObjectInspector() {
+ IcebergFixedObjectInspector oi = IcebergFixedObjectInspector.get();
+
+ Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
+ Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.BINARY, oi.getPrimitiveCategory());
+
+ Assert.assertEquals(TypeInfoFactory.binaryTypeInfo, oi.getTypeInfo());
+ Assert.assertEquals(TypeInfoFactory.binaryTypeInfo.getTypeName(), oi.getTypeName());
+
+ Assert.assertEquals(byte[].class, oi.getJavaPrimitiveClass());
+ Assert.assertEquals(BytesWritable.class, oi.getPrimitiveWritableClass());
+
+ Assert.assertNull(oi.copyObject(null));
+ Assert.assertNull(oi.getPrimitiveJavaObject(null));
+ Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
+
+ byte[] bytes = new byte[] { 0, 1 };
+ BytesWritable bytesWritable = new BytesWritable(bytes);
+
+ Assert.assertArrayEquals(bytes, oi.getPrimitiveJavaObject(bytes));
+ Assert.assertEquals(bytesWritable, oi.getPrimitiveWritableObject(bytes));
+ Assert.assertEquals(bytes, oi.convert(bytes));
+
+ byte[] copy = (byte[]) oi.copyObject(bytes);
+
+ Assert.assertArrayEquals(bytes, copy);
+ Assert.assertNotSame(bytes, copy);
+
+ Assert.assertFalse(oi.preferWritable());
+ }
+
+}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
index f4178d7..1ce9a9f 100644
--- a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergObjectInspector.java
@@ -26,7 +26,6 @@ import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
-import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.Schema;
import org.apache.iceberg.hive.MetastoreUtil;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
@@ -61,7 +60,8 @@ public class TestIcebergObjectInspector {
required(19, "struct_field", Types.StructType.of(
Types.NestedField.required(20, "nested_field", Types.StringType.get(), "nested field comment")),
"struct comment"
- )
+ ),
+ required(21, "time_field", Types.TimeType.get(), "time comment")
);
@Test
@@ -77,7 +77,7 @@ public class TestIcebergObjectInspector {
Assert.assertEquals(1, binaryField.getFieldID());
Assert.assertEquals("binary_field", binaryField.getFieldName());
Assert.assertEquals("binary comment", binaryField.getFieldComment());
- Assert.assertEquals(IcebergBinaryObjectInspector.byteBuffer(), binaryField.getFieldObjectInspector());
+ Assert.assertEquals(IcebergBinaryObjectInspector.get(), binaryField.getFieldObjectInspector());
// boolean
StructField booleanField = soi.getStructFieldRef("boolean_field");
@@ -118,7 +118,7 @@ public class TestIcebergObjectInspector {
Assert.assertEquals(6, fixedField.getFieldID());
Assert.assertEquals("fixed_field", fixedField.getFieldName());
Assert.assertEquals("fixed comment", fixedField.getFieldComment());
- Assert.assertEquals(IcebergBinaryObjectInspector.byteArray(), fixedField.getFieldObjectInspector());
+ Assert.assertEquals(IcebergFixedObjectInspector.get(), fixedField.getFieldObjectInspector());
// float
StructField floatField = soi.getStructFieldRef("float_field");
@@ -177,7 +177,7 @@ public class TestIcebergObjectInspector {
Assert.assertEquals(13, uuidField.getFieldID());
Assert.assertEquals("uuid_field", uuidField.getFieldName());
Assert.assertEquals("uuid comment", uuidField.getFieldComment());
- Assert.assertEquals(getPrimitiveObjectInspector(String.class), uuidField.getFieldObjectInspector());
+ Assert.assertEquals(IcebergUUIDObjectInspector.get(), uuidField.getFieldObjectInspector());
// list
StructField listField = soi.getStructFieldRef("list_field");
@@ -202,13 +202,13 @@ public class TestIcebergObjectInspector {
ObjectInspector expectedObjectInspector = new IcebergRecordObjectInspector(
(Types.StructType) schema.findType(19), ImmutableList.of(getPrimitiveObjectInspector(String.class)));
Assert.assertEquals(expectedObjectInspector, structField.getFieldObjectInspector());
- }
- @Test
- public void testIcebergObjectInspectorUnsupportedTypes() {
- AssertHelpers.assertThrows(
- "Hive does not support time type", IllegalArgumentException.class, "TIME type is not supported",
- () -> IcebergObjectInspector.create(required(1, "time_field", Types.TimeType.get())));
+ // time
+ StructField timeField = soi.getStructFieldRef("time_field");
+ Assert.assertEquals(21, timeField.getFieldID());
+ Assert.assertEquals("time_field", timeField.getFieldName());
+ Assert.assertEquals("time comment", timeField.getFieldComment());
+ Assert.assertEquals(IcebergTimeObjectInspector.get(), timeField.getFieldObjectInspector());
}
private static ObjectInspector getPrimitiveObjectInspector(Class<?> clazz) {
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimeObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimeObjectInspector.java
new file mode 100644
index 0000000..5ca5fe6
--- /dev/null
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergTimeObjectInspector.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import java.time.LocalTime;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestIcebergTimeObjectInspector {
+
+ @Test
+ public void testIcebergTimeObjectInspector() {
+
+ IcebergTimeObjectInspector oi = IcebergTimeObjectInspector.get();
+
+ Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
+ Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING, oi.getPrimitiveCategory());
+
+ Assert.assertEquals(TypeInfoFactory.stringTypeInfo, oi.getTypeInfo());
+ Assert.assertEquals(TypeInfoFactory.stringTypeInfo.getTypeName(), oi.getTypeName());
+
+ Assert.assertEquals(String.class, oi.getJavaPrimitiveClass());
+ Assert.assertEquals(Text.class, oi.getPrimitiveWritableClass());
+
+ Assert.assertNull(oi.copyObject(null));
+ Assert.assertNull(oi.getPrimitiveJavaObject(null));
+ Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
+
+ String time = LocalTime.now().toString();
+ Text text = new Text(time);
+
+ Assert.assertEquals(time, oi.getPrimitiveJavaObject(text));
+ Assert.assertEquals(text, oi.getPrimitiveWritableObject(time));
+ Assert.assertEquals(time, oi.convert(text));
+
+ Text copy = (Text) oi.copyObject(text);
+
+ Assert.assertEquals(text, copy);
+ Assert.assertNotSame(text, copy);
+
+ Assert.assertFalse(oi.preferWritable());
+ }
+}
diff --git a/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergUUIDObjectInspector.java b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergUUIDObjectInspector.java
new file mode 100644
index 0000000..da13b32
--- /dev/null
+++ b/mr/src/test/java/org/apache/iceberg/mr/hive/serde/objectinspector/TestIcebergUUIDObjectInspector.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.mr.hive.serde.objectinspector;
+
+import java.util.UUID;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.io.Text;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestIcebergUUIDObjectInspector {
+
+ @Test
+ public void testIcebergUUIDObjectInspector() {
+ IcebergUUIDObjectInspector oi = IcebergUUIDObjectInspector.get();
+
+ Assert.assertEquals(ObjectInspector.Category.PRIMITIVE, oi.getCategory());
+ Assert.assertEquals(PrimitiveObjectInspector.PrimitiveCategory.STRING, oi.getPrimitiveCategory());
+
+ Assert.assertEquals(TypeInfoFactory.stringTypeInfo, oi.getTypeInfo());
+ Assert.assertEquals(TypeInfoFactory.stringTypeInfo.getTypeName(), oi.getTypeName());
+
+ Assert.assertEquals(String.class, oi.getJavaPrimitiveClass());
+ Assert.assertEquals(Text.class, oi.getPrimitiveWritableClass());
+
+ Assert.assertNull(oi.copyObject(null));
+ Assert.assertNull(oi.getPrimitiveJavaObject(null));
+ Assert.assertNull(oi.getPrimitiveWritableObject(null));
+ Assert.assertNull(oi.convert(null));
+
+ String uuid = UUID.randomUUID().toString();
+ Text text = new Text(uuid);
+
+ Assert.assertEquals(uuid, oi.getPrimitiveJavaObject(text));
+ Assert.assertEquals(text, oi.getPrimitiveWritableObject(uuid));
+ Assert.assertEquals(uuid, oi.convert(text));
+
+ Text copy = (Text) oi.copyObject(text);
+
+ Assert.assertEquals(text, copy);
+ Assert.assertNotSame(text, copy);
+
+ Assert.assertFalse(oi.preferWritable());
+ }
+}