You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@iceberg.apache.org by GitBox <gi...@apache.org> on 2021/01/20 16:42:53 UTC
[GitHub] [iceberg] marton-bod commented on a change in pull request #2126: Hive: Add write test for all supported types.
marton-bod commented on a change in pull request #2126:
URL: https://github.com/apache/iceberg/pull/2126#discussion_r561108864
##########
File path: mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
##########
@@ -296,6 +300,45 @@ public void testInsert() throws IOException {
HiveIcebergTestUtils.validateData(table, new ArrayList<>(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS), 0);
}
+ @Test
+ public void testInsertSupportedTypes() throws IOException {
+ Assume.assumeTrue("Tez write is not implemented yet", executionEngine.equals("mr"));
+ for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
+ Type type = SUPPORTED_TYPES.get(i);
+ // TODO: remove this filter when issue #1881 is resolved
+ if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
+ continue;
+ }
+ // TODO: remove this filter when we figure out how we could test binary types
+ if (type.equals(Types.BinaryType.get()) || type.equals(Types.FixedType.ofLength(11))) {
+ continue;
+ }
+ String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
+ String columnName = type.typeId().toString().toLowerCase() + "_column";
+
+ Schema schema = new Schema(required(1, "id", Types.LongType.get()), required(2, columnName, type));
+ List<Record> expected = TestHelper.generateRandomRecords(schema, 5, 0L);
+ List<Record> records = new ArrayList<>();
+ if (type == Types.TimestampType.withoutZone()) {
+ expected.forEach(r -> records.add(r.copy()));
+ records.forEach(r -> r.set(1, Timestamp.valueOf((LocalDateTime) r.get(1))));
+ } else if (type == Types.TimestampType.withZone()) {
+ expected.forEach(r -> records.add(r.copy()));
+ records.forEach(r -> r.set(1, Timestamp.from(((OffsetDateTime) r.get(1)).toInstant())));
+ } else {
+ records.addAll(expected);
+ }
+
+ Table table = testTables.createTable(shell, tableName, schema, fileFormat, ImmutableList.of());
+ StringBuilder query = new StringBuilder("INSERT INTO ").append(tableName).append(" VALUES")
+ .append(String.join(",", records.stream().map(r ->
+ String.format(type == Types.BooleanType.get() ? "(%s,%s)" : "(%s,'%s')", r.get(0),
+ r.get(1))).collect(Collectors.toList())));
Review comment:
I think you can use `Collectors.joining` here and then you wouldn't need the extra `String.join` step
##########
File path: mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
##########
@@ -296,6 +300,45 @@ public void testInsert() throws IOException {
HiveIcebergTestUtils.validateData(table, new ArrayList<>(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS), 0);
}
+ @Test
+ public void testInsertSupportedTypes() throws IOException {
+ Assume.assumeTrue("Tez write is not implemented yet", executionEngine.equals("mr"));
+ for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
+ Type type = SUPPORTED_TYPES.get(i);
+ // TODO: remove this filter when issue #1881 is resolved
+ if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
+ continue;
+ }
+ // TODO: remove this filter when we figure out how we could test binary types
+ if (type.equals(Types.BinaryType.get()) || type.equals(Types.FixedType.ofLength(11))) {
+ continue;
+ }
+ String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
+ String columnName = type.typeId().toString().toLowerCase() + "_column";
+
+ Schema schema = new Schema(required(1, "id", Types.LongType.get()), required(2, columnName, type));
+ List<Record> expected = TestHelper.generateRandomRecords(schema, 5, 0L);
+ List<Record> records = new ArrayList<>();
+ if (type == Types.TimestampType.withoutZone()) {
+ expected.forEach(r -> records.add(r.copy()));
+ records.forEach(r -> r.set(1, Timestamp.valueOf((LocalDateTime) r.get(1))));
+ } else if (type == Types.TimestampType.withZone()) {
+ expected.forEach(r -> records.add(r.copy()));
+ records.forEach(r -> r.set(1, Timestamp.from(((OffsetDateTime) r.get(1)).toInstant())));
+ } else {
+ records.addAll(expected);
+ }
+
+ Table table = testTables.createTable(shell, tableName, schema, fileFormat, ImmutableList.of());
+ StringBuilder query = new StringBuilder("INSERT INTO ").append(tableName).append(" VALUES")
+ .append(String.join(",", records.stream().map(r ->
+ String.format(type == Types.BooleanType.get() ? "(%s,%s)" : "(%s,'%s')", r.get(0),
Review comment:
Can you please add a small clarifying comment as to why we need the special case for the boolean type?
##########
File path: mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
##########
@@ -296,6 +300,45 @@ public void testInsert() throws IOException {
HiveIcebergTestUtils.validateData(table, new ArrayList<>(HiveIcebergStorageHandlerTestUtils.CUSTOMER_RECORDS), 0);
}
+ @Test
+ public void testInsertSupportedTypes() throws IOException {
+ Assume.assumeTrue("Tez write is not implemented yet", executionEngine.equals("mr"));
+ for (int i = 0; i < SUPPORTED_TYPES.size(); i++) {
+ Type type = SUPPORTED_TYPES.get(i);
+ // TODO: remove this filter when issue #1881 is resolved
+ if (type == Types.UUIDType.get() && fileFormat == FileFormat.PARQUET) {
+ continue;
+ }
+ // TODO: remove this filter when we figure out how we could test binary types
+ if (type.equals(Types.BinaryType.get()) || type.equals(Types.FixedType.ofLength(11))) {
+ continue;
+ }
+ String tableName = type.typeId().toString().toLowerCase() + "_table_" + i;
+ String columnName = type.typeId().toString().toLowerCase() + "_column";
+
+ Schema schema = new Schema(required(1, "id", Types.LongType.get()), required(2, columnName, type));
+ List<Record> expected = TestHelper.generateRandomRecords(schema, 5, 0L);
+ List<Record> records = new ArrayList<>();
Review comment:
very nit: you can set the initialCapacity to be `expected.size()`
##########
File path: hive3/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergDateObjectInspectorHive3.java
##########
@@ -69,4 +69,8 @@ public Object copyObject(Object o) {
}
}
+ @Override
+ public LocalDate convert(Object o) {
+ return o == null ? null : LocalDate.of(((Date) o).getYear(), ((Date) o).getMonth(), ((Date) o).getDay());
Review comment:
nit: can we cast to Date only once and reuse that? might help slightly with readability too
##########
File path: mr/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerWithEngine.java
##########
@@ -88,9 +91,10 @@
private static final List<Type> SUPPORTED_TYPES =
ImmutableList.of(Types.BooleanType.get(), Types.IntegerType.get(), Types.LongType.get(),
- Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(), Types.TimestampType.withZone(),
+ Types.FloatType.get(), Types.DoubleType.get(), Types.DateType.get(),
+ Types.TimestampType.withZone(),
Types.TimestampType.withoutZone(), Types.StringType.get(), Types.BinaryType.get(),
- Types.DecimalType.of(3, 1), Types.UUIDType.get(), Types.FixedType.ofLength(5),
+ Types.DecimalType.of(3, 1), Types.UUIDType.get(), Types.FixedType.ofLength(11),
Review comment:
Can we write different decimal scales/precisions? If so, can we add a second one here, that has a different scale to test?
##########
File path: mr/src/main/java/org/apache/iceberg/mr/hive/serde/objectinspector/IcebergTimeObjectInspector.java
##########
@@ -49,8 +50,11 @@ public Text getPrimitiveWritableObject(Object o) {
}
@Override
- public Object convert(Object o) {
- return o == null ? null : o.toString();
+ public LocalTime convert(Object o) {
+ if (o == null) {
+ return null;
+ }
+ return LocalTime.parse(o.toString());
Review comment:
Can we rely on the string representation of `o` to be parsed correctly here?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@iceberg.apache.org
For additional commands, e-mail: issues-help@iceberg.apache.org