You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by lp...@apache.org on 2022/06/03 11:43:21 UTC
[hive] branch master updated: HIVE-26282: Improve iceberg CTAS error message for unsupported types (#3337) (Laszlo Pinter, reviewed by Peter Vary)
This is an automated email from the ASF dual-hosted git repository.
lpinter pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 8ec1d4c989 HIVE-26282: Improve iceberg CTAS error message for unsupported types (#3337) (Laszlo Pinter, reviewed by Peter Vary)
8ec1d4c989 is described below
commit 8ec1d4c989baa0c4863084297b73e80e5929044d
Author: László Pintér <47...@users.noreply.github.com>
AuthorDate: Fri Jun 3 13:43:10 2022 +0200
HIVE-26282: Improve iceberg CTAS error message for unsupported types (#3337) (Laszlo Pinter, reviewed by Peter Vary)
---
.../apache/iceberg/hive/HiveSchemaConverter.java | 6 ++-
.../iceberg/mr/hive/TestHiveIcebergCTAS.java | 57 ++++++++++++++++++++++
2 files changed, 61 insertions(+), 2 deletions(-)
diff --git a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
index 422546c112..9122577d41 100644
--- a/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
+++ b/iceberg/iceberg-catalog/src/main/java/org/apache/iceberg/hive/HiveSchemaConverter.java
@@ -83,7 +83,8 @@ class HiveSchemaConverter {
return Types.BooleanType.get();
case BYTE:
case SHORT:
- Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer instead",
+ Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer " +
+ "instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true",
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
LOG.debug("Using auto conversion from SHORT/BYTE to INTEGER");
@@ -96,7 +97,8 @@ class HiveSchemaConverter {
return Types.BinaryType.get();
case CHAR:
case VARCHAR:
- Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use string instead",
+ Preconditions.checkArgument(autoConvert, "Unsupported Hive type: %s, use integer " +
+ "instead or enable automatic type conversion, set 'iceberg.mr.schema.auto.conversion' to true",
((PrimitiveTypeInfo) typeInfo).getPrimitiveCategory());
LOG.debug("Using auto conversion from CHAR/VARCHAR to STRING");
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
index e41a4c6fc6..fda9652e0f 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergCTAS.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.mr.hive;
import java.io.IOException;
import java.util.Comparator;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.hive.serde.serdeConstants;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.PartitionSpec;
@@ -32,8 +33,10 @@ import org.apache.iceberg.TableProperties;
import org.apache.iceberg.catalog.TableIdentifier;
import org.apache.iceberg.data.Record;
import org.apache.iceberg.exceptions.NoSuchTableException;
+import org.apache.iceberg.mr.InputFormatConfig;
import org.apache.iceberg.mr.TestHelper;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.thrift.TException;
import org.junit.Assert;
@@ -222,4 +225,58 @@ public class TestHiveIcebergCTAS extends HiveIcebergStorageHandlerWithEngineBase
objects = shell.executeStatement("SELECT * FROM target");
Assert.assertTrue(objects.isEmpty());
}
+
+ @Test
+ public void testCTASUnsupportedTypeWithoutAutoConversion() {
+ Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType == TestTables.TestTableType.HIVE_CATALOG);
+ Map<String, Type> notSupportedTypes = ImmutableMap.of(
+ "TINYINT", Types.IntegerType.get(),
+ "SMALLINT", Types.IntegerType.get(),
+ "VARCHAR(1)", Types.StringType.get(),
+ "CHAR(1)", Types.StringType.get());
+
+
+ for (String notSupportedType : notSupportedTypes.keySet()) {
+ shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED AS ORC", notSupportedType));
+ AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class,
+ "Unsupported Hive type: ", () -> {
+ shell.executeStatement(String.format(
+ "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source",
+ testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")),
+ testTables.propertiesForCreateTableSQL(
+ ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())
+ )
+ ));
+ });
+ shell.executeStatement("DROP TABLE source");
+ }
+ }
+
+ @Test
+ public void testCTASUnsupportedTypeWithAutoConversion() {
+ Assume.assumeTrue(HiveIcebergSerDe.CTAS_EXCEPTION_MSG, testTableType == TestTables.TestTableType.HIVE_CATALOG);
+ Map<String, Type> notSupportedTypes = ImmutableMap.of(
+ "TINYINT", Types.IntegerType.get(),
+ "SMALLINT", Types.IntegerType.get(),
+ "VARCHAR(1)", Types.StringType.get(),
+ "CHAR(1)", Types.StringType.get());
+
+ shell.setHiveSessionValue(InputFormatConfig.SCHEMA_AUTO_CONVERSION, "true");
+
+ for (String notSupportedType : notSupportedTypes.keySet()) {
+ shell.executeStatement(String.format("CREATE TABLE source (s %s) STORED AS ORC", notSupportedType));
+ shell.executeStatement(String.format(
+ "CREATE TABLE target STORED BY ICEBERG %s %s AS SELECT * FROM source",
+ testTables.locationForCreateTableSQL(TableIdentifier.of("default", "target")),
+ testTables.propertiesForCreateTableSQL(
+ ImmutableMap.of(TableProperties.DEFAULT_FILE_FORMAT, fileFormat.toString())
+ )
+ ));
+
+ org.apache.iceberg.Table icebergTable = testTables.loadTable(TableIdentifier.of("default", "target"));
+ Assert.assertEquals(notSupportedTypes.get(notSupportedType), icebergTable.schema().columns().get(0).type());
+ shell.executeStatement("DROP TABLE source");
+ shell.executeStatement("DROP TABLE target");
+ }
+ }
}