You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ay...@apache.org on 2022/09/12 09:14:37 UTC
[hive] branch master updated: HIVE-26519: Iceberg: Add support for CTLT queries. (#3577). (Ayush Saxena, reviewed by Laszlo Pinter)
This is an automated email from the ASF dual-hosted git repository.
ayushsaxena pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new d96c31b2a87 HIVE-26519: Iceberg: Add support for CTLT queries. (#3577). (Ayush Saxena, reviewed by Laszlo Pinter)
d96c31b2a87 is described below
commit d96c31b2a87367279ef7e61ce8cda60d04db303c
Author: Ayush Saxena <ay...@apache.org>
AuthorDate: Mon Sep 12 14:44:23 2022 +0530
HIVE-26519: Iceberg: Add support for CTLT queries. (#3577). (Ayush Saxena, reviewed by Laszlo Pinter)
---
.../iceberg/mr/hive/HiveIcebergMetaHook.java | 3 ++
.../hive/TestHiveIcebergStorageHandlerNoScan.java | 48 ++++++++++++++++++++++
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 2 +
.../apache/hadoop/hive/ql/parse/StorageFormat.java | 20 ++++++++-
4 files changed, 71 insertions(+), 2 deletions(-)
diff --git a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
index f7591f79648..29e05652399 100644
--- a/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
+++ b/iceberg/iceberg-handler/src/main/java/org/apache/iceberg/mr/hive/HiveIcebergMetaHook.java
@@ -156,6 +156,9 @@ public class HiveIcebergMetaHook implements HiveMetaHook {
BaseMetastoreTableOperations.ICEBERG_TABLE_TYPE_VALUE.toUpperCase());
if (!Catalogs.hiveCatalog(conf, catalogProperties)) {
+ if (Boolean.parseBoolean(this.catalogProperties.getProperty(hive_metastoreConstants.TABLE_IS_CTLT))) {
+ throw new RuntimeException("CTLT target table must be a HiveCatalog table.");
+ }
// For non-HiveCatalog tables too, we should set the input and output format
// so that the table can be read by other engines like Impala
hmsTable.getSd().setInputFormat(HiveIcebergInputFormat.class.getCanonicalName());
diff --git a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
index b8938d49830..2a617e88fe2 100644
--- a/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
+++ b/iceberg/iceberg-handler/src/test/java/org/apache/iceberg/mr/hive/TestHiveIcebergStorageHandlerNoScan.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.EnvironmentContext;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
+import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.security.authorization.plugin.HiveAuthorizer;
@@ -1587,6 +1588,53 @@ public class TestHiveIcebergStorageHandlerNoScan {
});
}
+ @Test
+ public void testCTLT() throws TException, InterruptedException {
+ Assume.assumeTrue(" CTLT target table must be a HiveCatalog table",
+ testTableType == TestTables.TestTableType.HIVE_CATALOG);
+ // Create a normal table and add some data
+ shell.executeStatement("CREATE TABLE source(a int)");
+ shell.executeStatement("insert into source values(1)");
+
+ // Run a CTLT query.
+ shell.executeStatement(String.format("CREATE TABLE dest LIKE source STORED BY ICEBERG %s %s",
+ testTables.locationForCreateTableSQL(TableIdentifier.of("default", "dest")),
+ testTables.propertiesForCreateTableSQL(ImmutableMap.of())));
+
+ // Try a select query and check if the table is empty .
+ String result = shell.executeAndStringify("select a from " + TableIdentifier.of("default", "dest").name());
+ Assert.assertTrue(result.isEmpty());
+
+ // Validate the properties of the table.
+ org.apache.hadoop.hive.metastore.api.Table hmsTable = shell.metastore().getTable("default", "dest");
+ StorageDescriptor sd = hmsTable.getSd();
+ Assert.assertEquals("org.apache.iceberg.mr.hive.HiveIcebergSerDe", sd.getSerdeInfo().getSerializationLib());
+ Assert.assertEquals("org.apache.iceberg.mr.hive.HiveIcebergInputFormat", sd.getInputFormat());
+ Assert.assertEquals("org.apache.iceberg.mr.hive.HiveIcebergOutputFormat", sd.getOutputFormat());
+ Assert.assertEquals("org.apache.iceberg.mr.hive.HiveIcebergStorageHandler",
+ hmsTable.getParameters().get("storage_handler"));
+ Assert.assertEquals("ICEBERG", hmsTable.getParameters().get("table_type"));
+ }
+
+ @Test
+ public void testCTLTHiveCatalogValidation() throws TException, InterruptedException {
+ Assume.assumeTrue(" CTLT target table works on HiveCatalog table",
+ testTableType != TestTables.TestTableType.HIVE_CATALOG);
+
+ // Create a normal table and add some data
+ shell.executeStatement("CREATE TABLE source(a int)");
+ shell.executeStatement("insert into source values(1)");
+
+ // Run a CTLT query.
+ AssertHelpers.assertThrows("should throw exception", IllegalArgumentException.class,
+ " CTLT target table must be a HiveCatalog table", () -> {
+ shell.executeStatement(String.format("CREATE TABLE dest LIKE source STORED BY ICEBERG %s %s",
+ testTables.locationForCreateTableSQL(TableIdentifier.of("default", "dest")),
+ testTables.propertiesForCreateTableSQL(ImmutableMap.of())));
+ });
+ }
+
+
/**
* Checks that the new schema has newintcol and newstring col columns on both HMS and Iceberg sides
* @throws Exception - any test error
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index 9b6c995bbd1..7afea4a222b 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.DYNAMICPARTITIONCONV
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVEARCHIVEENABLED;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVE_DEFAULT_STORAGE_HANDLER;
import static org.apache.hadoop.hive.conf.HiveConf.ConfVars.HIVESTATSDBCLASS;
+import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.META_TABLE_STORAGE;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.TABLE_IS_CTAS;
import static org.apache.hadoop.hive.metastore.api.hive_metastoreConstants.DEFAULT_TABLE_TYPE;
import static org.apache.hadoop.hive.ql.ddl.view.create.AbstractCreateViewAnalyzer.validateTablesUsed;
@@ -14036,6 +14037,7 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
try {
storageHandler = (HiveStorageHandler) ReflectionUtils.newInstance(
conf.getClassByName(storageFormat.getStorageHandler()), SessionState.get().getConf());
+ t.setProperty(META_TABLE_STORAGE, storageHandler.getClass().getName());
} catch (ClassNotFoundException ex) {
LOG.error("Class not found. Storage handler will be set to null: "+ex.getMessage() , ex);
}
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java
index 0ab680f70fe..3e9b854d7ce 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/StorageFormat.java
@@ -48,16 +48,30 @@ public class StorageFormat {
private final Map<String, String> serdeProps;
private enum StorageHandlerTypes {
- ICEBERG("\'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler\'");
+ ICEBERG("\'org.apache.iceberg.mr.hive.HiveIcebergStorageHandler\'",
+ "org.apache.iceberg.mr.hive.HiveIcebergInputFormat", "org.apache.iceberg.mr.hive.HiveIcebergOutputFormat");
private final String className;
- StorageHandlerTypes(String className) {
+ private final String inputFormat;
+ private final String outputFormat;
+
+ private StorageHandlerTypes(String className, String inputFormat, String outputFormat) {
this.className = className;
+ this.inputFormat = inputFormat;
+ this.outputFormat = outputFormat;
}
public String className() {
return className;
}
+
+ public String inputFormat() {
+ return inputFormat;
+ }
+
+ public String outputFormat() {
+ return outputFormat;
+ }
}
public StorageFormat(Configuration conf) {
@@ -141,6 +155,8 @@ public class StorageFormat {
for (StorageHandlerTypes type : StorageHandlerTypes.values()) {
if (type.name().equalsIgnoreCase(name)) {
name = type.className();
+ inputFormat = type.inputFormat();
+ outputFormat = type.outputFormat();
break;
}
}