You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2019/09/16 03:22:11 UTC
[hive] branch master updated: HIVE-22158: HMS Translation
layer::Disallow non-ACID MANAGED tables (Naveen Gangam,
reviewed by Thejas Nair)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 5645a97 HIVE-22158: HMS Translation layer::Disallow non-ACID MANAGED tables (Naveen Gangam, reviewed by Thejas Nair)
5645a97 is described below
commit 5645a97e40edd6ec442b679bcad39d0619c9f0e0
Author: Naveen Gangam <ng...@apache.org>
AuthorDate: Sun Sep 15 23:21:55 2019 -0400
HIVE-22158: HMS Translation layer::Disallow non-ACID MANAGED tables (Naveen Gangam, reviewed by Thejas Nair)
---
.../metastore/TestHiveMetastoreTransformer.java | 29 +++++++++++++-
.../parse/TestReplicationOnHDFSEncryptedZones.java | 2 +
.../metastore/MetastoreDefaultTransformer.java | 44 +++++++++++-----------
3 files changed, 51 insertions(+), 24 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
index 4fea863..466cce6 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -920,13 +920,36 @@ public class TestHiveMetastoreTransformer {
Table tbl2 = client.getTable(dbName, tblName);
assertEquals("Table type expected to be EXTERNAL", "EXTERNAL_TABLE", tbl2.getTableType());
- tblName = "test_create_table_mgd_woc";
+ tblName = "test_create_table_mgd_wc";
type = TableType.MANAGED_TABLE;
tProps.put("TBLNAME", tblName);
tProps.put("TBLTYPE", TableType.MANAGED_TABLE);
table_params = new StringBuilder();
table_params.append("key1=val1");
+ table_params.append(";");
+ table_params.append("transactional_properties=default");
tProps.put("PROPERTIES", table_params.toString());
+
+ List<String> capabilities = new ArrayList<>();
+ capabilities.add("HIVEFULLACIDWRITE");
+ setHMSClient("TestCreateTableMGD#1", (String[])(capabilities.toArray(new String[0])));
+
+ table = createTableWithCapabilities(tProps);
+
+ // retrieve the table
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table type expected to be converted to EXTERNAL", "EXTERNAL_TABLE", tbl2.getTableType());
+ assertNotNull("external.table.purge is expected to be non-null", tbl2.getParameters().get("external.table.purge"));
+ assertTrue("external.table.purge is expected to be true",
+ tbl2.getParameters().get("external.table.purge").equalsIgnoreCase("TRUE"));
+ assertTrue("Table params expected to contain original properties", tbl2.getParameters().get("key1").equals("val1"));
+
+ resetHMSClient();
+
+ capabilities = new ArrayList<>();
+ capabilities.add("HIVEMANAGEDINSERTWRITE");
+ setHMSClient("TestCreateTableMGD#2", (String[])(capabilities.toArray(new String[0])));
+
table = createTableWithCapabilities(tProps);
// retrieve the table
@@ -937,6 +960,8 @@ public class TestHiveMetastoreTransformer {
tbl2.getParameters().get("external.table.purge").equalsIgnoreCase("TRUE"));
assertTrue("Table params expected to contain original properties", tbl2.getParameters().get("key1").equals("val1"));
+ resetHMSClient();
+
// Test for FULL ACID tables
tblName = "test_create_table_acid_mgd_woc";
type = TableType.MANAGED_TABLE;
@@ -955,7 +980,7 @@ public class TestHiveMetastoreTransformer {
LOG.info("Create table expected to fail as ACID table cannot be created without possessing capabilities");
}
- List<String> capabilities = new ArrayList<>();
+ capabilities = new ArrayList<>();
capabilities.add("CONNECTORWRITE");
setHMSClient("TestCreateTableACID#1", (String[])(capabilities.toArray(new String[0])));
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java
index 7557280..602394a 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/parse/TestReplicationOnHDFSEncryptedZones.java
@@ -71,6 +71,7 @@ public class TestReplicationOnHDFSEncryptedZones {
primary = new WarehouseInstance(LOG, miniDFSCluster, new HashMap<String, String>() {{
put(HiveConf.ConfVars.HIVE_IN_TEST.varname, "false");
put(HiveConf.ConfVars.HIVE_SERVER2_ENABLE_DOAS.varname, "false");
+ put(HiveConf.ConfVars.REPL_INCLUDE_EXTERNAL_TABLES.varname, "true");
}}, "test_key");
}
@@ -110,6 +111,7 @@ public class TestReplicationOnHDFSEncryptedZones {
replica
.run("repl load " + replicatedDbName + " from '" + tuple.dumpLocation
+ "' with('hive.repl.add.raw.reserved.namespace'='true', "
+ + "'hive.repl.replica.external.table.base.dir'='" + replica.externalTableWarehouseRoot + "', "
+ "'distcp.options.pugpbx'='', 'distcp.options.skipcrccheck'='')")
.run("use " + replicatedDbName)
.run("repl status " + replicatedDbName)
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index 06f3630..e723aad 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -543,38 +543,38 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
params = new HashMap<>();
String tableType = newTable.getTableType();
String txnal = null;
+ String txn_properties = null;
+ boolean isInsertAcid = false;
if (TableType.MANAGED_TABLE.name().equals(tableType)) {
LOG.info("Table is a MANAGED_TABLE");
txnal = params.get(TABLE_IS_TRANSACTIONAL);
- if (txnal == null || txnal.equalsIgnoreCase("FALSE")) { // non-ACID MANAGED TABLE
- if (processorCapabilities == null || (!processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) &&
- !processorCapabilities.contains(HIVEFULLACIDWRITE))) {
- LOG.info("Converting " + newTable.getTableName() + " to EXTERNAL tableType for " + processorId);
- newTable.setTableType(TableType.EXTERNAL_TABLE.toString());
- params.remove(TABLE_IS_TRANSACTIONAL);
- params.remove(TABLE_TRANSACTIONAL_PROPERTIES);
- params.put("EXTERNAL", "TRUE");
- params.put(EXTERNAL_TABLE_PURGE, "TRUE");
- params.put("TRANSLATED_TO_EXTERNAL", "TRUE");
- newTable.setParameters(params);
- LOG.info("Modified table params are:" + params.toString());
- if (table.getSd().getLocation() == null) {
- try {
- Path newPath = hmsHandler.getWh().getDefaultTablePath(table.getDbName(), table.getTableName(), true);
- newTable.getSd().setLocation(newPath.toString());
- LOG.info("Modified location from to " + newPath);
- } catch (Exception e) {
- LOG.warn("Exception determining external table location:" + e.getMessage());
- }
+ txn_properties = params.get(TABLE_TRANSACTIONAL_PROPERTIES);
+ isInsertAcid = (txn_properties != null && txn_properties.equalsIgnoreCase("insert_only"));
+ if ((txnal == null || txnal.equalsIgnoreCase("FALSE")) && !isInsertAcid) { // non-ACID MANAGED TABLE
+ LOG.info("Converting " + newTable.getTableName() + " to EXTERNAL tableType for " + processorId);
+ newTable.setTableType(TableType.EXTERNAL_TABLE.toString());
+ params.remove(TABLE_IS_TRANSACTIONAL);
+ params.remove(TABLE_TRANSACTIONAL_PROPERTIES);
+ params.put("EXTERNAL", "TRUE");
+ params.put(EXTERNAL_TABLE_PURGE, "TRUE");
+ params.put("TRANSLATED_TO_EXTERNAL", "TRUE");
+ newTable.setParameters(params);
+ LOG.info("Modified table params are:" + params.toString());
+ if (table.getSd().getLocation() == null) {
+ try {
+ Path newPath = hmsHandler.getWh().getDefaultTablePath(table.getDbName(), table.getTableName(), true);
+ newTable.getSd().setLocation(newPath.toString());
+ LOG.info("Modified location from null to " + newPath);
+ } catch (Exception e) {
+ LOG.warn("Exception determining external table location:" + e.getMessage());
}
}
} else { // ACID table
if (processorCapabilities == null || processorCapabilities.isEmpty()) {
throw new MetaException("Processor has no capabilities, cannot create an ACID table.");
}
- String txntype = params.get(TABLE_TRANSACTIONAL_PROPERTIES);
- if (txntype != null && txntype.equalsIgnoreCase("insert_only")) { // MICRO_MANAGED Tables
+ if (isInsertAcid) { // MICRO_MANAGED Tables
if (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE)) {
LOG.info("Processor has required capabilities to be able to create INSERT-only tables");
return newTable;