You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2020/04/28 14:14:49 UTC
[hive] branch master updated: HIVE-23260: Add support for
unmodified_metadata capability in HMS translation (Naveen Gangam reviewed
by Sam An)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 836e9b1 HIVE-23260: Add support for unmodified_metadata capability in HMS translation (Naveen Gangam reviewed by Sam An)
836e9b1 is described below
commit 836e9b18dfd98cc8c66687e2a86c4c4b3f66f043
Author: Naveen Gangam <ng...@cloudera.com>
AuthorDate: Tue Apr 28 10:14:33 2020 -0400
HIVE-23260: Add support for unmodified_metadata capability in HMS translation (Naveen Gangam reviewed by Sam An)
---
.../java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +-
.../metastore/TestHiveMetastoreTransformer.java | 35 ++++++++++++++++++----
.../metastore/MetastoreDefaultTransformer.java | 32 ++++++++++++--------
3 files changed, 50 insertions(+), 19 deletions(-)
diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
index a97a623..b3faf05 100644
--- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
+++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
@@ -5775,7 +5775,7 @@ public class HiveConf extends Configuration {
// set the hardcoded value first, so anything in hiveserver2-site.xml can override it
set(ConfVars.METASTORE_CLIENT_CAPABILITIES.varname, "EXTWRITE,EXTREAD,HIVEBUCKET2,HIVEFULLACIDREAD,"
+ "HIVEFULLACIDWRITE,HIVECACHEINVALIDATE,HIVEMANAGESTATS,HIVEMANAGEDINSERTWRITE,HIVEMANAGEDINSERTREAD,"
- + "HIVESQL,HIVEMQT,HIVEONLYMQTWRITE");
+ + "HIVESQL,HIVEMQT,HIVEONLYMQTWRITE,ACCEPTS_UNMODIFIED_METADATA");
if (hiveServer2SiteUrl != null) {
addResource(hiveServer2SiteUrl);
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
index eba6610..27a1557 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -193,7 +193,14 @@ public class TestHiveMetastoreTransformer {
tbl2 = client.getTable(dbName, tblName);
assertEquals("Expected buckets does not match:", buckets, tbl2.getSd().getNumBuckets()); // no transformation
assertEquals("Table access type does not match expected value:" + tblName,
- 8, tbl2.getAccessType()); // RW with HIVEBUCKET2 but no EXTWRITE
+ ACCESSTYPE_READWRITE, tbl2.getAccessType()); // RW with HIVEBUCKET2 but no EXTWRITE
+ resetHMSClient();
+
+ setHMSClient("testTransformerAcceptsUnmodifiedMetadata", (new String[] { "ACCEPTS_UNMODIFIED_METADATA" }));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Expected buckets does not match:", buckets, tbl2.getSd().getNumBuckets()); // no transformation
+ assertEquals("Table access type does not match expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType()); // RO without HIVEBUCKET2 but with ACCEPTS_UNMODIFIED_METADATA
resetHMSClient();
tblName = "test_ext_bucketed_wc";
@@ -217,7 +224,6 @@ public class TestHiveMetastoreTransformer {
assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has the HIVEBUCKET2 capability, retain bucketing info
assertNull(tbl2.getRequiredWriteCapabilities());
assertNull(tbl2.getRequiredReadCapabilities());
-
resetHMSClient();
setHMSClient("testTransformerExternalTableRO", (new String[] { "EXTREAD", "EXTWRITE"}));
@@ -229,7 +235,17 @@ public class TestHiveMetastoreTransformer {
tbl2.getRequiredWriteCapabilities());
assertTrue("Returned required capabilities list does not contain HIVEBUCKET2",
tbl2.getRequiredWriteCapabilities().contains("HIVEBUCKET2"));
+ resetHMSClient();
+ setHMSClient("testTransformerExternalTableROwAUM", (new String[] { "EXTREAD", "EXTWRITE", "ACCEPTS_UNMODIFIED_METADATA"}));
+ tbl2 = client.getTable(dbName, tblName);
+ assertEquals("Table access type does not match the expected value:" + tblName,
+ ACCESSTYPE_READONLY, tbl2.getAccessType());
+ assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has no HIVEBUCKET2 capability, but has ACCEPTS_UNMODIFIED_METADATA
+ assertNotNull("Required write capabilities is null",
+ tbl2.getRequiredWriteCapabilities());
+ assertTrue("Returned required capabilities list does not contain HIVEBUCKET2",
+ tbl2.getRequiredWriteCapabilities().contains("HIVEBUCKET2"));
resetHMSClient();
tblName = "test_ext_unbucketed_wc";
@@ -286,7 +302,6 @@ public class TestHiveMetastoreTransformer {
assertEquals("Table access type does not match the expected value:" + tblName,
ACCESSTYPE_READWRITE, tbl2.getAccessType());
assertEquals(buckets, tbl2.getSd().getNumBuckets()); // client has SPARKBUCKET capability
-
resetHMSClient();
LOG.info("Test execution complete:testTransformerExternalTable");
@@ -867,7 +882,17 @@ public class TestHiveMetastoreTransformer {
parts = client.getPartitionsByNames(dbName, tblName, partValues, false, null);
for (Partition part : parts) {
- assertEquals("Partition bucket count does not match", -1, part.getSd().getNumBuckets());
+ assertEquals("Partition bucket count does not match", bucketCount, part.getSd().getNumBuckets());
+ }
+
+ // processor has ACCEPTS_UNMODIFIED_METADATA
+ capabilities.clear();
+ capabilities.add("ACCEPTS_UNMODIFIED_METADATA");
+ setHMSClient("TestGetPartitionByNames#3", (String[])(capabilities.toArray(new String[0])));
+ parts = client.getPartitionsByNames(dbName, tblName, partValues, false, null);
+
+ for (Partition part : parts) {
+ assertEquals("Partition bucket count does not match", bucketCount, part.getSd().getNumBuckets());
}
tblName = "test_parts_mgd_insert_wc";
@@ -1860,7 +1885,7 @@ public class TestHiveMetastoreTransformer {
PartitionBuilder partitionBuilder = new PartitionBuilder().inTable(table);
values.forEach(val -> partitionBuilder.addValue(val));
Partition p = partitionBuilder.build(conf);
- p.getSd().setNumBuckets(-1); // PartitionBuilder uses 0 as default whereas we use -1 for Tables.
+ p.getSd().setNumBuckets(table.getSd().getNumBuckets());
client.add_partition(p);
}
}
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index fa16192..3c483ee 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -65,6 +65,7 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
private static final String HIVESQL = "HIVESQL".intern();
private static final String OBJCAPABILITIES = "OBJCAPABILITIES".intern();
private static final String MANAGERAWMETADATA = "MANAGE_RAW_METADATA".intern();
+ private static final String ACCEPTSUNMODIFIEDMETADATA = "ACCEPTS_UNMODIFIED_METADATA".intern();
private static final List<String> ACIDCOMMONWRITELIST = new ArrayList(Arrays.asList(
HIVEMANAGESTATS,
@@ -137,10 +138,12 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
newTable.setAccessType(ACCESSTYPE_READONLY);
requiredWrites.add(HIVEBUCKET2);
StorageDescriptor newSd = new StorageDescriptor(table.getSd());
- newSd.setNumBuckets(-1); // remove bucketing info
+ if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) {
+ LOG.debug("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table");
+ newSd.setNumBuckets(-1); // remove bucketing info
+ }
newTable.setSd(newSd);
newTable.setRequiredWriteCapabilities(requiredWrites);
- LOG.info("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table");
}
} else { // Unbucketed
if (processorCapabilities.contains(EXTWRITE) && processorCapabilities.contains(EXTREAD)) {
@@ -269,21 +272,20 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
}
Table newTable = new Table(table);
- boolean removedBucketing = false;
-
if (requiredCapabilities.contains(HIVEBUCKET2) && !processorCapabilities.contains(HIVEBUCKET2)) {
StorageDescriptor newSd = new StorageDescriptor(table.getSd());
- newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified
+ if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA)) {
+ newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified
+ LOG.debug("Bucketed table without HIVEBUCKET2 capability, removed bucketing info from table");
+ }
newTable.setSd(newSd);
- removedBucketing = true;
newTable.setAccessType(ACCESSTYPE_READONLY);
LOG.debug("Adding HIVEBUCKET2 to requiredWrites");
requiredWrites.add(HIVEBUCKET2);
- LOG.info("Removed bucketing information from table");
}
if (requiredCapabilities.contains(EXTWRITE) && processorCapabilities.contains(EXTWRITE)) {
- if (!removedBucketing) {
+ if (!isBucketed) {
LOG.info("EXTWRITE Matches, accessType=" + ACCESSTYPE_READWRITE);
newTable.setAccessType(ACCESSTYPE_READWRITE);
ret.put(newTable, requiredCapabilities);
@@ -468,7 +470,7 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
String tCapabilities = params.get(OBJCAPABILITIES);
if (partition.getSd() != null) {
partBuckets = partition.getSd().getNumBuckets();
- LOG.info("Number of original part buckets=" + partBuckets);
+ LOG.debug("Number of original part buckets=" + partBuckets);
} else {
partBuckets = 0;
}
@@ -481,7 +483,8 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
if (partBuckets > 0 && !processorCapabilities.contains(HIVEBUCKET2)) {
Partition newPartition = new Partition(partition);
StorageDescriptor newSd = new StorageDescriptor(partition.getSd());
- newSd.setNumBuckets(-1); // remove bucketing info
+ if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA))
+ newSd.setNumBuckets(-1); // remove bucketing info
newPartition.setSd(newSd);
ret.add(newPartition);
} else {
@@ -494,7 +497,8 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
if (partBuckets > 0 && !processorCapabilities.contains(HIVEBUCKET2)) {
Partition newPartition = new Partition(partition);
StorageDescriptor newSd = new StorageDescriptor(partition.getSd());
- newSd.setNumBuckets(-1); // remove bucketing info
+ if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA))
+ newSd.setNumBuckets(-1); // remove bucketing info
newPartition.setSd(newSd);
ret.add(newPartition);
break;
@@ -520,7 +524,8 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
if (requiredCapabilities.contains(HIVEBUCKET2) && !processorCapabilities.contains(HIVEBUCKET2)) {
Partition newPartition = new Partition(partition);
StorageDescriptor newSd = new StorageDescriptor(partition.getSd());
- newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified
+ if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA))
+ newSd.setNumBuckets(-1); // removing bucketing if HIVEBUCKET2 isnt specified
newPartition.setSd(newSd);
LOG.info("Removed bucketing information from partition");
ret.add(newPartition);
@@ -532,7 +537,8 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
if (!processorCapabilities.contains(HIVEBUCKET2)) {
Partition newPartition = new Partition(partition);
StorageDescriptor newSd = new StorageDescriptor(partition.getSd());
- newSd.setNumBuckets(-1); // remove bucketing info
+ if (!processorCapabilities.contains(ACCEPTSUNMODIFIEDMETADATA))
+ newSd.setNumBuckets(-1); // remove bucketing info
newPartition.setSd(newSd);
ret.add(newPartition);
break;