You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ng...@apache.org on 2020/07/21 16:39:18 UTC
[hive] branch master updated: HIVE-23603: transformDatabase()
should work with changes from HIVE-22995 (Naveen Gangam)
This is an automated email from the ASF dual-hosted git repository.
ngangam pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 1c0deeb HIVE-23603: transformDatabase() should work with changes from HIVE-22995 (Naveen Gangam)
1c0deeb is described below
commit 1c0deebc998ba15f83b33beb8c407dcc4b32141f
Author: Naveen Gangam <ng...@cloudera.com>
AuthorDate: Mon Jun 15 13:35:38 2020 -0400
HIVE-23603: transformDatabase() should work with changes from HIVE-22995 (Naveen Gangam)
---
.../metastore/TestHiveMetastoreTransformer.java | 27 ++++++++++++++++++
.../clientpositive/beeline/escape_comments.q.out | 4 +--
.../metastore/MetastoreDefaultTransformer.java | 32 +++++++++++++++-------
3 files changed, 51 insertions(+), 12 deletions(-)
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
index 27a1557..9eb7792 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/metastore/TestHiveMetastoreTransformer.java
@@ -1383,8 +1383,10 @@ public class TestHiveMetastoreTransformer {
resetHMSClient();
String dbName = "testdb";
+ String dbWithLocation = "dbWithLocation";
try {
silentDropDatabase(dbName);
+ silentDropDatabase(dbWithLocation);
} catch (Exception e) {
LOG.info("Drop database failed for " + dbName);
}
@@ -1417,6 +1419,31 @@ public class TestHiveMetastoreTransformer {
assertTrue("Database location expected to be external warehouse:actual=" + db.getLocationUri(),
db.getLocationUri().contains(conf.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname())));
resetHMSClient();
+
+ Warehouse wh = new Warehouse(conf);
+ String mgdPath = wh.getDefaultDatabasePath(dbWithLocation, false).toString();
+ new DatabaseBuilder()
+ .setName(dbWithLocation)
+ .setLocation(mgdPath)
+ .create(client, conf);
+
+ capabilities = new ArrayList<>();
+ capabilities.add("EXTWRITE");
+ setHMSClient("TestGetDatabaseWithLocation", (String[])(capabilities.toArray(new String[0])));
+
+ db = client.getDatabase(dbWithLocation);
+ assertTrue("Database location expected to be external warehouse:actual=" + db.getLocationUri(),
+ db.getLocationUri().contains(conf.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname())));
+ assertNull("Database managed location expected to be null", db.getManagedLocationUri());
+ resetHMSClient();
+
+ capabilities.add("HIVEMANAGEDINSERTWRITE");
+ setHMSClient("TestGetDatabaseWithLocation#2", (String[])(capabilities.toArray(new String[0])));
+
+ db = client.getDatabase(dbWithLocation);
+ assertTrue("Database location expected to be external warehouse", db.getLocationUri().contains(conf.get(MetastoreConf.ConfVars.WAREHOUSE_EXTERNAL.getVarname())));
+ assertEquals("Database managedLocationUri expected to be set to locationUri", mgdPath, db.getManagedLocationUri());
+ resetHMSClient();
} catch (Exception e) {
System.err.println(org.apache.hadoop.util.StringUtils.stringifyException(e));
System.err.println("testTransformerDatabase() failed.");
diff --git a/ql/src/test/results/clientpositive/beeline/escape_comments.q.out b/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
index 64b13f0..6627e25 100644
--- a/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
+++ b/ql/src/test/results/clientpositive/beeline/escape_comments.q.out
@@ -42,14 +42,14 @@ PREHOOK: Input: database:escape_comments_db
POSTHOOK: query: describe database extended escape_comments_db
POSTHOOK: type: DESCDATABASE
POSTHOOK: Input: database:escape_comments_db
-escape_comments_db a\nb location/in/test user USER
+#### A masked pattern was here ####
PREHOOK: query: describe database escape_comments_db
PREHOOK: type: DESCDATABASE
PREHOOK: Input: database:escape_comments_db
POSTHOOK: query: describe database escape_comments_db
POSTHOOK: type: DESCDATABASE
POSTHOOK: Input: database:escape_comments_db
-escape_comments_db a\nb location/in/test user USER
+#### A masked pattern was here ####
PREHOOK: query: show create table escape_comments_tbl1
PREHOOK: type: SHOW_CREATETABLE
PREHOOK: Input: escape_comments_db@escape_comments_tbl1
diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
index 0224ac7..bbd46d2 100644
--- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
+++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/MetastoreDefaultTransformer.java
@@ -684,16 +684,28 @@ public class MetastoreDefaultTransformer implements IMetaStoreMetadataTransforme
}
LOG.info("Starting translation for transformDatabase for processor " + processorId + " with " + processorCapabilities
- + " on database " + db.getName());
-
- if (!isTenantBasedStorage && (processorCapabilities == null || (!processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) &&
- !processorCapabilities.contains(HIVEFULLACIDWRITE)))) {
- LOG.info("Processor does not have any of ACID write capabilities, changing current location from " +
- db.getLocationUri() + " to external warehouse location");
- Path extWhLocation = hmsHandler.getWh().getDefaultExternalDatabasePath(db.getName());
- LOG.debug("Setting DBLocation to " + extWhLocation.toString());
- // TODO should not alter database now.
- db.setLocationUri(extWhLocation.toString());
+ + " on database {} locationUri={} managedLocationUri={}", db.getName(), db.getLocationUri(), db.getManagedLocationUri());
+
+ if (!isTenantBasedStorage) {
+ Path locationPath = Path.getPathWithoutSchemeAndAuthority(new Path(db.getLocationUri()));
+ Path whRootPath = Path.getPathWithoutSchemeAndAuthority(hmsHandler.getWh().getWhRoot());
+ if (FileUtils.isSubdirectory(whRootPath.toString(), locationPath.toString())) { // legacy path
+ if (processorCapabilities != null && (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) ||
+ processorCapabilities.contains(HIVEFULLACIDWRITE))) {
+ LOG.debug("Processor has atleast one of ACID write capabilities, setting current locationUri " + db.getLocationUri() + " as managedLocationUri");
+ db.setManagedLocationUri(new Path(db.getLocationUri()).toString());
+ }
+ Path extWhLocation = hmsHandler.getWh().getDefaultExternalDatabasePath(db.getName());
+ LOG.info("Database's location is a managed location, setting to a new default path based on external warehouse path:" + extWhLocation.toString());
+ db.setLocationUri(extWhLocation.toString());
+ } else {
+ if (processorCapabilities != null && (processorCapabilities.contains(HIVEMANAGEDINSERTWRITE) ||
+ processorCapabilities.contains(HIVEFULLACIDWRITE))) {
+ Path mgdWhLocation = hmsHandler.getWh().getDefaultDatabasePath(db.getName(), false);
+ LOG.debug("Processor has atleast one of ACID write capabilities, setting default managed path to " + mgdWhLocation.toString());
+ db.setManagedLocationUri(mgdWhLocation.toString());
+ }
+ }
}
LOG.info("Transformer returning database:" + db.toString());
return db;