You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by se...@apache.org on 2018/01/11 18:57:39 UTC
hive git commit: HIVE-18229 : add the unmanaged mapping command
(Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Repository: hive
Updated Branches:
refs/heads/master f0dc5c697 -> 7eac320ee
HIVE-18229 : add the unmanaged mapping command (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/7eac320e
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/7eac320e
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/7eac320e
Branch: refs/heads/master
Commit: 7eac320ee8bf6165fda43950c4bb8d765a208c81
Parents: f0dc5c6
Author: sergey <se...@apache.org>
Authored: Thu Jan 11 10:37:06 2018 -0800
Committer: sergey <se...@apache.org>
Committed: Thu Jan 11 10:37:06 2018 -0800
----------------------------------------------------------------------
.../upgrade/hive/hive-schema-3.0.0.hive.sql | 8 ++---
.../hive/ql/parse/DDLSemanticAnalyzer.java | 5 ++-
.../hadoop/hive/ql/parse/ResourcePlanParser.g | 10 +++---
.../test/queries/clientpositive/resourceplan.q | 5 +++
.../clientpositive/llap/resourceplan.q.out | 33 ++++++++++++++------
.../results/clientpositive/llap/sysdb.q.out | 20 ++++++------
.../hadoop/hive/metastore/ObjectStore.java | 10 ++++--
7 files changed, 59 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql
----------------------------------------------------------------------
diff --git a/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql b/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql
index 59bef52..82ad758 100644
--- a/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql
+++ b/metastore/scripts/upgrade/hive/hive-schema-3.0.0.hive.sql
@@ -1057,12 +1057,12 @@ TBLPROPERTIES (
WM_RESOURCEPLAN.NAME,
ENTITY_TYPE,
ENTITY_NAME,
- WM_POOL.PATH,
+ case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
ORDERING
-FROM
- WM_MAPPING
+FROM WM_MAPPING
JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID"
+LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+"
);
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index c413bc5..a1b4616 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -1254,7 +1254,10 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
String entityType = ast.getChild(1).getText();
String entityName = PlanUtils.stripQuotes(ast.getChild(2).getText());
WMMapping mapping = new WMMapping(rpName, entityType, entityName);
- mapping.setPoolPath(poolPath(ast.getChild(3)));
+ Tree dest = ast.getChild(3);
+ if (dest.getType() != HiveParser.TOK_UNMANAGED) {
+ mapping.setPoolPath(poolPath(dest));
+ } // Null path => unmanaged
if (ast.getChildCount() == 5) {
mapping.setOrdering(Integer.valueOf(ast.getChild(4).getText()));
}
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
index 990ec52..b47f7b3 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
@@ -70,6 +70,8 @@ withReplace : KW_WITH KW_REPLACE -> ^(TOK_REPLACE);
activate : KW_ACTIVATE withReplace? -> ^(TOK_ACTIVATE withReplace?);
enable : KW_ENABLE -> ^(TOK_ENABLE);
disable : KW_DISABLE -> ^(TOK_DISABLE);
+unmanaged : KW_UNMANAGED -> ^(TOK_UNMANAGED);
+
alterResourcePlanStatement
@init { gParent.pushMsg("alter resource plan statement", state); }
@@ -236,9 +238,9 @@ createMappingStatement
@after { gParent.popMsg(state); }
: (KW_CREATE mappingType=(KW_USER | KW_GROUP | KW_APPLICATION)
KW_MAPPING name=StringLiteral
- KW_IN rpName=identifier KW_TO poolPath
+ KW_IN rpName=identifier ((KW_TO path=poolPath) | unmanaged)
(KW_WITH KW_ORDER order=Number)?)
- -> ^(TOK_CREATE_MAPPING $rpName $mappingType $name poolPath $order?)
+ -> ^(TOK_CREATE_MAPPING $rpName $mappingType $name $path? unmanaged? $order?)
;
alterMappingStatement
@@ -246,9 +248,9 @@ alterMappingStatement
@after { gParent.popMsg(state); }
: (KW_ALTER mappingType=(KW_USER | KW_GROUP | KW_APPLICATION) KW_MAPPING
KW_MAPPING name=StringLiteral
- KW_IN rpName=identifier KW_TO poolPath
+ KW_IN rpName=identifier ((KW_TO path=poolPath) | unmanaged)
(KW_WITH KW_ORDER order=Number)?)
- -> ^(TOK_ALTER_MAPPING $rpName $mappingType $name poolPath $order?)
+ -> ^(TOK_ALTER_MAPPING $rpName $mappingType $name $path? unmanaged? $order?)
;
dropMappingStatement
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/ql/src/test/queries/clientpositive/resourceplan.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/resourceplan.q b/ql/src/test/queries/clientpositive/resourceplan.q
index 9c4feb5..b46c1ee 100644
--- a/ql/src/test/queries/clientpositive/resourceplan.q
+++ b/ql/src/test/queries/clientpositive/resourceplan.q
@@ -318,6 +318,10 @@ CREATE USER MAPPING 'user2' IN plan_2 TO def WITH ORDER 1;
CREATE GROUP MAPPING "group1" IN plan_2 TO def.c1;
CREATE APPLICATION MAPPING "app1" IN plan_2 TO def.c1;
CREATE GROUP MAPPING 'group2' IN plan_2 TO def.c2 WITH ORDER 1;
+CREATE GROUP MAPPING 'group3' IN plan_2 UNMANAGED WITH ORDER 1;
+ALTER USER MAPPING "user1" IN plan_2 UNMANAGED;
+
+
SELECT * FROM SYS.WM_MAPPINGS;
-- Drop pool failed, pool in use.
@@ -325,6 +329,7 @@ DROP POOL plan_2.def.c1;
DROP USER MAPPING "user2" in plan_2;
DROP GROUP MAPPING "group2" in plan_2;
+DROP GROUP MAPPING "group3" in plan_2;
DROP APPLICATION MAPPING "app1" in plan_2;
SELECT * FROM SYS.WM_MAPPINGS;
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/ql/src/test/results/clientpositive/llap/resourceplan.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
index e40994f..df87fba 100644
--- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out
+++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
@@ -2344,12 +2344,12 @@ TBLPROPERTIES (
WM_RESOURCEPLAN.NAME,
ENTITY_TYPE,
ENTITY_NAME,
- WM_POOL.PATH,
+ case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
ORDERING
-FROM
- WM_MAPPING
+FROM WM_MAPPING
JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID"
+LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+"
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@WM_MAPPINGS
@@ -2369,12 +2369,12 @@ TBLPROPERTIES (
WM_RESOURCEPLAN.NAME,
ENTITY_TYPE,
ENTITY_NAME,
- WM_POOL.PATH,
+ case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
ORDERING
-FROM
- WM_MAPPING
+FROM WM_MAPPING
JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID"
+LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+"
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@WM_MAPPINGS
@@ -4044,6 +4044,14 @@ PREHOOK: query: CREATE GROUP MAPPING 'group2' IN plan_2 TO def.c2 WITH ORDER 1
PREHOOK: type: CREATE MAPPING
POSTHOOK: query: CREATE GROUP MAPPING 'group2' IN plan_2 TO def.c2 WITH ORDER 1
POSTHOOK: type: CREATE MAPPING
+PREHOOK: query: CREATE GROUP MAPPING 'group3' IN plan_2 UNMANAGED WITH ORDER 1
+PREHOOK: type: CREATE MAPPING
+POSTHOOK: query: CREATE GROUP MAPPING 'group3' IN plan_2 UNMANAGED WITH ORDER 1
+POSTHOOK: type: CREATE MAPPING
+PREHOOK: query: ALTER USER MAPPING "user1" IN plan_2 UNMANAGED
+PREHOOK: type: ALTER MAPPING
+POSTHOOK: query: ALTER USER MAPPING "user1" IN plan_2 UNMANAGED
+POSTHOOK: type: ALTER MAPPING
PREHOOK: query: SELECT * FROM SYS.WM_MAPPINGS
PREHOOK: type: QUERY
PREHOOK: Input: sys@wm_mappings
@@ -4055,7 +4063,8 @@ POSTHOOK: Input: sys@wm_mappings
plan_2 APPLICATION app1 def.c1 0
plan_2 GROUP group1 def.c1 0
plan_2 GROUP group2 def.c2 1
-plan_2 USER user1 def 0
+plan_2 GROUP group3 <unmanaged> 1
+plan_2 USER user1 <unmanaged> 0
plan_2 USER user2 def 1
PREHOOK: query: DROP POOL plan_2.def.c1
PREHOOK: type: DROP POOL
@@ -4068,6 +4077,10 @@ PREHOOK: query: DROP GROUP MAPPING "group2" in plan_2
PREHOOK: type: DROP MAPPING
POSTHOOK: query: DROP GROUP MAPPING "group2" in plan_2
POSTHOOK: type: DROP MAPPING
+PREHOOK: query: DROP GROUP MAPPING "group3" in plan_2
+PREHOOK: type: DROP MAPPING
+POSTHOOK: query: DROP GROUP MAPPING "group3" in plan_2
+POSTHOOK: type: DROP MAPPING
PREHOOK: query: DROP APPLICATION MAPPING "app1" in plan_2
PREHOOK: type: DROP MAPPING
POSTHOOK: query: DROP APPLICATION MAPPING "app1" in plan_2
@@ -4081,7 +4094,7 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: sys@wm_mappings
#### A masked pattern was here ####
plan_2 GROUP group1 def.c1 0
-plan_2 USER user1 def 0
+plan_2 USER user1 <unmanaged> 0
PREHOOK: query: CREATE RESOURCE PLAN plan_4
PREHOOK: type: CREATE RESOURCEPLAN
POSTHOOK: query: CREATE RESOURCE PLAN plan_4
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/ql/src/test/results/clientpositive/llap/sysdb.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/sysdb.q.out b/ql/src/test/results/clientpositive/llap/sysdb.q.out
index 3bd407b..5ed427f 100644
--- a/ql/src/test/results/clientpositive/llap/sysdb.q.out
+++ b/ql/src/test/results/clientpositive/llap/sysdb.q.out
@@ -2412,12 +2412,12 @@ TBLPROPERTIES (
WM_RESOURCEPLAN.NAME,
ENTITY_TYPE,
ENTITY_NAME,
- WM_POOL.PATH,
+ case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
ORDERING
-FROM
- WM_MAPPING
+FROM WM_MAPPING
JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID"
+LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+"
)
PREHOOK: type: CREATETABLE
PREHOOK: Output: SYS@WM_MAPPINGS
@@ -2437,12 +2437,12 @@ TBLPROPERTIES (
WM_RESOURCEPLAN.NAME,
ENTITY_TYPE,
ENTITY_NAME,
- WM_POOL.PATH,
+ case when WM_POOL.PATH is null then '<unmanaged>' else WM_POOL.PATH end,
ORDERING
-FROM
- WM_MAPPING
+FROM WM_MAPPING
JOIN WM_RESOURCEPLAN ON WM_MAPPING.RP_ID = WM_RESOURCEPLAN.RP_ID
-LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID"
+LEFT OUTER JOIN WM_POOL ON WM_POOL.POOL_ID = WM_MAPPING.POOL_ID
+"
)
POSTHOOK: type: CREATETABLE
POSTHOOK: Output: SYS@WM_MAPPINGS
@@ -3227,7 +3227,7 @@ POSTHOOK: query: select count(*) from cds
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@cds
#### A masked pattern was here ####
-70
+71
PREHOOK: query: select column_name, type_name, integer_idx from columns_v2 order by column_name, integer_idx limit 5
PREHOOK: type: QUERY
PREHOOK: Input: sys@columns_v2
@@ -3381,7 +3381,7 @@ POSTHOOK: query: select count(*) from sds
POSTHOOK: type: QUERY
POSTHOOK: Input: sys@sds
#### A masked pattern was here ####
-76
+77
PREHOOK: query: select param_key, param_value from sd_params order by param_key, param_value limit 5
PREHOOK: type: QUERY
PREHOOK: Input: sys@sd_params
http://git-wip-us.apache.org/repos/asf/hive/blob/7eac320e/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
----------------------------------------------------------------------
diff --git a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
index 5bfa624..c9ff295 100644
--- a/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
+++ b/standalone-metastore/src/main/java/org/apache/hadoop/hive/metastore/ObjectStore.java
@@ -10456,18 +10456,22 @@ public class ObjectStore implements RawStore, Configurable {
try {
openTransaction();
MWMResourcePlan resourcePlan = getMWMResourcePlan(mapping.getResourcePlanName(), true);
- MWMPool pool = getPool(resourcePlan, mapping.getPoolPath());
+ MWMPool pool = null;
+ if (mapping.isSetPoolPath()) {
+ pool = getPool(resourcePlan, mapping.getPoolPath());
+ }
if (!update) {
MWMMapping mMapping = new MWMMapping(resourcePlan, entityType, entityName, pool,
mapping.getOrdering());
pm.makePersistent(mMapping);
} else {
- query = pm.newQuery(MWMPool.class, "resourcePlan == rp && entityType == type " +
+ query = pm.newQuery(MWMMapping.class, "resourcePlan == rp && entityType == type " +
"&& entityName == name");
query.declareParameters(
"MWMResourcePlan rp, java.lang.String type, java.lang.String name");
query.setUnique(true);
- MWMMapping mMapping = (MWMMapping) query.execute(resourcePlan, entityType, entityName);
+ MWMMapping mMapping = (MWMMapping) query.execute(
+ resourcePlan, entityType.toString(), entityName);
mMapping.setPool(pool);
}
commited = commitTransaction();