You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by ha...@apache.org on 2018/08/27 21:02:04 UTC
hive git commit: HIVE-20467 : Allow IF NOT EXISTS/IF EXISTS in
Resource plan creation/drop (Miklos Gergely via Sergey Shelukhin)
Repository: hive
Updated Branches:
refs/heads/master 824cd1731 -> 06fb30235
HIVE-20467 : Allow IF NOT EXISTS/IF EXISTS in Resource plan creation/drop (Miklos Gergely via Sergey Shelukhin)
Signed-off-by: Ashutosh Chauhan <ha...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/06fb3023
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/06fb3023
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/06fb3023
Branch: refs/heads/master
Commit: 06fb30235e8aa9929ba9944055163af11eee559b
Parents: 824cd17
Author: Miklos Gergely <mg...@hortonworks.com>
Authored: Mon Aug 27 14:01:09 2018 -0700
Committer: Ashutosh Chauhan <ha...@apache.org>
Committed: Mon Aug 27 14:01:09 2018 -0700
----------------------------------------------------------------------
.../hive/hcatalog/templeton/JsonBuilder.java | 2 +-
.../java/org/apache/hadoop/hive/ql/ErrorMsg.java | 5 +++--
.../org/apache/hadoop/hive/ql/exec/DDLTask.java | 6 +++---
.../org/apache/hadoop/hive/ql/metadata/Hive.java | 12 ++++++++++--
.../hadoop/hive/ql/parse/DDLSemanticAnalyzer.java | 18 ++++++++++++++++--
.../hadoop/hive/ql/parse/ResourcePlanParser.g | 8 ++++----
.../hive/ql/plan/CreateResourcePlanDesc.java | 11 ++++++++++-
.../hadoop/hive/ql/plan/DropResourcePlanDesc.java | 14 +++++++++++++-
ql/src/test/queries/clientpositive/resourceplan.q | 10 ++++++++++
.../clientpositive/llap/resourceplan.q.out | 18 ++++++++++++++++++
10 files changed, 88 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JsonBuilder.java
----------------------------------------------------------------------
diff --git a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JsonBuilder.java b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JsonBuilder.java
index 6a38719..dfcdaef 100644
--- a/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JsonBuilder.java
+++ b/hcatalog/webhcat/svr/src/main/java/org/apache/hive/hcatalog/templeton/JsonBuilder.java
@@ -49,7 +49,7 @@ public class JsonBuilder {
hiveError2HttpStatusCode.put(ErrorMsg.INVALID_PARTITION.getErrorCode(), HttpStatus.NOT_FOUND_404);
hiveError2HttpStatusCode.put(ErrorMsg.DUPLICATE_COLUMN_NAMES.getErrorCode(), HttpStatus.CONFLICT_409);
- hiveError2HttpStatusCode.put(ErrorMsg.DATABSAE_ALREADY_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
+ hiveError2HttpStatusCode.put(ErrorMsg.DATABASE_ALREADY_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
hiveError2HttpStatusCode.put(ErrorMsg.PARTITION_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
hiveError2HttpStatusCode.put(ErrorMsg.TABLE_ALREADY_EXISTS.getErrorCode(), HttpStatus.CONFLICT_409);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 0d74671..e7d7159 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -334,7 +334,7 @@ public enum ErrorMsg {
"A column on which a partition/table is list bucketed cannot be truncated."),
TABLE_NOT_PARTITIONED(10241, "Table {0} is not a partitioned table", true),
- DATABSAE_ALREADY_EXISTS(10242, "Database {0} already exists", true),
+ DATABASE_ALREADY_EXISTS(10242, "Database {0} already exists", true),
CANNOT_REPLACE_COLUMNS(10243, "Replace columns is not supported for table {0}. SerDe may be incompatible.", true),
BAD_LOCATION_VALUE(10244, "{0} is not absolute. Please specify a complete absolute uri."),
UNSUPPORTED_ALTER_TBL_OP(10245, "{0} alter table options is not supported"),
@@ -467,7 +467,8 @@ public enum ErrorMsg {
"insert-only transactional", true),
LOAD_DATA_LAUNCH_JOB_IO_ERROR(10415, "Encountered I/O error while parsing rewritten load data into insert query"),
LOAD_DATA_LAUNCH_JOB_PARSE_ERROR(10416, "Encountered parse error while parsing rewritten load data into insert query"),
-
+ RESOURCE_PLAN_ALREADY_EXISTS(10417, "Resource plan {0} already exists", true),
+ RESOURCE_PLAN_NOT_EXISTS(10418, "Resource plan {0} does not exist", true),
//========================== 20000 range starts here ========================//
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
index 58c3ae1..d620243 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/DDLTask.java
@@ -672,7 +672,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
private int createResourcePlan(Hive db, CreateResourcePlanDesc createResourcePlanDesc)
throws HiveException {
db.createResourcePlan(createResourcePlanDesc.getResourcePlan(),
- createResourcePlanDesc.getCopyFromName());
+ createResourcePlanDesc.getCopyFromName(), createResourcePlanDesc.getIfNotExists());
return 0;
}
@@ -782,7 +782,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
private int dropResourcePlan(Hive db, DropResourcePlanDesc desc) throws HiveException {
- db.dropResourcePlan(desc.getRpName());
+ db.dropResourcePlan(desc.getRpName(), desc.getIfExists());
return 0;
}
@@ -4839,7 +4839,7 @@ public class DDLTask extends Task<DDLWork> implements Serializable {
}
catch (AlreadyExistsException ex) {
//it would be better if AlreadyExistsException had an errorCode field....
- throw new HiveException(ex, ErrorMsg.DATABSAE_ALREADY_EXISTS, crtDb.getName());
+ throw new HiveException(ex, ErrorMsg.DATABASE_ALREADY_EXISTS, crtDb.getName());
}
return 0;
}
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
index 36dc694..08a4506 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
@@ -5341,10 +5341,14 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
- public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName)
+ public void createResourcePlan(WMResourcePlan resourcePlan, String copyFromName, boolean ifNotExists)
throws HiveException {
try {
getMSC().createResourcePlan(resourcePlan, copyFromName);
+ } catch (AlreadyExistsException e) {
+ if (!ifNotExists) {
+ throw new HiveException(e, ErrorMsg.RESOURCE_PLAN_ALREADY_EXISTS, resourcePlan.getName());
+ }
} catch (Exception e) {
throw new HiveException(e);
}
@@ -5368,9 +5372,13 @@ private void constructOneLBLocationMap(FileStatus fSta,
}
}
- public void dropResourcePlan(String rpName) throws HiveException {
+ public void dropResourcePlan(String rpName, boolean ifExists) throws HiveException {
try {
getMSC().dropResourcePlan(rpName);
+ } catch (NoSuchObjectException e) {
+ if (!ifExists) {
+ throw new HiveException(e, ErrorMsg.RESOURCE_PLAN_NOT_EXISTS, rpName);
+ }
} catch (Exception e) {
throw new HiveException(e);
}
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
index 273fccb..412fca2 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
@@ -905,6 +905,7 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
String resourcePlanName = unescapeIdentifier(ast.getChild(0).getText());
Integer queryParallelism = null;
String likeName = null;
+ boolean ifNotExists = false;
for (int i = 1; i < ast.getChildCount(); ++i) {
Tree child = ast.getChild(i);
switch (child.getType()) {
@@ -923,11 +924,14 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException("Conflicting create arguments " + ast.toStringTree());
}
break;
+ case HiveParser.TOK_IFNOTEXISTS:
+ ifNotExists = true;
+ break;
default: throw new SemanticException("Invalid create arguments " + ast.toStringTree());
}
}
CreateResourcePlanDesc desc = new CreateResourcePlanDesc(
- resourcePlanName, queryParallelism, likeName);
+ resourcePlanName, queryParallelism, likeName, ifNotExists);
addServiceOutput();
rootTasks.add(TaskFactory.get(new DDLWork(getInputs(), getOutputs(), desc)));
}
@@ -1070,7 +1074,17 @@ public class DDLSemanticAnalyzer extends BaseSemanticAnalyzer {
throw new SemanticException("Expected name in DROP RESOURCE PLAN statement");
}
String rpName = unescapeIdentifier(ast.getChild(0).getText());
- DropResourcePlanDesc desc = new DropResourcePlanDesc(rpName);
+ boolean ifExists = false;
+ for (int i = 1; i < ast.getChildCount(); ++i) {
+ Tree child = ast.getChild(i);
+ switch (child.getType()) {
+ case HiveParser.TOK_IFEXISTS:
+ ifExists = true;
+ break;
+ default: throw new SemanticException("Invalid create arguments " + ast.toStringTree());
+ }
+ }
+ DropResourcePlanDesc desc = new DropResourcePlanDesc(rpName, ifExists);
addServiceOutput();
rootTasks.add(TaskFactory.get(
new DDLWork(getInputs(), getOutputs(), desc)));
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g b/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
index f8c47f9..0479c78 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ResourcePlanParser.g
@@ -93,9 +93,9 @@ rpUnassignList
createResourcePlanStatement
@init { gParent.pushMsg("create resource plan statement", state); }
@after { gParent.popMsg(state); }
- : KW_CREATE KW_RESOURCE KW_PLAN (
- (name=identifier KW_LIKE likeName=identifier -> ^(TOK_CREATE_RP $name ^(TOK_LIKERP $likeName)))
- | (name=identifier (KW_WITH rpAssignList)? -> ^(TOK_CREATE_RP $name rpAssignList?))
+ : KW_CREATE KW_RESOURCE KW_PLAN ifNotExists? (
+ (name=identifier KW_LIKE likeName=identifier -> ^(TOK_CREATE_RP $name ifNotExists? ^(TOK_LIKERP $likeName)))
+ | (name=identifier (KW_WITH rpAssignList)? -> ^(TOK_CREATE_RP $name ifNotExists? rpAssignList?))
)
;
@@ -140,7 +140,7 @@ replaceResourcePlanStatement
dropResourcePlanStatement
@init { gParent.pushMsg("drop resource plan statement", state); }
@after { gParent.popMsg(state); }
- : KW_DROP KW_RESOURCE KW_PLAN name=identifier -> ^(TOK_DROP_RP $name)
+ : KW_DROP KW_RESOURCE KW_PLAN ifExists? name=identifier -> ^(TOK_DROP_RP $name ifExists?)
;
poolPath
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateResourcePlanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateResourcePlanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateResourcePlanDesc.java
index f645aa2..9c18f59 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateResourcePlanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/CreateResourcePlanDesc.java
@@ -28,17 +28,20 @@ public class CreateResourcePlanDesc extends DDLDesc implements Serializable {
private WMResourcePlan resourcePlan;
private String copyFromName;
+ private boolean ifNotExists;
// For serialization only.
public CreateResourcePlanDesc() {
}
- public CreateResourcePlanDesc(String planName, Integer queryParallelism, String copyFromName) {
+ public CreateResourcePlanDesc(String planName, Integer queryParallelism, String copyFromName,
+ boolean ifNotExists) {
resourcePlan = new WMResourcePlan(planName);
if (queryParallelism != null) {
resourcePlan.setQueryParallelism(queryParallelism);
}
this.copyFromName = copyFromName;
+ this.ifNotExists = ifNotExists;
}
@Explain(displayName="resourcePlan", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
@@ -50,4 +53,10 @@ public class CreateResourcePlanDesc extends DDLDesc implements Serializable {
public String getCopyFromName() {
return copyFromName;
}
+
+ @Explain(displayName="If not exists", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED },
+ displayOnlyOnTrue = true)
+ public boolean getIfNotExists() {
+ return ifNotExists;
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/java/org/apache/hadoop/hive/ql/plan/DropResourcePlanDesc.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/plan/DropResourcePlanDesc.java b/ql/src/java/org/apache/hadoop/hive/ql/plan/DropResourcePlanDesc.java
index b67472d..efaf078 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/plan/DropResourcePlanDesc.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/plan/DropResourcePlanDesc.java
@@ -27,11 +27,13 @@ public class DropResourcePlanDesc extends DDLDesc implements Serializable {
private static final long serialVersionUID = 1258596919510047766L;
private String rpName;
+ private boolean ifExists;
public DropResourcePlanDesc() {}
- public DropResourcePlanDesc(String rpName) {
+ public DropResourcePlanDesc(String rpName, boolean ifExists) {
this.setRpName(rpName);
+ this.setIfExists(ifExists);
}
@Explain(displayName="resourcePlanName", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED })
@@ -39,8 +41,18 @@ public class DropResourcePlanDesc extends DDLDesc implements Serializable {
return rpName;
}
+ @Explain(displayName="ifExists", explainLevels = { Level.USER, Level.DEFAULT, Level.EXTENDED },
+ displayOnlyOnTrue = true)
+ public boolean getIfExists() {
+ return ifExists;
+ }
+
public void setRpName(String rpName) {
this.rpName = rpName;
}
+ public void setIfExists(boolean ifExists) {
+ this.ifExists = ifExists;
+ }
+
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/test/queries/clientpositive/resourceplan.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientpositive/resourceplan.q b/ql/src/test/queries/clientpositive/resourceplan.q
index 5658b99..fae9701 100644
--- a/ql/src/test/queries/clientpositive/resourceplan.q
+++ b/ql/src/test/queries/clientpositive/resourceplan.q
@@ -35,6 +35,11 @@ SHOW RESOURCE PLANS;
SHOW RESOURCE PLAN plan_2;
SELECT * FROM SYS.WM_RESOURCEPLANS;
+-- Create plan with existing name, should fail
+CREATE RESOURCE PLAN plan_2;
+-- Create plan with existing name with IF NOT EXISTS
+CREATE RESOURCE PLAN IF NOT EXISTS plan_2;
+
-- Should fail cannot set pool in create.
CREATE RESOURCE PLAN plan_3 WITH QUERY_PARALLELISM=5, DEFAULT POOL = `all`;
@@ -139,6 +144,11 @@ DROP RESOURCE PLAN plan_2;
DROP RESOURCE PLAN plan_3;
SELECT * FROM SYS.WM_RESOURCEPLANS;
+-- Drop non existing resource plan, should fail
+DROP RESOURCE PLAN plan_99999;
+-- Drop non existing resource plan with IF EXISTS
+DROP RESOURCE PLAN IF EXISTS plan_99999;
+
-- Use reserved keyword table as name.
CREATE RESOURCE PLAN `table`;
ALTER RESOURCE PLAN `table` SET QUERY_PARALLELISM = 1;
http://git-wip-us.apache.org/repos/asf/hive/blob/06fb3023/ql/src/test/results/clientpositive/llap/resourceplan.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientpositive/llap/resourceplan.q.out b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
index 3933273..9e8dd1c 100644
--- a/ql/src/test/results/clientpositive/llap/resourceplan.q.out
+++ b/ql/src/test/results/clientpositive/llap/resourceplan.q.out
@@ -3263,6 +3263,15 @@ POSTHOOK: Input: sys@wm_resourceplans
#### A masked pattern was here ####
plan_1 DISABLED 0 default
plan_2 DISABLED 10 default
+PREHOOK: query: CREATE RESOURCE PLAN plan_2
+PREHOOK: type: CREATE RESOURCEPLAN
+PREHOOK: Output: dummyHostnameForTest
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Resource plan plan_2 already exists
+PREHOOK: query: CREATE RESOURCE PLAN IF NOT EXISTS plan_2
+PREHOOK: type: CREATE RESOURCEPLAN
+PREHOOK: Output: dummyHostnameForTest
+POSTHOOK: query: CREATE RESOURCE PLAN IF NOT EXISTS plan_2
+POSTHOOK: type: CREATE RESOURCEPLAN
FAILED: SemanticException Invalid create arguments (tok_create_rp plan_3 (tok_query_parallelism 5) (tok_default_pool all))
PREHOOK: query: ALTER RESOURCE PLAN plan_1 RENAME TO plan_2
PREHOOK: type: ALTER RESOURCEPLAN
@@ -3580,6 +3589,15 @@ POSTHOOK: type: QUERY
POSTHOOK: Input: sys@wm_resourceplans
#### A masked pattern was here ####
plan_2 ACTIVE 10 default
+PREHOOK: query: DROP RESOURCE PLAN plan_99999
+PREHOOK: type: DROP RESOURCEPLAN
+PREHOOK: Output: dummyHostnameForTest
+FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.DDLTask. Resource plan plan_99999 does not exist
+PREHOOK: query: DROP RESOURCE PLAN IF EXISTS plan_99999
+PREHOOK: type: DROP RESOURCEPLAN
+PREHOOK: Output: dummyHostnameForTest
+POSTHOOK: query: DROP RESOURCE PLAN IF EXISTS plan_99999
+POSTHOOK: type: DROP RESOURCEPLAN
PREHOOK: query: CREATE RESOURCE PLAN `table`
PREHOOK: type: CREATE RESOURCEPLAN
PREHOOK: Output: dummyHostnameForTest