You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by we...@apache.org on 2017/02/20 02:17:22 UTC
hive git commit: HIVE-15891 : Detect query rewrite scenario for
UPDATE/DELETE/MERGE and fail fast (Wei Zheng, reviewed by Eugene Koifman)
Repository: hive
Updated Branches:
refs/heads/master 1e00fb357 -> 091ac8e05
HIVE-15891 : Detect query rewrite scenario for UPDATE/DELETE/MERGE and fail fast (Wei Zheng, reviewed by Eugene Koifman)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/091ac8e0
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/091ac8e0
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/091ac8e0
Branch: refs/heads/master
Commit: 091ac8e05940ce327d068cee77817bbe428797e2
Parents: 1e00fb3
Author: Wei Zheng <we...@apache.org>
Authored: Sun Feb 19 18:17:10 2017 -0800
Committer: Wei Zheng <we...@apache.org>
Committed: Sun Feb 19 18:17:10 2017 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/hive/ql/Context.java | 12 +++++++++++
.../org/apache/hadoop/hive/ql/ErrorMsg.java | 3 +++
.../hadoop/hive/ql/parse/SemanticAnalyzer.java | 5 +++++
.../ql/parse/UpdateDeleteSemanticAnalyzer.java | 1 +
.../clientnegative/masking_acid_delete.q | 10 ++++++++++
.../queries/clientnegative/masking_acid_merge.q | 15 ++++++++++++++
.../clientnegative/masking_acid_update.q | 10 ++++++++++
.../clientnegative/masking_acid_delete.q.out | 13 ++++++++++++
.../clientnegative/masking_acid_merge.q.out | 21 ++++++++++++++++++++
.../clientnegative/masking_acid_update.q.out | 13 ++++++++++++
10 files changed, 103 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/java/org/apache/hadoop/hive/ql/Context.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/Context.java b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
index fae2a12..d1d2789 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/Context.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/Context.java
@@ -125,6 +125,10 @@ public class Context {
private Heartbeater heartbeater;
private boolean skipTableMasking;
+
+ // Identify whether the query involves an UPDATE, DELETE or MERGE
+ private boolean isUpdateDeleteMerge;
+
/**
* This determines the prefix of the
* {@link org.apache.hadoop.hive.ql.parse.SemanticAnalyzer.Phase1Ctx#dest}
@@ -955,4 +959,12 @@ public class Context {
opContext = new CompilationOpContext();
sequencer = new AtomicInteger();
}
+
+ public boolean getIsUpdateDeleteMerge() {
+ return isUpdateDeleteMerge;
+ }
+
+ public void setIsUpdateDeleteMerge(boolean isUpdate) {
+ this.isUpdateDeleteMerge = isUpdate;
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
index 6013218..2ffc130 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/ErrorMsg.java
@@ -398,6 +398,9 @@ public enum ErrorMsg {
"time."),
DISTINCT_NOT_SUPPORTED(10285, "Distinct keyword is not support in current context"),
NONACID_COMPACTION_NOT_SUPPORTED(10286, "Compaction is not allowed on non-ACID table {0}.{1}", true),
+ MASKING_FILTERING_ON_ACID_NOT_SUPPORTED(10287,
+ "Detected {0}.{1} has row masking/column filtering enabled, " +
+ "which is not supported for query involving ACID operations", true),
UPDATEDELETE_PARSE_ERROR(10290, "Encountered parse error while parsing rewritten merge/update or " +
"delete query"),
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
index f053093..9c37af8 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
@@ -10820,6 +10820,11 @@ public class SemanticAnalyzer extends BaseSemanticAnalyzer {
.applyRowFilterAndColumnMasking(basicPrivObjs);
if (needRewritePrivObjs != null && !needRewritePrivObjs.isEmpty()) {
for (HivePrivilegeObject privObj : needRewritePrivObjs) {
+ // We don't support masking/filtering against ACID query at the moment
+ if (ctx.getIsUpdateDeleteMerge()) {
+ throw new SemanticException(ErrorMsg.MASKING_FILTERING_ON_ACID_NOT_SUPPORTED,
+ privObj.getDbname(), privObj.getObjectName());
+ }
MaskAndFilterInfo info = basicInfos.get(privObj);
String replacementText = tableMask.create(privObj, info);
if (replacementText != null) {
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
index 725f2ce..865c03a 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/UpdateDeleteSemanticAnalyzer.java
@@ -293,6 +293,7 @@ public class UpdateDeleteSemanticAnalyzer extends SemanticAnalyzer {
HiveConf.setVar(conf, HiveConf.ConfVars.DYNAMICPARTITIONINGMODE, "nonstrict");
rewrittenCtx = new Context(conf);
rewrittenCtx.setExplainConfig(ctx.getExplainConfig());
+ rewrittenCtx.setIsUpdateDeleteMerge(true);
} catch (IOException e) {
throw new SemanticException(ErrorMsg.UPDATEDELETE_IO_ERROR.getMsg());
}
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/test/queries/clientnegative/masking_acid_delete.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/masking_acid_delete.q b/ql/src/test/queries/clientnegative/masking_acid_delete.q
new file mode 100644
index 0000000..2785d7a
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/masking_acid_delete.q
@@ -0,0 +1,10 @@
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true");
+
+delete from masking_test where value='ddd';
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/test/queries/clientnegative/masking_acid_merge.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/masking_acid_merge.q b/ql/src/test/queries/clientnegative/masking_acid_merge.q
new file mode 100644
index 0000000..50471e0
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/masking_acid_merge.q
@@ -0,0 +1,15 @@
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+create table nonacid (key int, value string) stored as orc;
+
+create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true");
+
+MERGE INTO masking_test as t using nonacid as s ON t.key = s.key
+WHEN MATCHED AND s.key < 5 THEN DELETE
+WHEN MATCHED AND s.key < 3 THEN UPDATE set key = 1
+WHEN NOT MATCHED THEN INSERT VALUES (s.key, s.value);
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/test/queries/clientnegative/masking_acid_update.q
----------------------------------------------------------------------
diff --git a/ql/src/test/queries/clientnegative/masking_acid_update.q b/ql/src/test/queries/clientnegative/masking_acid_update.q
new file mode 100644
index 0000000..586d20b
--- /dev/null
+++ b/ql/src/test/queries/clientnegative/masking_acid_update.q
@@ -0,0 +1,10 @@
+set hive.mapred.mode=nonstrict;
+set hive.security.authorization.manager=org.apache.hadoop.hive.ql.security.authorization.plugin.sqlstd.SQLStdHiveAuthorizerFactoryForTest;
+set hive.support.concurrency=true;
+set hive.txn.manager=org.apache.hadoop.hive.ql.lockmgr.DbTxnManager;
+
+create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true");
+
+update masking_test set key=1 where value='ddd';
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/test/results/clientnegative/masking_acid_delete.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/masking_acid_delete.q.out b/ql/src/test/results/clientnegative/masking_acid_delete.q.out
new file mode 100644
index 0000000..021f2cd
--- /dev/null
+++ b/ql/src/test/results/clientnegative/masking_acid_delete.q.out
@@ -0,0 +1,13 @@
+PREHOOK: query: create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test
+POSTHOOK: query: create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test
+FAILED: SemanticException [Error 10287]: Detected default.masking_test has row masking/column filtering enabled, which is not supported for query involving ACID operations
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/test/results/clientnegative/masking_acid_merge.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/masking_acid_merge.q.out b/ql/src/test/results/clientnegative/masking_acid_merge.q.out
new file mode 100644
index 0000000..917c44a
--- /dev/null
+++ b/ql/src/test/results/clientnegative/masking_acid_merge.q.out
@@ -0,0 +1,21 @@
+PREHOOK: query: create table nonacid (key int, value string) stored as orc
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@nonacid
+POSTHOOK: query: create table nonacid (key int, value string) stored as orc
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@nonacid
+PREHOOK: query: create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test
+POSTHOOK: query: create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test
+FAILED: SemanticException [Error 10287]: Detected default.masking_test has row masking/column filtering enabled, which is not supported for query involving ACID operations
http://git-wip-us.apache.org/repos/asf/hive/blob/091ac8e0/ql/src/test/results/clientnegative/masking_acid_update.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/masking_acid_update.q.out b/ql/src/test/results/clientnegative/masking_acid_update.q.out
new file mode 100644
index 0000000..021f2cd
--- /dev/null
+++ b/ql/src/test/results/clientnegative/masking_acid_update.q.out
@@ -0,0 +1,13 @@
+PREHOOK: query: create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true")
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+PREHOOK: Output: default@masking_test
+POSTHOOK: query: create table masking_test (key int, value string)
+clustered by (value) into 2 buckets stored as orc
+tblproperties ("transactional"="true")
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@masking_test
+FAILED: SemanticException [Error 10287]: Detected default.masking_test has row masking/column filtering enabled, which is not supported for query involving ACID operations