You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hive.apache.org by da...@apache.org on 2017/08/04 00:40:21 UTC
hive git commit: HIVE-17208: Repl dump should pass in db/table
information to authorization API (Daniel Dai, reviewed by Thejas Nair)
Repository: hive
Updated Branches:
refs/heads/master 0bf831461 -> ceec58314
HIVE-17208: Repl dump should pass in db/table information to authorization API (Daniel Dai, reviewed by Thejas Nair)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo
Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/ceec5831
Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/ceec5831
Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/ceec5831
Branch: refs/heads/master
Commit: ceec583142eb3e2c8f1cca8086e3f23bfebe50dd
Parents: 0bf8314
Author: Daniel Dai <da...@hortonworks.com>
Authored: Thu Aug 3 17:40:09 2017 -0700
Committer: Daniel Dai <da...@hortonworks.com>
Committed: Thu Aug 3 17:40:09 2017 -0700
----------------------------------------------------------------------
.../TestHiveAuthorizerCheckInvocation.java | 38 ++++++++++++++++++++
.../hadoop/hive/ql/exec/repl/ReplDumpTask.java | 31 ++--------------
.../ql/parse/ReplicationSemanticAnalyzer.java | 14 ++++++++
.../hadoop/hive/ql/parse/repl/dump/Utils.java | 27 ++++++++++++++
.../repl_dump_requires_admin.q.out | 2 ++
.../repl_load_requires_admin.q.out | 2 ++
6 files changed, 85 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hive/blob/ceec5831/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
----------------------------------------------------------------------
diff --git a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
index f973f8d..456e80d 100644
--- a/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
+++ b/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java
@@ -50,6 +50,10 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+
+import static org.mockito.Mockito.when;
/**
* Test HiveAuthorizer api invocation
@@ -390,6 +394,40 @@ public class TestHiveAuthorizerCheckInvocation {
assertEquals(dbName.toLowerCase(), dbObj.getDbname().toLowerCase());
}
+ private void resetAuthorizer() throws HiveAuthzPluginException, HiveAccessControlException {
+ reset(mockedAuthorizer);
+ // HiveAuthorizer.filterListCmdObjects should not filter any object
+ when(mockedAuthorizer.filterListCmdObjects(any(List.class),
+ any(HiveAuthzContext.class))).thenAnswer(new Answer<List<HivePrivilegeObject>>() {
+ @Override
+ public List<HivePrivilegeObject> answer(InvocationOnMock invocation) throws Throwable {
+ Object[] args = invocation.getArguments();
+ return (List<HivePrivilegeObject>) args[0];
+ }
+ });
+ }
+
+ @Test
+ public void testReplDump() throws HiveAuthzPluginException, HiveAccessControlException,
+ CommandNeedRetryException {
+
+ resetAuthorizer();
+ int status = driver.compile("repl dump " + dbName);
+ assertEquals(0, status);
+ List<HivePrivilegeObject> inputs = getHivePrivilegeObjectInputs().getLeft();
+ HivePrivilegeObject dbObj = inputs.get(0);
+ assertEquals("input type", HivePrivilegeObjectType.DATABASE, dbObj.getType());
+ assertEquals("db name", dbName.toLowerCase(), dbObj.getDbname());
+
+ resetAuthorizer();
+ status = driver.compile("repl dump " + dbName + "." + inDbTableName);
+ assertEquals(0, status);
+ inputs = getHivePrivilegeObjectInputs().getLeft();
+ dbObj = inputs.get(0);
+ assertEquals("input type", HivePrivilegeObjectType.TABLE_OR_VIEW, dbObj.getType());
+ assertEquals("db name", dbName.toLowerCase(), dbObj.getDbname());
+ assertEquals("table name", inDbTableName.toLowerCase(), dbObj.getObjectName());
+ }
private void checkSingleTableInput(List<HivePrivilegeObject> inputs) {
assertEquals("number of inputs", 1, inputs.size());
http://git-wip-us.apache.org/repos/asf/hive/blob/ceec5831/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
index f9bdff8..f3ba3e4 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/repl/ReplDumpTask.java
@@ -17,7 +17,6 @@
*/
package org.apache.hadoop.hive.ql.exec.repl;
-import com.google.common.collect.Collections2;
import com.google.common.primitives.Ints;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -40,7 +39,6 @@ import org.apache.hadoop.hive.ql.metadata.InvalidTableException;
import org.apache.hadoop.hive.ql.parse.BaseSemanticAnalyzer.TableSpec;
import org.apache.hadoop.hive.ql.parse.EximUtil;
import org.apache.hadoop.hive.ql.parse.ReplicationSpec;
-import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.hive.ql.parse.repl.DumpType;
import org.apache.hadoop.hive.ql.parse.repl.dump.HiveWrapper;
@@ -65,9 +63,6 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
private static final String FUNCTIONS_ROOT_DIR_NAME = "_functions";
private static final String FUNCTION_METADATA_DIR_NAME = "_metadata";
- private final static String TMP_TABLE_PREFIX =
- SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase();
-
private Logger LOG = LoggerFactory.getLogger(ReplDumpTask.class);
private Logger REPL_STATE_LOG = LoggerFactory.getLogger("ReplState");
@@ -176,7 +171,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
private Long bootStrapDump(Path dumpRoot, DumpMetaData dmd, Path cmRoot) throws Exception {
// bootstrap case
Long bootDumpBeginReplId = getHive().getMSC().getCurrentNotificationEventId().getEventId();
- for (String dbName : matchesDb()) {
+ for (String dbName : Utils.matchesDb(getHive(), work.dbNameOrPattern)) {
REPL_STATE_LOG
.info("Repl Dump: Started analyzing Repl Dump for DB: {}, Dump Type: BOOTSTRAP",
dbName);
@@ -184,7 +179,7 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
Path dbRoot = dumpDbMetadata(dbName, dumpRoot);
dumpFunctionMetadata(dbName, dumpRoot);
- for (String tblName : matchesTbl(dbName, work.tableNameOrPattern)) {
+ for (String tblName : Utils.matchesTbl(getHive(), dbName, work.tableNameOrPattern)) {
LOG.debug(
"analyzeReplDump dumping table: " + tblName + " to db root " + dbRoot.toUri());
dumpTable(dbName, tblName, dbRoot);
@@ -223,28 +218,6 @@ public class ReplDumpTask extends Task<ReplDumpWork> implements Serializable {
return bootDumpEndReplId;
}
- private Iterable<? extends String> matchesDb() throws HiveException {
- if (work.dbNameOrPattern == null) {
- return getHive().getAllDatabases();
- } else {
- return getHive().getDatabasesByPattern(work.dbNameOrPattern);
- }
- }
-
- private Iterable<? extends String> matchesTbl(String dbName, String tblPattern)
- throws HiveException {
- Hive db = Hive.get();
- if (tblPattern == null) {
- return Collections2.filter(db.getAllTables(dbName),
- tableName -> {
- assert tableName != null;
- return !tableName.toLowerCase().startsWith(TMP_TABLE_PREFIX);
- });
- } else {
- return db.getTablesByPattern(dbName, tblPattern);
- }
- }
-
private Path dumpDbMetadata(String dbName, Path dumpRoot) throws Exception {
Path dbRoot = new Path(dumpRoot, dbName);
// TODO : instantiating FS objects are generally costly. Refactor
http://git-wip-us.apache.org/repos/asf/hive/blob/ceec5831/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
index 48d9c94..b41facd 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ReplicationSemanticAnalyzer.java
@@ -23,14 +23,17 @@ import org.antlr.runtime.tree.Tree;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.metastore.api.Database;
import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryState;
import org.apache.hadoop.hive.ql.exec.Task;
import org.apache.hadoop.hive.ql.exec.TaskFactory;
+import org.apache.hadoop.hive.ql.exec.repl.ReplDumpTask;
import org.apache.hadoop.hive.ql.exec.repl.ReplDumpWork;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.hooks.WriteEntity;
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Table;
@@ -174,6 +177,17 @@ public class ReplicationSemanticAnalyzer extends BaseSemanticAnalyzer {
ctx.getResFile().toUri().toString()
), conf);
rootTasks.add(replDumpWorkTask);
+ if (dbNameOrPattern != null) {
+ for (String dbName : Utils.matchesDb(db, dbNameOrPattern)) {
+ if (tblNameOrPattern != null) {
+ for (String tblName : Utils.matchesTbl(db, dbName, tblNameOrPattern)) {
+ inputs.add(new ReadEntity(db.getTable(dbName, tblName)));
+ }
+ } else {
+ inputs.add(new ReadEntity(db.getDatabase(dbName)));
+ }
+ }
+ }
setFetchTask(createFetchTask(dumpSchema));
} catch (Exception e) {
// TODO : simple wrap & rethrow for now, clean up with error codes
http://git-wip-us.apache.org/repos/asf/hive/blob/ceec5831/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
----------------------------------------------------------------------
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
index 846b6f5..f40c703 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/repl/dump/Utils.java
@@ -21,9 +21,14 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.Utilities;
+import org.apache.hadoop.hive.ql.metadata.Hive;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.parse.SemanticAnalyzer;
import org.apache.hadoop.hive.ql.parse.SemanticException;
import org.apache.hadoop.io.IOUtils;
+import com.google.common.collect.Collections2;
+
import java.io.DataOutputStream;
import java.io.IOException;
import java.util.List;
@@ -47,4 +52,26 @@ public class Utils {
IOUtils.closeStream(outStream);
}
}
+
+ public static Iterable<? extends String> matchesDb(Hive db, String dbPattern) throws HiveException {
+ if (dbPattern == null) {
+ return db.getAllDatabases();
+ } else {
+ return db.getDatabasesByPattern(dbPattern);
+ }
+ }
+
+ public static Iterable<? extends String> matchesTbl(Hive db, String dbName, String tblPattern)
+ throws HiveException {
+ if (tblPattern == null) {
+ return Collections2.filter(db.getAllTables(dbName),
+ tableName -> {
+ assert tableName != null;
+ return !tableName.toLowerCase().startsWith(
+ SemanticAnalyzer.VALUES_TMP_TABLE_NAME_PREFIX.toLowerCase());
+ });
+ } else {
+ return db.getTablesByPattern(dbName, tblPattern);
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hive/blob/ceec5831/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out b/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out
index 29e72f4..ac5103e 100644
--- a/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out
+++ b/ql/src/test/results/clientnegative/repl_dump_requires_admin.q.out
@@ -55,8 +55,10 @@ POSTHOOK: type: SHOWTABLES
POSTHOOK: Input: database:default
PREHOOK: query: repl dump test_repldump_adminpriv
PREHOOK: type: REPLDUMP
+PREHOOK: Input: database:test_repldump_adminpriv
POSTHOOK: query: repl dump test_repldump_adminpriv
POSTHOOK: type: REPLDUMP
+POSTHOOK: Input: database:test_repldump_adminpriv
#### A masked pattern was here ####
PREHOOK: query: show tables test_repldump_adminpriv
PREHOOK: type: SHOWTABLES
http://git-wip-us.apache.org/repos/asf/hive/blob/ceec5831/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out
----------------------------------------------------------------------
diff --git a/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out b/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out
index df0b5dc..16cd9b7 100644
--- a/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out
+++ b/ql/src/test/results/clientnegative/repl_load_requires_admin.q.out
@@ -63,8 +63,10 @@ POSTHOOK: type: SHOWTABLES
POSTHOOK: Input: database:default
PREHOOK: query: repl dump test_replload_adminpriv_src
PREHOOK: type: REPLDUMP
+PREHOOK: Input: database:test_replload_adminpriv_src
POSTHOOK: query: repl dump test_replload_adminpriv_src
POSTHOOK: type: REPLDUMP
+POSTHOOK: Input: database:test_replload_adminpriv_src
#### A masked pattern was here ####
PREHOOK: type: REPLLOAD
#### A masked pattern was here ####