You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2019/01/04 21:52:12 UTC
[3/4] impala git commit: IMPALA-7795: Implement REFRESH AUTHORIZATION
statement
IMPALA-7795: Implement REFRESH AUTHORIZATION statement
This patch implements REFRESH AUTHORIZATION statement to explicitly
refresh authorization metadata. This statement is useful to force
Impala to refresh its authorization metadata when there is an external
update to authorization metadata without having to wait for the Sentry
polling or call INVALIDATE METADATA. Some tests were updated to use
REFRESH AUTHORIZATION instead of INVALIDATE METADATA to make the tests
run faster.
Syntax:
REFRESH AUTHORIZATION (authorization must be enabled to execute this
statement)
Testing:
- Added new FE tests
- Added new E2E authorization tests
- Ran all FE tests
- Ran all E2E authorization tests
Change-Id: I5459e1c97b12dee307e0cf85b94a9f66fd9d9a8c
Reviewed-on: http://gerrit.cloudera.org:8080/11888
Reviewed-by: Fredy Wijaya <fw...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/05b09f25
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/05b09f25
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/05b09f25
Branch: refs/heads/master
Commit: 05b09f259988f3022ed2d20055a4a57de540788f
Parents: a954d54
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Tue Nov 6 11:59:46 2018 -0800
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Fri Jan 4 07:22:37 2019 +0000
----------------------------------------------------------------------
common/thrift/CatalogService.thrift | 3 +
fe/src/main/cup/sql-parser.cup | 14 +-
.../impala/analysis/ResetMetadataStmt.java | 189 +++++++++++++------
.../impala/catalog/CatalogServiceCatalog.java | 32 ++--
.../impala/service/CatalogOpExecutor.java | 8 +
.../org/apache/impala/util/SentryProxy.java | 129 ++++++++-----
fe/src/main/jflex/sql-scanner.flex | 15 +-
.../apache/impala/analysis/AnalyzerTest.java | 69 ++++---
.../impala/analysis/AuthorizationStmtTest.java | 20 +-
.../org/apache/impala/analysis/ParserTest.java | 2 +
.../impala/analysis/StmtMetadataLoaderTest.java | 1 +
.../org/apache/impala/analysis/ToSqlTest.java | 25 ++-
.../apache/impala/common/FrontendTestBase.java | 12 ++
.../org/apache/impala/util/SentryProxyTest.java | 3 +-
tests/authorization/test_authorization.py | 116 +++++++++++-
tests/authorization/test_grant_revoke.py | 33 ++--
tests/authorization/test_owner_privileges.py | 70 ++++---
tests/common/sentry_cache_test_suite.py | 11 +-
18 files changed, 527 insertions(+), 225 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/common/thrift/CatalogService.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/CatalogService.thrift b/common/thrift/CatalogService.thrift
index 6237c1f..9add6a6 100644
--- a/common/thrift/CatalogService.thrift
+++ b/common/thrift/CatalogService.thrift
@@ -216,6 +216,9 @@ struct TResetMetadataRequest {
// True if SYNC_DDL is set in query options
7: required bool sync_ddl
+
+ // If set, refreshes authorization metadata.
+ 8: optional bool authorization
}
// Response from TResetMetadataRequest
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/main/cup/sql-parser.cup
----------------------------------------------------------------------
diff --git a/fe/src/main/cup/sql-parser.cup b/fe/src/main/cup/sql-parser.cup
index 12fd55d..bbd5ccf 100644
--- a/fe/src/main/cup/sql-parser.cup
+++ b/fe/src/main/cup/sql-parser.cup
@@ -273,8 +273,8 @@ parser code {:
// ALL KEYWORDS ALSO NEED TO BE ADDED TO THE word PRODUCTION.
terminal
KW_ADD, KW_AGGREGATE, KW_ALL, KW_ALTER, KW_ANALYTIC, KW_AND, KW_ANTI, KW_API_VERSION,
- KW_ARRAY, KW_AS, KW_ASC, KW_AVRO, KW_BETWEEN, KW_BIGINT, KW_BINARY, KW_BLOCKSIZE,
- KW_BOOLEAN, KW_BY, KW_CACHED, KW_CASCADE, KW_CASE, KW_CAST, KW_CHANGE,
+ KW_ARRAY, KW_AS, KW_ASC, KW_AUTHORIZATION, KW_AVRO, KW_BETWEEN, KW_BIGINT, KW_BINARY,
+ KW_BLOCKSIZE, KW_BOOLEAN, KW_BY, KW_CACHED, KW_CASCADE, KW_CASE, KW_CAST, KW_CHANGE,
KW_CHAR, KW_CLASS, KW_CLOSE_FN, KW_COLUMN, KW_COLUMNS, KW_COMMENT, KW_COMPRESSION,
KW_COMPUTE, KW_CREATE, KW_CROSS, KW_CURRENT, KW_DATA, KW_DATABASE, KW_DATABASES,
KW_DATE, KW_DATETIME, KW_DECIMAL, KW_DEFAULT, KW_DELETE, KW_DELIMITED, KW_DESC,
@@ -714,15 +714,17 @@ overwrite_val ::=
reset_metadata_stmt ::=
KW_INVALIDATE KW_METADATA
- {: RESULT = ResetMetadataStmt.createInvalidateStmt(null); :}
+ {: RESULT = ResetMetadataStmt.createInvalidateStmt(); :}
| KW_INVALIDATE KW_METADATA table_name:table
{: RESULT = ResetMetadataStmt.createInvalidateStmt(table); :}
| KW_REFRESH table_name:table
- {: RESULT = ResetMetadataStmt.createRefreshTableStmt(table, null); :}
+ {: RESULT = ResetMetadataStmt.createRefreshTableStmt(table); :}
| KW_REFRESH table_name:table partition_spec:partition
- {: RESULT = ResetMetadataStmt.createRefreshTableStmt(table, partition); :}
+ {: RESULT = ResetMetadataStmt.createRefreshPartitionStmt(table, partition); :}
| KW_REFRESH KW_FUNCTIONS ident_or_default:db
{: RESULT = ResetMetadataStmt.createRefreshFunctionsStmt(db); :}
+ | KW_REFRESH KW_AUTHORIZATION
+ {: RESULT = ResetMetadataStmt.createRefreshAuthorizationStmt(); :}
;
explain_stmt ::=
@@ -3432,6 +3434,8 @@ word ::=
{: RESULT = r.toString(); :}
| KW_ASC:r
{: RESULT = r.toString(); :}
+ | KW_AUTHORIZATION:r
+ {: RESULT = r.toString(); :}
| KW_AVRO:r
{: RESULT = r.toString(); :}
| KW_BETWEEN:r
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
index 966571a..bf47067 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ResetMetadataStmt.java
@@ -19,6 +19,7 @@ package org.apache.impala.analysis;
import java.util.List;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.impala.authorization.Privilege;
import org.apache.impala.authorization.PrivilegeRequest;
import org.apache.impala.authorization.PrivilegeRequestBuilder;
@@ -36,50 +37,85 @@ import com.google.common.base.Preconditions;
* REFRESH <table>
* REFRESH <table> PARTITION <partition>
* REFRESH FUNCTIONS <database>
+ * REFRESH AUTHORIZATION
*/
public class ResetMetadataStmt extends StatementBase {
+ public enum Action {
+ INVALIDATE_METADATA_ALL(false),
+ INVALIDATE_METADATA_TABLE(false),
+ REFRESH_TABLE(true),
+ REFRESH_PARTITION(true),
+ REFRESH_FUNCTIONS(true),
+ REFRESH_AUTHORIZATION(true);
+
+ private final boolean isRefresh_;
+
+ Action(boolean isRefresh) {
+ isRefresh_ = isRefresh;
+ }
+
+ public boolean isRefresh() { return isRefresh_; }
+ }
+
// Updated during analysis. Null if invalidating the entire catalog or refreshing
// database functions.
private TableName tableName_;
- // true if it is a REFRESH statement.
- private final boolean isRefresh_;
-
// not null when refreshing a single partition
private final PartitionSpec partitionSpec_;
// not null when refreshing functions in a database.
private final String database_;
- private ResetMetadataStmt(TableName tableName, boolean isRefresh,
- PartitionSpec partitionSpec, String db) {
- Preconditions.checkArgument(!isRefresh || (tableName != null || db != null));
- Preconditions.checkArgument(isRefresh || (partitionSpec == null && db == null));
- Preconditions.checkArgument(db == null || (
- tableName == null && isRefresh && partitionSpec == null));
-
- this.database_ = db;
- this.tableName_ = tableName;
- this.isRefresh_ = isRefresh;
- this.partitionSpec_ = partitionSpec;
+ // The type of action.
+ private final Action action_;
+
+ private ResetMetadataStmt(Action action, String db, TableName tableName,
+ PartitionSpec partitionSpec) {
+ Preconditions.checkNotNull(action);
+ action_ = action;
+ database_ = db;
+ tableName_ = tableName;
+ partitionSpec_ = partitionSpec;
if (partitionSpec_ != null) partitionSpec_.setTableName(tableName_);
}
+ public static ResetMetadataStmt createInvalidateStmt() {
+ return new ResetMetadataStmt(Action.INVALIDATE_METADATA_ALL, /*db*/ null,
+ /*table*/ null, /*partition*/ null);
+ }
+
public static ResetMetadataStmt createInvalidateStmt(TableName tableName) {
- return new ResetMetadataStmt(tableName, false, null, null);
+ return new ResetMetadataStmt(Action.INVALIDATE_METADATA_TABLE, /*db*/ null,
+ /*table*/ Preconditions.checkNotNull(tableName), /*partition*/ null);
}
- public static ResetMetadataStmt createRefreshTableStmt(TableName tableName,
+ public static ResetMetadataStmt createRefreshTableStmt(TableName tableName) {
+ return new ResetMetadataStmt(Action.REFRESH_TABLE, /*db*/ null,
+ Preconditions.checkNotNull(tableName), /*partition*/ null);
+ }
+
+ public static ResetMetadataStmt createRefreshPartitionStmt(TableName tableName,
PartitionSpec partitionSpec) {
- return new ResetMetadataStmt(tableName, true, partitionSpec, null);
+ return new ResetMetadataStmt(Action.REFRESH_PARTITION, /*db*/ null,
+ Preconditions.checkNotNull(tableName), Preconditions.checkNotNull(partitionSpec));
+ }
+
+ public static ResetMetadataStmt createRefreshFunctionsStmt(String db) {
+ return new ResetMetadataStmt(Action.REFRESH_FUNCTIONS, Preconditions.checkNotNull(db),
+ /*table*/ null, /*partition*/ null);
}
- public static ResetMetadataStmt createRefreshFunctionsStmt(String database) {
- return new ResetMetadataStmt(null, true, null, database);
+ public static ResetMetadataStmt createRefreshAuthorizationStmt() {
+ return new ResetMetadataStmt(Action.REFRESH_AUTHORIZATION, /*db*/ null,
+ /*table*/ null, /*partition*/ null);
}
public TableName getTableName() { return tableName_; }
+ @VisibleForTesting
+ protected Action getAction() { return action_; }
+
@Override
public void collectTableRefs(List<TableRef> tblRefs) {
// Only need table metadata for REFRESH <tbl> PARTITION (<partition>)
@@ -90,69 +126,98 @@ public class ResetMetadataStmt extends StatementBase {
@Override
public void analyze(Analyzer analyzer) throws AnalysisException {
- if (tableName_ != null) {
- String dbName = analyzer.getTargetDbName(tableName_);
- tableName_ = new TableName(dbName, tableName_.getTbl());
-
- if (isRefresh_) {
- // Verify the user has privileges to access this table. Will throw if the parent
- // database does not exists. Don't call getTable() to avoid loading the table
- // metadata if it is not yet in this impalad's catalog cache.
- if (!analyzer.dbContainsTable(dbName, tableName_.getTbl(), Privilege.REFRESH)) {
- // Only throw an exception when the table does not exist for refresh statements
- // since 'invalidate metadata' should add/remove tables created/dropped external
- // to Impala.
- throw new AnalysisException(Analyzer.TBL_DOES_NOT_EXIST_ERROR_MSG + tableName_);
+ switch (action_) {
+ case INVALIDATE_METADATA_TABLE:
+ case REFRESH_TABLE:
+ case REFRESH_PARTITION:
+ Preconditions.checkNotNull(tableName_);
+ String dbName = analyzer.getTargetDbName(tableName_);
+ tableName_ = new TableName(dbName, tableName_.getTbl());
+ if (action_.isRefresh()) {
+ // Verify the user has privileges to access this table. Will throw if the parent
+ // database does not exists. Don't call getTable() to avoid loading the table
+ // metadata if it is not yet in this impalad's catalog cache.
+ if (!analyzer.dbContainsTable(dbName, tableName_.getTbl(), Privilege.REFRESH)) {
+ // Only throw an exception when the table does not exist for refresh
+ // statements since 'invalidate metadata' should add/remove tables
+ // created/dropped external to Impala.
+ throw new AnalysisException(Analyzer.TBL_DOES_NOT_EXIST_ERROR_MSG +
+ tableName_);
+ }
+ if (partitionSpec_ != null) {
+ partitionSpec_.setPrivilegeRequirement(Privilege.ANY);
+ partitionSpec_.analyze(analyzer);
+ }
+ } else {
+ // Verify the user has privileges to access this table.
+ analyzer.registerPrivReq(new PrivilegeRequestBuilder()
+ .onTable(dbName, tableName_.getTbl()).allOf(Privilege.REFRESH)
+ .toRequest());
}
- if (partitionSpec_ != null) {
- partitionSpec_.setPrivilegeRequirement(Privilege.ANY);
- partitionSpec_.analyze(analyzer);
+ break;
+ case REFRESH_AUTHORIZATION:
+ if (!analyzer.getAuthzConfig().isEnabled()) {
+ throw new AnalysisException("Authorization is not enabled. To enable " +
+ "authorization restart Impala with the --server_name=<name> flag.");
}
- } else {
- // Verify the user has privileges to access this table.
+ analyzer.registerPrivReq(new PrivilegeRequest(Privilege.REFRESH));
+ break;
+ case REFRESH_FUNCTIONS:
analyzer.registerPrivReq(new PrivilegeRequestBuilder()
- .onTable(dbName, tableName_.getTbl()).allOf(Privilege.REFRESH)
- .toRequest());
- }
- } else if (database_ != null) {
- analyzer.registerPrivReq(new PrivilegeRequestBuilder()
- .onDb(database_).allOf(Privilege.REFRESH).toRequest());
- } else {
- analyzer.registerPrivReq(new PrivilegeRequest(Privilege.REFRESH));
-
- if (BackendConfig.INSTANCE.getBackendCfg().use_local_catalog) {
- throw new AnalysisException("Global INVALIDATE METADATA is not supported " +
- "when --use_local_catalog is configured.");
- }
+ .onDb(database_).allOf(Privilege.REFRESH).toRequest());
+ break;
+ case INVALIDATE_METADATA_ALL:
+ analyzer.registerPrivReq(new PrivilegeRequest(Privilege.REFRESH));
+ if (BackendConfig.INSTANCE.getBackendCfg().use_local_catalog) {
+ throw new AnalysisException("Global INVALIDATE METADATA is not supported " +
+ "when --use_local_catalog is configured.");
+ }
+ break;
+ default:
+ throw new IllegalStateException("Invalid reset metadata action: " + action_);
}
}
@Override
public String toSql(ToSqlOptions options) {
StringBuilder result = new StringBuilder();
- if (isRefresh_) {
- result.append("REFRESH");
- if (database_ == null) {
- result.append(" ").append(tableName_);
- if (partitionSpec_ != null) result.append(" " + partitionSpec_.toSql(options));
- } else {
- result.append(" FUNCTIONS ").append(database_);
- }
- } else {
- result.append("INVALIDATE METADATA");
- if (tableName_ != null) result.append(" ").append(tableName_);
+ switch (action_) {
+ case REFRESH_AUTHORIZATION:
+ result.append("REFRESH AUTHORIZATION");
+ break;
+ case REFRESH_FUNCTIONS:
+ result.append("REFRESH FUNCTIONS ").append(database_);
+ break;
+ case REFRESH_TABLE:
+ result.append("REFRESH ").append(tableName_.toSql());
+ break;
+ case REFRESH_PARTITION:
+ result.append("REFRESH ").append(tableName_.toSql()).append(" ")
+ .append(partitionSpec_.toSql(options));
+ break;
+ case INVALIDATE_METADATA_ALL:
+ result.append("INVALIDATE METADATA");
+ break;
+ case INVALIDATE_METADATA_TABLE:
+ result.append("INVALIDATE METADATA ").append(tableName_.toSql());
+ break;
+ default:
+ throw new IllegalStateException("Invalid reset metadata action: " + action_);
}
return result.toString();
}
public TResetMetadataRequest toThrift() {
TResetMetadataRequest params = new TResetMetadataRequest();
- params.setIs_refresh(isRefresh_);
+ params.setIs_refresh(action_.isRefresh());
if (tableName_ != null) {
params.setTable_name(new TTableName(tableName_.getDb(), tableName_.getTbl()));
}
if (partitionSpec_ != null) params.setPartition_spec(partitionSpec_.toThrift());
if (database_ != null) params.setDb_name(database_);
+ if (action_ == Action.REFRESH_AUTHORIZATION) {
+ params.setAuthorization(true);
+ }
return params;
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index b2f113b..8b4cd8e 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -1134,6 +1134,22 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
+ * Refreshes Sentry authorization metadata. When authorization is not enabled, this
+ * method is a no-op.
+ */
+ public void refreshAuthorization(boolean resetVersions, List<TCatalogObject> added,
+ List<TCatalogObject> removed) throws CatalogException {
+ // Do nothing if authorization is not enabled.
+ if (sentryProxy_ == null) return;
+ try {
+ // Update the authorization policy, waiting for the result to complete.
+ sentryProxy_.refresh(resetVersions, added, removed);
+ } catch (Exception e) {
+ throw new CatalogException("Error refreshing authorization policy: ", e);
+ }
+ }
+
+ /**
* Resets this catalog instance by clearing all cached table and database metadata.
* Returns the current catalog version before reset has taken any effect. The
* requesting impalad will use that version to determine when the
@@ -1143,15 +1159,8 @@ public class CatalogServiceCatalog extends Catalog {
long currentCatalogVersion = getCatalogVersion();
LOG.info("Invalidating all metadata. Version: " + currentCatalogVersion);
// First update the policy metadata.
- if (sentryProxy_ != null) {
- // Sentry Service is enabled.
- try {
- // Update the authorization policy, waiting for the result to complete.
- sentryProxy_.refresh(true);
- } catch (Exception e) {
- throw new CatalogException("Error updating authorization policy: ", e);
- }
- }
+ refreshAuthorization(true, /*catalog objects added*/ new ArrayList<>(),
+ /*catalog objects removed*/ new ArrayList<>());
// Update the HDFS cache pools
CachePoolReader reader = new CachePoolReader(true);
@@ -1972,8 +1981,9 @@ public class CatalogServiceCatalog extends Catalog {
// an operation using SYNC_DDL must wait for.
long maxNumAttempts = 5;
if (result.isSetUpdated_catalog_objects()) {
- maxNumAttempts =
- result.getUpdated_catalog_objects().size() * (MAX_NUM_SKIPPED_TOPIC_UPDATES + 1);
+ maxNumAttempts = Math.max(maxNumAttempts,
+ result.getUpdated_catalog_objects().size() *
+ (MAX_NUM_SKIPPED_TOPIC_UPDATES + 1));
}
long numAttempts = 0;
long begin = System.currentTimeMillis();
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index aeb23df..04ec23d 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -3355,6 +3355,7 @@ public class CatalogOpExecutor {
* 2) invalidate a specific table, forcing the metadata to be reloaded
* on the next access.
* 3) perform a synchronous incremental refresh of a specific table.
+ * 4) perform a refresh on authorization metadata.
*
* For details on the specific commands see comments on their respective
* methods in CatalogServiceCatalog.java.
@@ -3446,6 +3447,13 @@ public class CatalogOpExecutor {
resp.getResult().addToUpdated_catalog_objects(addedDb.toTCatalogObject());
}
resp.getResult().setVersion(updatedThriftTable.getCatalog_version());
+ } else if (req.isAuthorization()) {
+ List<TCatalogObject> added = new ArrayList<>();
+ List<TCatalogObject> removed = new ArrayList<>();
+ catalog_.refreshAuthorization(false, added, removed);
+ resp.result.setUpdated_catalog_objects(added);
+ resp.result.setRemoved_catalog_objects(removed);
+ resp.result.setVersion(catalog_.getCatalogVersion());
} else {
// Invalidate the entire catalog if no table name is provided.
Preconditions.checkArgument(!req.isIs_refresh());
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/main/java/org/apache/impala/util/SentryProxy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryProxy.java b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
index 0087652..2b6b564 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryProxy.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
@@ -17,6 +17,7 @@
package org.apache.impala.util;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -35,6 +36,7 @@ import org.apache.impala.catalog.Role;
import org.apache.impala.common.Reference;
import org.apache.impala.common.SentryPolicyReaderException;
import org.apache.impala.common.SentryUnavailableException;
+import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TPrincipalType;
import org.apache.log4j.Logger;
import org.apache.sentry.api.service.thrift.TSentryGroup;
@@ -69,6 +71,28 @@ import org.apache.thrift.transport.TTransportException;
* synchronize all modifications.
*/
public class SentryProxy {
+ /**
+ * This class keeps track of catalog objects added and removed.
+ */
+ @VisibleForTesting
+ protected static class AuthorizationDelta {
+ private final List<TCatalogObject> added_;
+ private final List<TCatalogObject> removed_;
+
+ public AuthorizationDelta() {
+ this(new ArrayList<>(), new ArrayList<>());
+ }
+
+ public AuthorizationDelta(List<TCatalogObject> added,
+ List<TCatalogObject> removed) {
+ added_ = added;
+ removed_ = removed;
+ }
+
+ public List<TCatalogObject> getAddedCatalogObjects() { return added_; }
+ public List<TCatalogObject> getRemovedCatalogObjects() { return removed_; }
+ }
+
private static final Logger LOG = Logger.getLogger(SentryProxy.class);
// Used to periodically poll the Sentry Service and updates the catalog with any
@@ -113,7 +137,7 @@ public class SentryProxy {
// We configure the PolicyReader to swallow any exception because we do not want to
// kill the PolicyReader background thread on exception.
policyReader_.scheduleAtFixedRate(new PolicyReader(/*reset versions*/ false,
- /*swallow exception*/ true), 0,
+ /*swallow exception*/ true, new AuthorizationDelta()), 0,
BackendConfig.INSTANCE.getSentryCatalogPollingFrequency(), TimeUnit.SECONDS);
}
@@ -135,16 +159,20 @@ public class SentryProxy {
private final boolean resetVersions_;
// A flag to indicate whether or not to swallow any exception thrown.
private final boolean swallowException_;
+ // Keep track of catalog objects added/removed.
+ private final AuthorizationDelta authzDelta_;
- public PolicyReader(boolean resetVersions, boolean swallowException) {
+ public PolicyReader(boolean resetVersions, boolean swallowException,
+ AuthorizationDelta authzDelta) {
resetVersions_ = resetVersions;
swallowException_ = swallowException;
+ authzDelta_ = authzDelta;
}
public void run() {
synchronized (SentryProxy.this) {
refreshSentryAuthorization(catalog_, sentryPolicyService_, processUser_,
- resetVersions_, swallowException_);
+ resetVersions_, swallowException_, authzDelta_);
}
}
}
@@ -155,15 +183,13 @@ public class SentryProxy {
@VisibleForTesting
static void refreshSentryAuthorization(CatalogServiceCatalog catalog,
SentryPolicyService sentryPolicyService, User processUser, boolean resetVersions,
- boolean swallowException) {
- Set<String> rolesToRemove;
- Set<String> usersToRemove;
+ boolean swallowException, AuthorizationDelta authzDelta) {
long startTime = System.currentTimeMillis();
try {
- rolesToRemove = refreshRolePrivileges(catalog, sentryPolicyService, processUser,
- resetVersions);
- usersToRemove = refreshUserPrivileges(catalog, sentryPolicyService, processUser,
- resetVersions);
+ refreshRolePrivileges(catalog, sentryPolicyService, processUser, resetVersions,
+ authzDelta);
+ refreshUserPrivileges(catalog, sentryPolicyService, processUser, resetVersions,
+ authzDelta);
} catch (Exception e) {
LOG.error("Error refreshing Sentry policy: ", e);
if (swallowException) return;
@@ -181,27 +207,16 @@ public class SentryProxy {
LOG.debug("Refreshing Sentry policy took " +
(System.currentTimeMillis() - startTime) + "ms");
}
-
- // Remove all the roles, incrementing the catalog version to indicate
- // a change.
- for (String roleName: rolesToRemove) {
- catalog.removeRole(roleName);
- }
- // Remove all the users, incrementing the catalog version to indicate
- // a change.
- for (String userName: usersToRemove) {
- catalog.removeUser(userName);
- }
}
/**
* Updates all roles and their associated privileges in the catalog by adding,
* removing, and replacing the catalog objects to match those in Sentry since
- * the last sentry sync update. This method returns a list of roles to be removed.
+ * the last sentry sync update.
*/
- private static Set<String> refreshRolePrivileges(CatalogServiceCatalog catalog,
- SentryPolicyService sentryPolicyService, User processUser, boolean resetVersions)
- throws ImpalaException {
+ private static void refreshRolePrivileges(CatalogServiceCatalog catalog,
+ SentryPolicyService sentryPolicyService, User processUser, boolean resetVersions,
+ AuthorizationDelta authzDelta) throws ImpalaException {
// Assume all roles should be removed. Then query the Policy Service and remove
// roles from this set that actually exist.
Set<String> rolesToRemove = catalog.getAuthPolicy().getAllRoleNames();
@@ -230,26 +245,34 @@ public class SentryProxy {
role.setCatalogVersion(catalog.incrementAndGetCatalogVersion());
}
} else {
+ LOG.debug("Adding role: " + sentryRole.getRoleName());
role = catalog.addRole(sentryRole.getRoleName(), grantGroups);
+ authzDelta.getAddedCatalogObjects().add(role.toTCatalogObject());
}
// allRolesPrivileges keys and sentryRole.getName() are used here since they both
// come from Sentry so they agree in case.
refreshPrivilegesInCatalog(catalog, resetVersions, sentryRole.getRoleName(), role,
- allRolesPrivileges);
+ allRolesPrivileges, authzDelta);
+ }
+ // Remove all the roles, incrementing the catalog version to indicate
+ // a change.
+ for (String roleName: rolesToRemove) {
+ LOG.debug("Removing role: " + roleName);
+ authzDelta.getRemovedCatalogObjects().add(
+ catalog.removeRole(roleName).toTCatalogObject());
}
- return rolesToRemove;
}
/**
* Updates all users and their associated privileges in the catalog by adding,
* removing, and replacing the catalog objects to match those in Sentry since the
* last Sentry sync update. Take note that we only store the users with privileges
- * stored in Sentry and not all available users in the system. This method returns a
- * list of users to be removed. User privileges do not support grant groups.
+ * stored in Sentry and not all available users in the system. User privileges do not
+ * support grant groups.
*/
- private static Set<String> refreshUserPrivileges(CatalogServiceCatalog catalog,
- SentryPolicyService sentryPolicyService, User processUser, boolean resetVersions)
- throws ImpalaException {
+ private static void refreshUserPrivileges(CatalogServiceCatalog catalog,
+ SentryPolicyService sentryPolicyService, User processUser, boolean resetVersions,
+ AuthorizationDelta authzDelta) throws ImpalaException {
// Assume all users should be removed. Then query the Policy Service and remove
// users from this set that actually exist.
Set<String> usersToRemove = catalog.getAuthPolicy().getAllUserNames();
@@ -268,13 +291,22 @@ public class SentryProxy {
existingUser);
if (existingUser.getRef() && resetVersions) {
user.setCatalogVersion(catalog.incrementAndGetCatalogVersion());
+ } else if (!existingUser.getRef()) {
+ LOG.debug("Adding user: " + user.getName());
+ authzDelta.getAddedCatalogObjects().add(user.toTCatalogObject());
}
// allUsersPrivileges keys and userPrivilegesEntry.getKey() are used here since
// they both come from Sentry so they agree in case.
refreshPrivilegesInCatalog(catalog, resetVersions, userPrivilegesEntry.getKey(),
- user, allUsersPrivileges);
+ user, allUsersPrivileges, authzDelta);
+ }
+ // Remove all the users, incrementing the catalog version to indicate
+ // a change.
+ for (String userName: usersToRemove) {
+ LOG.debug("Removing user: " + userName);
+ authzDelta.getRemovedCatalogObjects().add(
+ catalog.removeUser(userName).toTCatalogObject());
}
- return usersToRemove;
}
/**
@@ -285,8 +317,8 @@ public class SentryProxy {
*/
private static void refreshPrivilegesInCatalog(CatalogServiceCatalog catalog,
boolean resetVersions, String sentryPrincipalName, Principal principal,
- Map<String, Set<TSentryPrivilege>> allPrincipalPrivileges)
- throws CatalogException {
+ Map<String, Set<TSentryPrivilege>> allPrincipalPrivileges,
+ AuthorizationDelta authzDelta) throws CatalogException {
// Assume all privileges should be removed. Privileges that still exist are
// deleted from this set and we are left with the set of privileges that need
// to be removed.
@@ -317,18 +349,28 @@ public class SentryProxy {
continue;
}
if (principal.getPrincipalType() == TPrincipalType.ROLE) {
- catalog.addRolePrivilege(principal.getName(), thriftPriv);
+ LOG.debug("Adding role privilege: " + privilegeName);
+ authzDelta.getAddedCatalogObjects().add(
+ catalog.addRolePrivilege(principal.getName(), thriftPriv).toTCatalogObject());
} else {
- catalog.addUserPrivilege(principal.getName(), thriftPriv);
+ LOG.debug("Adding user privilege: " + privilegeName);
+ authzDelta.getAddedCatalogObjects().add(
+ catalog.addUserPrivilege(principal.getName(), thriftPriv).toTCatalogObject());
}
}
// Remove the privileges that no longer exist.
for (String privilegeName: privilegesToRemove) {
if (principal.getPrincipalType() == TPrincipalType.ROLE) {
- catalog.removeRolePrivilege(principal.getName(), privilegeName);
+ LOG.debug("Removing role privilege: " + privilegeName);
+ authzDelta.getRemovedCatalogObjects().add(
+ catalog.removeRolePrivilege(principal.getName(), privilegeName)
+ .toTCatalogObject());
} else {
- catalog.removeUserPrivilege(principal.getName(), privilegeName);
+ LOG.debug("Removing user privilege: " + privilegeName);
+ authzDelta.getRemovedCatalogObjects().add(
+ catalog.removeUserPrivilege(principal.getName(), privilegeName)
+ .toTCatalogObject());
}
}
}
@@ -570,12 +612,13 @@ public class SentryProxy {
* the Catalog with any changes. Throws an ImpalaRuntimeException if there are any
* errors executing the refresh job.
*/
- public void refresh(boolean resetVersions) throws ImpalaRuntimeException {
+ public void refresh(boolean resetVersions, List<TCatalogObject> added,
+ List<TCatalogObject> removed) throws ImpalaRuntimeException {
try {
// Since this is a synchronous refresh, any exception thrown while running the
// refresh should be thrown here instead of silently swallowing it.
- policyReader_.submit(new PolicyReader(resetVersions, /*swallow exception*/ false))
- .get();
+ policyReader_.submit(new PolicyReader(resetVersions, /*swallow exception*/ false,
+ new AuthorizationDelta(added, removed))).get();
} catch (Exception e) {
if (e.getCause() != null && e.getCause() instanceof SentryUnavailableException) {
throw new ImpalaRuntimeException("Error refreshing authorization policy. " +
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/main/jflex/sql-scanner.flex
----------------------------------------------------------------------
diff --git a/fe/src/main/jflex/sql-scanner.flex b/fe/src/main/jflex/sql-scanner.flex
index 3d3fea6..54ab07e 100644
--- a/fe/src/main/jflex/sql-scanner.flex
+++ b/fe/src/main/jflex/sql-scanner.flex
@@ -75,6 +75,7 @@ import org.apache.impala.thrift.TReservedWordsVersion;
keywordMap.put("array", SqlParserSymbols.KW_ARRAY);
keywordMap.put("as", SqlParserSymbols.KW_AS);
keywordMap.put("asc", SqlParserSymbols.KW_ASC);
+ keywordMap.put("authorization", SqlParserSymbols.KW_AUTHORIZATION);
keywordMap.put("avro", SqlParserSymbols.KW_AVRO);
keywordMap.put("between", SqlParserSymbols.KW_BETWEEN);
keywordMap.put("bigint", SqlParserSymbols.KW_BIGINT);
@@ -318,13 +319,13 @@ import org.apache.impala.thrift.TReservedWordsVersion;
// Add SQL:2016 reserved words
reservedWords.addAll(Arrays.asList(new String[] {
"abs", "acos", "allocate", "any", "are", "array_agg", "array_max_cardinality",
- "asensitive", "asin", "asymmetric", "at", "atan", "atomic", "authorization",
- "avg", "begin", "begin_frame", "begin_partition", "blob", "both", "call",
- "called", "cardinality", "cascaded", "ceil", "ceiling", "char_length",
- "character", "character_length", "check", "classifier", "clob", "close",
- "coalesce", "collate", "collect", "commit", "condition", "connect", "constraint",
- "contains", "convert", "copy", "corr", "corresponding", "cos", "cosh", "count",
- "covar_pop", "covar_samp", "cube", "cume_dist", "current_catalog", "current_date",
+ "asensitive", "asin", "asymmetric", "at", "atan", "atomic", "avg", "begin",
+ "begin_frame", "begin_partition", "blob", "both", "call", "called", "cardinality",
+ "cascaded", "ceil", "ceiling", "char_length", "character", "character_length",
+ "check", "classifier", "clob", "close", "coalesce", "collate", "collect",
+ "commit", "condition", "connect", "constraint", "contains", "convert", "copy",
+ "corr", "corresponding", "cos", "cosh", "count", "covar_pop", "covar_samp",
+ "cube", "cume_dist", "current_catalog", "current_date",
"current_default_transform_group", "current_path", "current_path", "current_role",
"current_role", "current_row", "current_schema", "current_time",
"current_timestamp", "current_transform_group_for_type", "current_user", "cursor",
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
index 9a98977..8d2a4f2 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzerTest.java
@@ -20,6 +20,7 @@ package org.apache.impala.analysis;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.function.BiConsumer;
import org.apache.impala.catalog.Function;
import org.apache.impala.catalog.ScalarType;
@@ -33,6 +34,8 @@ import org.slf4j.LoggerFactory;
import com.google.common.base.Preconditions;
+import static org.junit.Assert.assertEquals;
+
public class AnalyzerTest extends FrontendTestBase {
protected final static Logger LOG = LoggerFactory.getLogger(AnalyzerTest.class);
@@ -116,7 +119,7 @@ public class AnalyzerTest extends FrontendTestBase {
tupleDesc.materializeSlots();
descTbl.computeMemLayout();
- Assert.assertEquals(89.0f, tupleDesc.getAvgSerializedSize(), 0.0);
+ assertEquals(89.0f, tupleDesc.getAvgSerializedSize(), 0.0);
checkLayoutParams("functional.alltypes.timestamp_col", 16, 0, 80, 0, analyzer);
checkLayoutParams("functional.alltypes.date_string_col", 12, 16, 80, 1, analyzer);
checkLayoutParams("functional.alltypes.string_col", 12, 28, 80, 2, analyzer);
@@ -143,8 +146,8 @@ public class AnalyzerTest extends FrontendTestBase {
TupleDescriptor aggDesc = descTbl.getTupleDesc(new TupleId(1));
aggDesc.materializeSlots();
descTbl.computeMemLayout();
- Assert.assertEquals(16.0f, aggDesc.getAvgSerializedSize(), 0.0);
- Assert.assertEquals(16, aggDesc.getByteSize());
+ assertEquals(16.0f, aggDesc.getAvgSerializedSize(), 0.0);
+ assertEquals(16, aggDesc.getByteSize());
checkLayoutParams(aggDesc.getSlots().get(0), 8, 0, 0, -1);
checkLayoutParams(aggDesc.getSlots().get(1), 8, 8, 0, -1);
}
@@ -161,8 +164,8 @@ public class AnalyzerTest extends FrontendTestBase {
TupleDescriptor aggDesc = descTbl.getTupleDesc(new TupleId(1));
aggDesc.materializeSlots();
descTbl.computeMemLayout();
- Assert.assertEquals(16.0f, aggDesc.getAvgSerializedSize(), 0.0);
- Assert.assertEquals(17, aggDesc.getByteSize());
+ assertEquals(16.0f, aggDesc.getAvgSerializedSize(), 0.0);
+ assertEquals(17, aggDesc.getByteSize());
checkLayoutParams(aggDesc.getSlots().get(0), 8, 0, 16, 0);
checkLayoutParams(aggDesc.getSlots().get(1), 8, 8, 0, -1);
}
@@ -183,7 +186,7 @@ public class AnalyzerTest extends FrontendTestBase {
slots.get(9).setIsMaterialized(false);
descTbl.computeMemLayout();
- Assert.assertEquals(64.0f, tupleDesc.getAvgSerializedSize(), 0.0);
+ assertEquals(64.0f, tupleDesc.getAvgSerializedSize(), 0.0);
// Check non-materialized slots.
checkLayoutParams("functional.alltypes.id", 0, -1, 0, 0, analyzer);
checkLayoutParams("functional.alltypes.double_col", 0, -1, 0, 0, analyzer);
@@ -203,10 +206,10 @@ public class AnalyzerTest extends FrontendTestBase {
private void checkLayoutParams(SlotDescriptor d, int byteSize, int byteOffset,
int nullIndicatorByte, int nullIndicatorBit) {
- Assert.assertEquals(byteSize, d.getByteSize());
- Assert.assertEquals(byteOffset, d.getByteOffset());
- Assert.assertEquals(nullIndicatorByte, d.getNullIndicatorByte());
- Assert.assertEquals(nullIndicatorBit, d.getNullIndicatorBit());
+ assertEquals(byteSize, d.getByteSize());
+ assertEquals(byteOffset, d.getByteOffset());
+ assertEquals(nullIndicatorByte, d.getNullIndicatorByte());
+ assertEquals(nullIndicatorBit, d.getNullIndicatorBit());
}
private void checkLayoutParams(String colAlias, int byteSize, int byteOffset,
@@ -219,7 +222,7 @@ public class AnalyzerTest extends FrontendTestBase {
// Requires query to parse to a SelectStmt.
protected void checkExprType(String query, Type type) {
SelectStmt select = (SelectStmt) AnalyzesOk(query);
- Assert.assertEquals(select.getResultExprs().get(0).getType(), type);
+ assertEquals(select.getResultExprs().get(0).getType(), type);
}
/**
@@ -319,20 +322,42 @@ public class AnalyzerTest extends FrontendTestBase {
@Test
public void TestResetMetadata() {
- AnalyzesOk("invalidate metadata");
- AnalyzesOk("invalidate metadata functional.alltypessmall");
- AnalyzesOk("invalidate metadata functional.alltypes_view");
- AnalyzesOk("invalidate metadata functional.bad_serde");
- AnalyzesOk("refresh functional.alltypessmall");
- AnalyzesOk("refresh functional.alltypes_view");
- AnalyzesOk("refresh functional.bad_serde");
- AnalyzesOk("refresh functional.alltypessmall partition (year=2009, month=1)");
- AnalyzesOk("refresh functional.alltypessmall partition (year=2009, month=NULL)");
+ BiConsumer<ParseNode, ResetMetadataStmt.Action> assertAction =
+ (parseNode, action) -> {
+ Preconditions.checkArgument(parseNode instanceof ResetMetadataStmt);
+ assertEquals(action, ((ResetMetadataStmt) parseNode).getAction());
+ };
+
+ assertAction.accept(AnalyzesOk("invalidate metadata"),
+ ResetMetadataStmt.Action.INVALIDATE_METADATA_ALL);
+ assertAction.accept(AnalyzesOk("invalidate metadata functional.alltypessmall"),
+ ResetMetadataStmt.Action.INVALIDATE_METADATA_TABLE);
+ assertAction.accept(AnalyzesOk("invalidate metadata functional.alltypes_view"),
+ ResetMetadataStmt.Action.INVALIDATE_METADATA_TABLE);
+ assertAction.accept(AnalyzesOk("invalidate metadata functional.bad_serde"),
+ ResetMetadataStmt.Action.INVALIDATE_METADATA_TABLE);
+ assertAction.accept(AnalyzesOk("refresh functional.alltypessmall"),
+ ResetMetadataStmt.Action.REFRESH_TABLE);
+ assertAction.accept(AnalyzesOk("refresh functional.alltypes_view"),
+ ResetMetadataStmt.Action.REFRESH_TABLE);
+ assertAction.accept(AnalyzesOk("refresh functional.bad_serde"),
+ ResetMetadataStmt.Action.REFRESH_TABLE);
+ assertAction.accept(AnalyzesOk(
+ "refresh functional.alltypessmall partition (year=2009, month=1)"),
+ ResetMetadataStmt.Action.REFRESH_PARTITION);
+ assertAction.accept(AnalyzesOk(
+ "refresh functional.alltypessmall partition (year=2009, month=NULL)"),
+ ResetMetadataStmt.Action.REFRESH_PARTITION);
+ assertAction.accept(AnalyzesOk(
+ "refresh authorization", createAnalysisCtx(createAuthorizationConfig())),
+ ResetMetadataStmt.Action.REFRESH_AUTHORIZATION);
// invalidate metadata <table name> checks the Hive Metastore for table existence
// and should not throw an AnalysisError if the table or db does not exist.
- AnalyzesOk("invalidate metadata functional.unknown_table");
- AnalyzesOk("invalidate metadata unknown_db.unknown_table");
+ assertAction.accept(AnalyzesOk("invalidate metadata functional.unknown_table"),
+ ResetMetadataStmt.Action.INVALIDATE_METADATA_TABLE);
+ assertAction.accept(AnalyzesOk("invalidate metadata unknown_db.unknown_table"),
+ ResetMetadataStmt.Action.INVALIDATE_METADATA_TABLE);
AnalysisError("refresh functional.unknown_table",
"Table does not exist: functional.unknown_table");
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
index 02ae21b..2d7ada8 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
@@ -74,10 +74,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
private final Frontend authzFrontend_;
public AuthorizationStmtTest() {
- AuthorizationConfig authzConfig = AuthorizationConfig.createHadoopGroupAuthConfig(
- SENTRY_SERVER, null, System.getenv("IMPALA_HOME") +
- "/fe/src/test/resources/sentry-site.xml");
- authzConfig.validateConfig();
+ AuthorizationConfig authzConfig = createAuthorizationConfig();
analysisContext_ = createAnalysisCtx(authzConfig, USER.getName());
authzCatalog_ = new ImpaladTestCatalog(authzConfig);
authzFrontend_ = new Frontend(authzConfig, authzCatalog_);
@@ -971,12 +968,15 @@ public class AuthorizationStmtTest extends FrontendTestBase {
@Test
public void testResetMetadata() throws ImpalaException {
- // Invalidate metadata on server.
- authorize("invalidate metadata")
- .ok(onServer(TPrivilegeLevel.ALL))
- .ok(onServer(TPrivilegeLevel.OWNER))
- .ok(onServer(TPrivilegeLevel.REFRESH))
- .error(refreshError("server"));
+ // Invalidate metadata/refresh authorization on server.
+ for (AuthzTest test: new AuthzTest[]{
+ authorize("invalidate metadata"),
+ authorize("refresh authorization")}) {
+ test.ok(onServer(TPrivilegeLevel.ALL))
+ .ok(onServer(TPrivilegeLevel.OWNER))
+ .ok(onServer(TPrivilegeLevel.REFRESH))
+ .error(refreshError("server"));
+ }
// Invalidate metadata/refresh on a table / view
for(String name: new String[] {"alltypes", "alltypes_view"}) {
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
index 62c74c5..0704188 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ParserTest.java
@@ -3195,6 +3195,7 @@ public class ParserTest extends FrontendTestBase {
ParsesOk("refresh Foo.S partition (col=2)");
ParsesOk("refresh Foo.S partition (col1 = 2, col2 = 3)");
ParsesOk("refresh functions Foo");
+ ParsesOk("refresh authorization");
ParserError("invalidate");
ParserError("invalidate metadata Foo.S.S");
@@ -3205,6 +3206,7 @@ public class ParserTest extends FrontendTestBase {
ParserError("refresh Foo.S partition (col1 = 2, col2)");
ParserError("refresh Foo.S partition ()");
ParserError("refresh functions Foo.S");
+ ParserError("refresh authorization Foo");
}
@Test
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
index b58b88b..9ef923a 100644
--- a/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/StmtMetadataLoaderTest.java
@@ -193,6 +193,7 @@ public class StmtMetadataLoaderTest {
testNoLoad("invalidate metadata functional.alltypes");
testNoLoad("refresh functional.alltypes");
testNoLoad("refresh functions functional");
+ testNoLoad("refresh authorization");
// This stmt requires the table to be loaded.
testLoadTables("refresh functional.alltypes partition (year=2009, month=1)", 1, 1,
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
index dcdd814..b570056 100644
--- a/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/ToSqlTest.java
@@ -73,19 +73,37 @@ public class ToSqlTest extends FrontendTestBase {
testToSql(query, query);
}
+ private void testToSql(AnalysisContext ctx, String query) {
+ testToSql(ctx, query, query);
+ }
+
private void testToSql(String query, String expected) {
testToSql(query, System.getProperty("user.name"), expected);
}
+ private void testToSql(AnalysisContext ctx, String query, String expected) {
+ testToSql(ctx, query, System.getProperty("user.name"), expected);
+ }
+
private void testToSql(String query, String defaultDb, String expected) {
testToSql(query, defaultDb, expected, false);
}
+ private void testToSql(AnalysisContext ctx, String query, String defaultDb,
+ String expected) {
+ testToSql(ctx, query, defaultDb, expected, false);
+ }
+
private void testToSql(String query, String defaultDb, String expected,
boolean ignoreWhitespace) {
+ testToSql(createAnalysisCtx(defaultDb), query, defaultDb, expected, ignoreWhitespace);
+ }
+
+ private void testToSql(AnalysisContext ctx, String query, String defaultDb,
+ String expected, boolean ignoreWhitespace) {
String actual = null;
try {
- ParseNode node = AnalyzesOk(query, createAnalysisCtx(defaultDb));
+ ParseNode node = AnalyzesOk(query, ctx);
if (node instanceof QueryStmt) {
actual = ((QueryStmt)node).getOrigSqlString();
} else {
@@ -97,7 +115,7 @@ public class ToSqlTest extends FrontendTestBase {
}
if (!actual.equals(expected)) {
String msg = "\n<<< Expected(length:" + expected.length() + "): [" + expected
- + "]\n>>> Actual(length:" + actual.length() + "): [" + actual + "]\n";
+ + "]\n>>> Actual(length:" + actual.length() + "): [" + actual + "]\n";
System.err.println(msg);
fail(msg);
}
@@ -106,7 +124,7 @@ public class ToSqlTest extends FrontendTestBase {
fail("Failed to analyze query: " + query + "\n" + e.getMessage());
}
// Parse and analyze the resulting SQL to ensure its validity.
- AnalyzesOk(actual, createAnalysisCtx(defaultDb));
+ AnalyzesOk(actual, ctx);
}
private void runTestTemplate(String sql, String expectedSql, String[]... testDims) {
@@ -1447,6 +1465,7 @@ public class ToSqlTest extends FrontendTestBase {
testToSql("REFRESH functional.alltypes");
testToSql("REFRESH functional.alltypes PARTITION (year=2009, month=1)");
testToSql("REFRESH FUNCTIONS functional");
+ testToSql(createAnalysisCtx(createAuthorizationConfig()), "REFRESH AUTHORIZATION");
}
/**
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
index de4266b..0ea33a7 100644
--- a/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
+++ b/fe/src/test/java/org/apache/impala/common/FrontendTestBase.java
@@ -441,4 +441,16 @@ public class FrontendTestBase {
StmtTableCache stmtTableCache = mdLoader.loadTables(parsedStmt);
return ctx.analyzeAndAuthorize(parsedStmt, stmtTableCache, fe.getAuthzChecker());
}
+
+ /**
+ * Creates an authorization config for creating an AnalysisContext with
+ * authorization enabled.
+ */
+ protected AuthorizationConfig createAuthorizationConfig() {
+ AuthorizationConfig authzConfig = AuthorizationConfig.createHadoopGroupAuthConfig(
+ "server1", null, System.getenv("IMPALA_HOME") +
+ "/fe/src/test/resources/sentry-site.xml");
+ authzConfig.validateConfig();
+ return authzConfig;
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/fe/src/test/java/org/apache/impala/util/SentryProxyTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/util/SentryProxyTest.java b/fe/src/test/java/org/apache/impala/util/SentryProxyTest.java
index a95bc7f..d3246ab 100644
--- a/fe/src/test/java/org/apache/impala/util/SentryProxyTest.java
+++ b/fe/src/test/java/org/apache/impala/util/SentryProxyTest.java
@@ -32,6 +32,7 @@ import org.apache.impala.thrift.TPrincipalType;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TPrivilegeLevel;
import org.apache.impala.thrift.TPrivilegeScope;
+import org.apache.impala.util.SentryProxy.AuthorizationDelta;
import org.apache.sentry.api.service.thrift.TSentryPrivilege;
import org.apache.sentry.api.service.thrift.TSentryRole;
import org.junit.After;
@@ -516,7 +517,7 @@ public class SentryProxyTest {
private static CatalogState refreshSentryAuthorization(CatalogServiceCatalog catalog,
SentryPolicyService sentryService, boolean resetVersions) {
SentryProxy.refreshSentryAuthorization(catalog, sentryService, USER, resetVersions,
- false);
+ false, new AuthorizationDelta());
return new CatalogState(catalog.getCatalogVersion(), getAuthCatalogSize(catalog));
}
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/tests/authorization/test_authorization.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py
index e315f6d..245e2ba 100644
--- a/tests/authorization/test_authorization.py
+++ b/tests/authorization/test_authorization.py
@@ -24,6 +24,8 @@ import tempfile
import json
import grp
import re
+import sys
+import subprocess
import urllib
from time import sleep, time
@@ -500,7 +502,7 @@ class TestAuthorization(CustomClusterTestSuite):
# Calling INVALIDATE METADATA when Sentry is unavailable should return an error.
result = self.execute_query_expect_failure(self.client, "invalidate metadata")
result_str = str(result)
- assert "MESSAGE: CatalogException: Error updating authorization policy:" \
+ assert "MESSAGE: CatalogException: Error refreshing authorization policy:" \
in result_str
assert "CAUSED BY: ImpalaRuntimeException: Error refreshing authorization policy." \
" Sentry is unavailable. Ensure Sentry is up:" in result_str
@@ -510,3 +512,115 @@ class TestAuthorization(CustomClusterTestSuite):
self.execute_query_expect_success(self.client, "invalidate metadata")
finally:
self.role_cleanup(unique_role)
+
+ @pytest.mark.execute_serially
+ @CustomClusterTestSuite.with_args(
+ impalad_args="--server_name=server1 --sentry_config=%s" % SENTRY_CONFIG_FILE,
+ catalogd_args="--sentry_config=%s --sentry_catalog_polling_frequency_s=3600 " %
+ SENTRY_CONFIG_FILE,
+ impala_log_dir=tempfile.mkdtemp(prefix="test_refresh_authorization_",
+ dir=os.getenv("LOG_DIR")))
+ def test_refresh_authorization(self, unique_role):
+ """Tests refresh authorization statement by adding and removing roles and privileges
+ externally. The long Sentry polling is used so that any authorization metadata
+ updated externally does not get polled by Impala in order to test an an explicit
+ call to refresh authorization statement."""
+ group_name = grp.getgrnam(getuser()).gr_name
+ self.role_cleanup(unique_role)
+ for sync_ddl in [1, 0]:
+ query_options = {'sync_ddl': sync_ddl}
+ clients = []
+ if sync_ddl:
+ # When sync_ddl is True, we want to ensure the changes are propagated to all
+ # coordinators.
+ for impalad in self.cluster.impalads:
+ clients.append(impalad.service.create_beeswax_client())
+ else:
+ clients.append(self.client)
+ try:
+ self.client.execute("create role %s" % unique_role)
+ self.client.execute("grant role %s to group `%s`" % (unique_role, group_name))
+ self.client.execute("grant refresh on server to %s" % unique_role)
+
+ self.validate_refresh_authorization_roles(unique_role, query_options, clients)
+ self.validate_refresh_authorization_privileges(unique_role, query_options,
+ clients)
+ finally:
+ self.role_cleanup(unique_role)
+
+ def validate_refresh_authorization_roles(self, unique_role, query_options, clients):
+ """This method tests refresh authorization statement by adding and removing
+ roles externally."""
+ try:
+ # Create two roles inside Impala.
+ self.client.execute("create role %s_internal1" % unique_role)
+ self.client.execute("create role %s_internal2" % unique_role)
+ # Drop an existing role (_internal1) outside Impala.
+ role = "%s_internal1" % unique_role
+ subprocess.check_call(
+ ["/bin/bash", "-c",
+ "%s/bin/sentryShell --conf %s/sentry-site.xml -dr -r %s" %
+ (os.getenv("SENTRY_HOME"), os.getenv("SENTRY_CONF_DIR"), role)],
+ stdout=sys.stdout, stderr=sys.stderr)
+
+ result = self.execute_query_expect_success(self.client, "show roles")
+ assert any(role in x for x in result.data)
+ self.execute_query_expect_success(self.client, "refresh authorization",
+ query_options=query_options)
+ for client in clients:
+ result = self.execute_query_expect_success(client, "show roles")
+ assert not any(role in x for x in result.data)
+
+ # Add a new role outside Impala.
+ role = "%s_external" % unique_role
+ subprocess.check_call(
+ ["/bin/bash", "-c",
+ "%s/bin/sentryShell --conf %s/sentry-site.xml -cr -r %s" %
+ (os.getenv("SENTRY_HOME"), os.getenv("SENTRY_CONF_DIR"), role)],
+ stdout=sys.stdout, stderr=sys.stderr)
+
+ result = self.execute_query_expect_success(self.client, "show roles")
+ assert not any(role in x for x in result.data)
+ self.execute_query_expect_success(self.client, "refresh authorization",
+ query_options=query_options)
+ for client in clients:
+ result = self.execute_query_expect_success(client, "show roles")
+ assert any(role in x for x in result.data)
+ finally:
+ for suffix in ["internal1", "internal2", "external"]:
+ self.role_cleanup("%s_%s" % (unique_role, suffix))
+
+ def validate_refresh_authorization_privileges(self, unique_role, query_options,
+ clients):
+ """This method tests refresh authorization statement by adding and removing
+ privileges externally."""
+ # Grant select privilege outside Impala.
+ subprocess.check_call(
+ ["/bin/bash", "-c",
+ "%s/bin/sentryShell --conf %s/sentry-site.xml -gpr -p "
+ "'server=server1->db=functional->table=alltypes->action=select' -r %s" %
+ (os.getenv("SENTRY_HOME"), os.getenv("SENTRY_CONF_DIR"), unique_role)],
+ stdout=sys.stdout, stderr=sys.stderr)
+
+ # Before refresh authorization, there should only be one refresh privilege.
+ result = self.execute_query_expect_success(self.client, "show grant role %s" %
+ unique_role)
+ assert len(result.data) == 1
+ assert any("refresh" in x for x in result.data)
+
+ for client in clients:
+ self.execute_query_expect_failure(client,
+ "select * from functional.alltypes limit 1")
+
+ self.execute_query_expect_success(self.client, "refresh authorization",
+ query_options=query_options)
+
+ for client in clients:
+ # Ensure select privilege was granted after refresh authorization.
+ result = self.execute_query_expect_success(client, "show grant role %s" %
+ unique_role)
+ assert len(result.data) == 2
+ assert any("select" in x for x in result.data)
+ assert any("refresh" in x for x in result.data)
+ self.execute_query_expect_success(client,
+ "select * from functional.alltypes limit 1")
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/tests/authorization/test_grant_revoke.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_grant_revoke.py b/tests/authorization/test_grant_revoke.py
index d568859..abcd7d7 100644
--- a/tests/authorization/test_grant_revoke.py
+++ b/tests/authorization/test_grant_revoke.py
@@ -113,21 +113,18 @@ class TestGrantRevoke(SentryCacheTestSuite):
"""Tests grant and revoke for all objects. In these tests, we run tests twice. One
with just using cache, hence the long sentry poll, and another one by ensuring
refreshes happen from Sentry."""
- for invalidate in [True, False]:
+ for refresh in [True, False]:
for priv in ["all", "select"]:
- self._execute_with_grant_option_tests(TestObject(TestObject.SERVER), priv,
- invalidate)
- self._execute_with_grant_option_tests(TestObject(TestObject.DATABASE,
- "grant_rev_db"), priv,
- invalidate)
- self._execute_with_grant_option_tests(TestObject(TestObject.TABLE,
- "grant_rev_db.tbl1"), priv,
- invalidate)
- self._execute_with_grant_option_tests(TestObject(TestObject.VIEW,
- "grant_rev_db.tbl1"), priv,
- invalidate)
-
- def _execute_with_grant_option_tests(self, test_obj, privilege, invalidate_metadata):
+ self._execute_with_grant_option_tests(
+ TestObject(TestObject.SERVER), priv, refresh)
+ self._execute_with_grant_option_tests(TestObject(
+ TestObject.DATABASE, "grant_rev_db"), priv, refresh)
+ self._execute_with_grant_option_tests(TestObject(
+ TestObject.TABLE, "grant_rev_db.tbl1"), priv, refresh)
+ self._execute_with_grant_option_tests(
+ TestObject(TestObject.VIEW, "grant_rev_db.tbl1"), priv, refresh)
+
+ def _execute_with_grant_option_tests(self, test_obj, privilege, refresh_authorization):
"""
Executes grant/revoke tests with grant option.
"""
@@ -161,7 +158,7 @@ class TestGrantRevoke(SentryCacheTestSuite):
# Ensure role has privilege.
self.validate_privileges(self.client, "show grant role grant_revoke_test_role",
- test_obj, invalidate_metadata=invalidate_metadata)
+ test_obj, refresh_authorization=refresh_authorization)
# Try with grant option on existing privilege.
test_obj.grant = True
@@ -170,7 +167,7 @@ class TestGrantRevoke(SentryCacheTestSuite):
% (privilege, test_obj.grant_name, test_obj.obj_name), user="root")
# Ensure role has updated privilege.
self.validate_privileges(self.client, "show grant role grant_revoke_test_role",
- test_obj, invalidate_metadata=invalidate_metadata)
+ test_obj, refresh_authorization=refresh_authorization)
# Revoke the grant option
self.user_query(self.client, "revoke grant option for %s on %s %s from role "
@@ -180,7 +177,7 @@ class TestGrantRevoke(SentryCacheTestSuite):
# Ensure role has updated privilege.
test_obj.grant = False
self.validate_privileges(self.client, "show grant role grant_revoke_test_role",
- test_obj, invalidate_metadata=invalidate_metadata)
+ test_obj, refresh_authorization=refresh_authorization)
# Add the grant option back, then add a regular privilege
self.user_query(self.client,
@@ -190,7 +187,7 @@ class TestGrantRevoke(SentryCacheTestSuite):
(privilege, test_obj.grant_name, test_obj.obj_name), user="root")
test_obj.grant = True
self.validate_privileges(self.client, "show grant role grant_revoke_test_role",
- test_obj, invalidate_metadata=invalidate_metadata)
+ test_obj, refresh_authorization=refresh_authorization)
# Revoke the privilege
self.user_query(self.client, "revoke %s on %s %s from role grant_revoke_test_role" %
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/tests/authorization/test_owner_privileges.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_owner_privileges.py b/tests/authorization/test_owner_privileges.py
index 26e7a97..38bb8f7 100644
--- a/tests/authorization/test_owner_privileges.py
+++ b/tests/authorization/test_owner_privileges.py
@@ -94,8 +94,8 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
total += 1
return total
- def _validate_no_user_privileges(self, client, user, invalidate_metadata):
- if invalidate_metadata: self.execute_query("invalidate metadata")
+ def _validate_no_user_privileges(self, client, user, refresh_authorization):
+ if refresh_authorization: self.execute_query("refresh authorization")
result = self.user_query(client, "show grant user %s" % user, user=user)
return TestOwnerPrivileges.count_user_privileges(result) == 0
@@ -128,31 +128,31 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
sentry_log_dir="{0}/test_owner_privileges_with_grant".format(SENTRY_BASE_LOG_DIR))
def test_owner_privileges_with_grant(self, vector, unique_database):
"""Tests owner privileges with grant on database, table, and view.
- - invalidate_metadata=True: With Sentry refresh to make sure privileges are really
- stored in Sentry.
- - invalidate_metadata=False: No Sentry refresh to make sure user can use owner
- privileges right away without a Sentry refresh."""
- for invalidate in [True, False]:
+ - refresh_authorization=True: With Sentry refresh to make sure privileges are really
+ stored in Sentry.
+ - refresh_authorization=False: No Sentry refresh to make sure user can use owner
+ privileges right away without a Sentry refresh."""
+ for refresh in [True, False]:
try:
self._setup_ownership_test()
self._execute_owner_privilege_tests(TestObject(TestObject.DATABASE,
"owner_priv_db",
grant=True),
- invalidate_metadata=invalidate)
+ refresh_authorization=refresh)
self._execute_owner_privilege_tests(TestObject(TestObject.TABLE,
unique_database +
".owner_priv_tbl",
grant=True),
- invalidate_metadata=invalidate)
+ refresh_authorization=refresh)
self._execute_owner_privilege_tests(TestObject(TestObject.VIEW,
unique_database +
".owner_priv_view",
grant=True),
- invalidate_metadata=invalidate)
+ refresh_authorization=refresh)
finally:
self._cleanup_ownership_test()
- def _execute_owner_privilege_tests(self, test_obj, invalidate_metadata):
+ def _execute_owner_privilege_tests(self, test_obj, refresh_authorization):
"""
Executes all the statements required to validate owner privileges work correctly
for a specific database, table, or view.
@@ -166,7 +166,7 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
test_obj.view_select), user="oo_user1")
self.validate_privileges(self.oo_user1_impalad_client, "show grant user oo_user1",
test_obj, user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ refresh_authorization=refresh_authorization)
# Ensure grant works.
self.user_query(self.oo_user1_impalad_client,
@@ -180,11 +180,10 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
self.user_query(self.oo_user1_impalad_client, "alter %s %s set owner user oo_user2" %
(test_obj.obj_type, test_obj.obj_name), user="oo_user1")
assert self._validate_no_user_privileges(self.oo_user1_impalad_client,
- user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ user="oo_user1",
+ refresh_authorization=refresh_authorization)
# Ensure oo_user1 cannot drop database after owner change.
- # Use a delay to avoid cache consistency issue that could occur after alter.
self.user_query(self.oo_user1_impalad_client, "drop %s %s" %
(test_obj.obj_type, test_obj.obj_name), user="oo_user1",
error_msg="does not have privileges to execute 'DROP'")
@@ -192,7 +191,7 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
# oo_user2 should have privileges for object now.
self.validate_privileges(self.oo_user2_impalad_client, "show grant user oo_user2",
test_obj, user="oo_user2",
- invalidate_metadata=invalidate_metadata)
+ refresh_authorization=refresh_authorization)
# Change the owner to a role and ensure oo_user2 doesn't have privileges.
# Set the owner back to oo_user1 since for views, oo_user2 doesn't have select
@@ -201,40 +200,37 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
(test_obj.obj_type, test_obj.obj_name),
query_options={"sync_ddl": 1})
assert self._validate_no_user_privileges(self.oo_user2_impalad_client,
- user="oo_user2",
- invalidate_metadata=invalidate_metadata)
+ user="oo_user2",
+ refresh_authorization=refresh_authorization)
self.user_query(self.oo_user1_impalad_client,
"alter %s %s set owner role owner_priv_test_owner_role" %
(test_obj.obj_type, test_obj.obj_name), user="oo_user1")
# Ensure oo_user1 does not have user privileges.
assert self._validate_no_user_privileges(self.oo_user1_impalad_client,
- user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ user="oo_user1",
+ refresh_authorization=refresh_authorization)
# Ensure role has owner privileges.
self.validate_privileges(self.oo_user1_impalad_client,
"show grant role owner_priv_test_owner_role", test_obj,
- user="oo_user1", invalidate_metadata=invalidate_metadata)
+ user="oo_user1", refresh_authorization=refresh_authorization)
# Drop the object and ensure no role privileges.
- # Use a delay to avoid cache consistency issue that could occur after alter.
self.user_query(self.oo_user1_impalad_client, "drop %s %s " %
(test_obj.obj_type, test_obj.obj_name), user="oo_user1")
assert self._validate_no_user_privileges(self.oo_user1_impalad_client,
user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ refresh_authorization=refresh_authorization)
# Ensure user privileges are gone after drop.
- # Use a delay to avoid cache consistency issue that could occur after drop.
self.user_query(self.oo_user1_impalad_client, "create %s if not exists %s %s %s" %
(test_obj.obj_type, test_obj.obj_name, test_obj.table_def,
test_obj.view_select), user="oo_user1")
- # Use a delay to avoid cache consistency issue that could occur after create.
self.user_query(self.oo_user1_impalad_client, "drop %s %s " %
(test_obj.obj_type, test_obj.obj_name), user="oo_user1")
assert self._validate_no_user_privileges(self.oo_user1_impalad_client,
user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ refresh_authorization=refresh_authorization)
@pytest.mark.execute_serially
@SentryCacheTestSuite.with_args(
@@ -311,28 +307,28 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
.format(SENTRY_BASE_LOG_DIR))
def test_owner_privileges_without_grant(self, vector, unique_database):
"""Tests owner privileges without grant on database, table, and view.
- - invalidate_metadata=True: With Sentry refresh to make sure privileges are really
- stored in Sentry.
- - invalidate_metadata=False: No Sentry refresh to make sure user can use owner
- privileges right away without a Sentry refresh."""
- for invalidate in [True, False]:
+ - refresh_authorization=True: With Sentry refresh to make sure privileges are really
+ stored in Sentry.
+ - refresh_authorization=False: No Sentry refresh to make sure user can use owner
+ privileges right away without a Sentry refresh."""
+ for refresh in [True, False]:
try:
self._setup_ownership_test()
self._execute_owner_privilege_tests_oo_nogrant(TestObject(TestObject.DATABASE,
"owner_priv_db"),
- invalidate_metadata=invalidate)
+ refresh_authorization=refresh)
self._execute_owner_privilege_tests_oo_nogrant(TestObject(TestObject.TABLE,
unique_database +
".owner_priv_tbl"),
- invalidate_metadata=invalidate)
+ refresh_authorization=refresh)
self._execute_owner_privilege_tests_oo_nogrant(TestObject(TestObject.VIEW,
unique_database +
".owner_priv_view"),
- invalidate_metadata=invalidate)
+ refresh_authorization=refresh)
finally:
self._cleanup_ownership_test()
- def _execute_owner_privilege_tests_oo_nogrant(self, test_obj, invalidate_metadata):
+ def _execute_owner_privilege_tests_oo_nogrant(self, test_obj, refresh_authorization):
"""
Executes all the statements required to validate owner privileges work correctly
for a specific database, table, or view.
@@ -344,7 +340,7 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
test_obj.view_select), user="oo_user1")
self.validate_privileges(self.oo_user1_impalad_client, "show grant user oo_user1",
test_obj, user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ refresh_authorization=refresh_authorization)
# Ensure grant doesn't work.
self.user_query(self.oo_user1_impalad_client,
@@ -366,7 +362,7 @@ class TestOwnerPrivileges(SentryCacheTestSuite):
(test_obj.obj_type, test_obj.obj_name), user="oo_user1")
assert self._validate_no_user_privileges(self.oo_user1_impalad_client,
user="oo_user1",
- invalidate_metadata=invalidate_metadata)
+ refresh_authorization=refresh_authorization)
@pytest.mark.execute_serially
@SentryCacheTestSuite.with_args(
http://git-wip-us.apache.org/repos/asf/impala/blob/05b09f25/tests/common/sentry_cache_test_suite.py
----------------------------------------------------------------------
diff --git a/tests/common/sentry_cache_test_suite.py b/tests/common/sentry_cache_test_suite.py
index 4aca3be..2890911 100644
--- a/tests/common/sentry_cache_test_suite.py
+++ b/tests/common/sentry_cache_test_suite.py
@@ -55,15 +55,16 @@ class SentryCacheTestSuite(CustomClusterTestSuite):
return True
def validate_privileges(self, client, query, test_obj, user=None,
- invalidate_metadata=False):
- """Validate privileges. When invalidate_metadata is set to True, this function
- will call "invalidate metadata" to ensure the privileges get refreshed from Sentry.
+ refresh_authorization=False):
+ """Validate privileges. When refresh_authorization is set to True, this function
+ will call "refresh authorization" to ensure the privileges get refreshed from Sentry.
"""
show_user = True if 'show grant user' in query else False
- if invalidate_metadata: self.execute_query('invalidate metadata')
+ if refresh_authorization: self.execute_query('refresh authorization')
result = self.execute_query_expect_success(client, query, user=user)
return SentryCacheTestSuite.check_privileges(result, test_obj,
- null_create_date=not invalidate_metadata,
+ null_create_date=(not
+ refresh_authorization),
show_user=show_user)
def user_query(self, client, query, user=None, error_msg=None):