You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2018/08/16 00:29:45 UTC
[1/6] impala git commit: IMPALA-7206: [DOCS] THREAD_RESERVATION_LIMIT
& THREAD_RESERVATION_AGGREGATE_LIMIT
Repository: impala
Updated Branches:
refs/heads/master b1a5aebb8 -> 29905d1ea
IMPALA-7206: [DOCS] THREAD_RESERVATION_LIMIT & THREAD_RESERVATION_AGGREGATE_LIMIT
Change-Id: I94b560f6098711a6458ac5a7c8584f6fe3e3fdb8
Reviewed-on: http://gerrit.cloudera.org:8080/11237
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a7c20aef
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a7c20aef
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a7c20aef
Branch: refs/heads/master
Commit: a7c20aef8cd9f682ba8300bb6ff5ef18e661a107
Parents: b1a5aeb
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Wed Aug 15 13:59:53 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Wed Aug 15 21:53:38 2018 +0000
----------------------------------------------------------------------
docs/impala.ditamap | 2 +
...mpala_thread_reservation_aggregate_limit.xml | 87 ++++++++++++++++++
docs/topics/impala_thread_reservation_limit.xml | 97 ++++++++++++++++++++
3 files changed, 186 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/a7c20aef/docs/impala.ditamap
----------------------------------------------------------------------
diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index 68558fd..1ea0e6d 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -228,6 +228,8 @@ under the License.
<topicref href="topics/impala_shuffle_distinct_exprs.xml"/>
<topicref href="topics/impala_support_start_over.xml"/>
<topicref href="topics/impala_sync_ddl.xml"/>
+ <topicref href="topics/impala_thread_reservation_aggregate_limit.xml"/>
+ <topicref href="topics/impala_thread_reservation_limit.xml"/>
</topicref>
</topicref>
<topicref href="topics/impala_show.xml"/>
http://git-wip-us.apache.org/repos/asf/impala/blob/a7c20aef/docs/topics/impala_thread_reservation_aggregate_limit.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_thread_reservation_aggregate_limit.xml b/docs/topics/impala_thread_reservation_aggregate_limit.xml
new file mode 100644
index 0000000..fe08cc0
--- /dev/null
+++ b/docs/topics/impala_thread_reservation_aggregate_limit.xml
@@ -0,0 +1,87 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="thread_reservation_aggregate_limit">
+
+ <title>THREAD_RESERVATION_AGGREGATE_LIMIT Query Option (<keyword
+ keyref="impala31"/>or higher only)</title>
+
+ <titlealts audience="PDF">
+
+ <navtitle>THREAD_RESERVATION_AGGREGATE_LIMIT</navtitle>
+
+ </titlealts>
+
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Impala Query Options"/>
+ <data name="Category" value="Scalability"/>
+ <data name="Category" value="Memory"/>
+ <data name="Category" value="Troubleshooting"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Data Analysts"/>
+ </metadata>
+ </prolog>
+
+ <conbody>
+
+ <p>
+ The <codeph>THREAD_RESERVATION_AGGREGATE_LIMIT</codeph> query option limits the number of
+ reserved threads for a query across all nodes on which it is executing. The option is
+ intended to prevent execution of complex queries that can consume excessive CPU or
+ operating system resources on a cluster. Queries that have more threads than this
+ threshold are rejected by Impala’s admission controller before they start executing.
+ </p>
+
+ <p>
+ For example, an Impala administrator could set a default value of
+ <codeph>THREAD_RESERVATION_AGGREGATE_LIMIT=2000</codeph> for a resource pool on a 100 node
+ where they expect only relatively simple queries with less than 20 threads per node to
+ run. This will reject queries that require more than 2000 reserved threads across all
+ nodes, for example a query with 21 fragments running on all 100 nodes of the cluster.
+ </p>
+
+ <p>
+ You can override the default value per-query or per-session, in the same way as other
+ query options, if you do not want the default
+ <codeph>THREAD_RESERVATION_AGGREGATE_LIMIT</codeph> value to apply to a specific query or
+ session.
+ </p>
+
+ <p>
+ <b>Syntax:</b> <codeph>SET THREAD_RESERVATION_AGGREGATE_LIMIT=number;</codeph>
+ </p>
+
+ <p>
+ <b>Type:</b> numeric
+ </p>
+
+ <p>
+ <b>Default:</b> 0 (no limit)
+ </p>
+
+ <p>
+ <b>Added in:</b> <keyword keyref="impala31"/>
+ </p>
+
+ </conbody>
+
+</concept>
http://git-wip-us.apache.org/repos/asf/impala/blob/a7c20aef/docs/topics/impala_thread_reservation_limit.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_thread_reservation_limit.xml b/docs/topics/impala_thread_reservation_limit.xml
new file mode 100644
index 0000000..6730c4a
--- /dev/null
+++ b/docs/topics/impala_thread_reservation_limit.xml
@@ -0,0 +1,97 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="thread_reservation_limit">
+
+ <title>THREAD_RESERVATION_LIMIT Query Option (<keyword keyref="impala31"/>or higher only)</title>
+
+ <titlealts audience="PDF">
+
+ <navtitle>THREAD_RESERVATION_LIMIT</navtitle>
+
+ </titlealts>
+
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Impala Query Options"/>
+ <data name="Category" value="Scalability"/>
+ <data name="Category" value="Memory"/>
+ <data name="Category" value="Troubleshooting"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Data Analysts"/>
+ </metadata>
+ </prolog>
+
+ <conbody>
+
+ <p>
+ The <codeph>THREAD_RESERVATION_LIMIT</codeph> query option limits the number of reserved
+ threads for a query on each node. The option is intended to prevent execution of complex
+ queries that can consume excessive CPU or operating system resources on a single node.
+ Queries that have more threads per node than this threshold are rejected by Impala’s
+ admission controller before they start executing. You can see the number of reserved
+ threads for a query in its
+ <xref
+ href="https://www.cloudera.com/documentation/enterprise/latest/topics/impala_explain_plan.html"
+ format="html" scope="external">
+ <u>explain plan</u>
+ </xref> in the “Per-Host Resource Reservation" line.
+ </p>
+
+ <p>
+ For example, an Impala administrator could set a default value of
+ <codeph>THREAD_RESERVATION_LIMIT=100</codeph> for a resource pool where they expect only
+ relatively simple queries to run. This will reject queries that require more than 100
+ reserved threads on a node, for example, queries with more than 100 fragments.
+ </p>
+
+ <p>
+ You can override the default value per-query or per-session, in the same way as other
+ query options, if you do not want the default <codeph>THREAD_RESERVATION_LIMIT</codeph>
+ value to apply to a specific query or session.
+ </p>
+
+ <p>
+ <note>
+ The number of reserved threads on a node may be lower than the maximum value in the
+ explain plan if not all fragments of that query are scheduled on every node.
+ </note>
+ </p>
+
+ <p>
+ <b>Syntax:</b> <codeph>SET THREAD_RESERVATION_LIMIT=number;</codeph>
+ </p>
+
+ <p>
+ <b>Type:</b> numeric
+ </p>
+
+ <p rev="">
+ <b>Default:</b> 3000
+ </p>
+
+ <p>
+ <b>Added in:</b> <keyword keyref="impala31"/>
+ </p>
+
+ </conbody>
+
+</concept>
[2/6] impala git commit: IMPALA-6481: [DOCS] Documented WIDTH_BUCKET
function
Posted by ta...@apache.org.
IMPALA-6481: [DOCS] Documented WIDTH_BUCKET function
Change-Id: Ife9577a65fe342fde160c7cb5fa666e407d5b093
Reviewed-on: http://gerrit.cloudera.org:8080/11170
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Zoltan Borok-Nagy <bo...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/e4ea2317
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/e4ea2317
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/e4ea2317
Branch: refs/heads/master
Commit: e4ea23178663eaf1860add436897e20cbd2392e0
Parents: a7c20ae
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Wed Aug 8 15:52:10 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Wed Aug 15 22:00:13 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_math_functions.xml | 73 ++++++++++++++++++++++++++++++
1 file changed, 73 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/e4ea2317/docs/topics/impala_math_functions.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_math_functions.xml b/docs/topics/impala_math_functions.xml
index 430afe4..7961a11 100644
--- a/docs/topics/impala_math_functions.xml
+++ b/docs/topics/impala_math_functions.xml
@@ -1665,6 +1665,79 @@ select trunc(d) from dbl order by d;
</dd>
</dlentry>
+ <dlentry>
+ <dt>
+ <codeph>width_bucket(decimal expr, decimal min_value, decimal
+ max_value, int num_buckets)</codeph>
+ </dt>
+ <dd>
+ <b>Purpose:</b> Returns the bucket number in which the
+ <codeph>expr</codeph> value would fall in the histogram where its
+ range between <codeph>min_value</codeph> and
+ <codeph>max_value</codeph> is divided into
+ <codeph>num_buckets</codeph> buckets of identical sizes. </dd>
+ <dd>The function returns: <ul>
+ <li><codeph>NULL</codeph> if any argument is
+ <codeph>NULL</codeph>.</li>
+ <li><codeph>0</codeph> if <codeph>expr</codeph> <
+ <codeph>min_value</codeph>.</li>
+ <li><codeph>num_buckets + 1</codeph> if <codeph>expr</codeph> >=
+ <codeph>max_val</codeph>.</li>
+ <li>If none of the above, the bucket number where
+ <codeph>expr</codeph> falls.</li>
+ </ul>
+ <p>
+ <b>Arguments:</b>The following rules apply to the arguments. <ul>
+ <li>
+ <codeph>min_val</codeph> is the minimum value of the histogram
+ range. </li>
+ <li>
+ <codeph>max_val</codeph> is the maximum value of the histogram
+ range. </li>
+ <li>
+ <codeph>num_buckets</codeph> must be greater than
+ <codeph>0</codeph>. </li>
+ <li>
+ <codeph>min_value</codeph> must be less than
+ <codeph>max_value</codeph>. </li>
+ </ul>
+ </p>
+ <p>
+ <b>Usage notes:</b></p><p>Each bucket contains values equal to or
+ greater than the base value of that bucket and less than the base
+ value of the next bucket. For example, with <codeph>width_bucket(8,
+ 1, 10, 3)</codeph>, the bucket ranges are actually the 0th
+ "underflow bucket" with the range (-infinity to 0.999...), (1 to
+ 3.999...), (4, to 6.999...), (7 to 9.999...), and the "overflow
+ bucket" with the range (10 to infinity).</p>
+ <p>
+ <b>Return type:</b>
+ <codeph>bigint</codeph>
+ </p>
+ <p>
+ <b>Added in:</b>
+ <keyword keyref="impala31"/>. </p>
+ <p>
+ <b>Examples:</b>
+ </p>
+ <p> The below function creates <codeph>3</codeph> buckets between the
+ range of <codeph>1</codeph> and <codeph>20</codeph> with the bucket
+ width of 6.333, and returns <codeph>2</codeph> for the bucket #2
+ where the value <codeph>8</codeph> falls
+ in:<codeblock>width_bucket(8, 1, 20, 3)</codeblock>
+ </p>
+ <p> The below statement returns a list of accounts with the energy
+ spending and the spending bracket each account falls in, between 0
+ and 11. Bucket 0 (underflow bucket) will be assigned to the accounts
+ whose energy spendings are less than $50. Bucket 11 (overflow
+ bucket) will be assigned to the accounts whose energy spendings are
+ more than or equal to $1000.
+ </p>
+<codeblock>SELECT account, invoice_amount, WIDTH_BUCKET(invoice_amount,50,1000,10)
+FROM invoices_june2018
+ORDER BY 3;</codeblock>
+ </dd>
+ </dlentry>
</dl>
</conbody>
</concept>
[4/6] impala git commit: IMPALA-7342: Add initial support for
user-level permissions
Posted by ta...@apache.org.
IMPALA-7342: Add initial support for user-level permissions
This patch refactors the authorization code in preparation to add initial
support for for user-level permissions (IMPALA-6794) and object ownership
(IMPALA-7075). It introduces the notion of Principal that can be either
Role or User. The authorization tests are updated to run the tests with
user and role permissions.
Testing:
- Update authorization tests
- Ran core tests
Change-Id: I07e0d46d2e50d35bd64ee573b5aa4b779eb9e62f
Reviewed-on: http://gerrit.cloudera.org:8080/11039
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a23e6f29
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a23e6f29
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a23e6f29
Branch: refs/heads/master
Commit: a23e6f296369854b7fade98bf476242c1201dacc
Parents: e4ea231
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Tue Jul 24 14:56:28 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Wed Aug 15 22:02:46 2018 +0000
----------------------------------------------------------------------
be/src/catalog/catalog-util.cc | 13 +-
common/thrift/CatalogObjects.thrift | 61 ++--
.../impala/analysis/GrantRevokePrivStmt.java | 3 +-
.../apache/impala/analysis/PrivilegeSpec.java | 4 +-
.../impala/analysis/ShowGrantRoleStmt.java | 3 +-
.../impala/catalog/AuthorizationPolicy.java | 311 +++++++++++++------
.../java/org/apache/impala/catalog/Catalog.java | 44 +--
.../impala/catalog/CatalogServiceCatalog.java | 200 ++++++++----
.../apache/impala/catalog/ImpaladCatalog.java | 44 +--
.../org/apache/impala/catalog/Principal.java | 181 +++++++++++
.../impala/catalog/PrincipalPrivilege.java | 154 +++++++++
.../java/org/apache/impala/catalog/Role.java | 128 +-------
.../apache/impala/catalog/RolePrivilege.java | 151 ---------
.../java/org/apache/impala/catalog/User.java | 39 +++
.../impala/service/CatalogOpExecutor.java | 11 +-
.../org/apache/impala/service/JniFrontend.java | 1 -
.../apache/impala/util/SentryPolicyService.java | 8 +-
.../org/apache/impala/util/SentryProxy.java | 47 +--
.../impala/analysis/AnalyzeAuthStmtsTest.java | 4 +-
.../impala/analysis/AuthorizationStmtTest.java | 247 ++++++++++-----
.../org/apache/impala/catalog/CatalogTest.java | 87 ++++++
.../impala/testutil/ImpaladTestCatalog.java | 20 +-
22 files changed, 1146 insertions(+), 615 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/be/src/catalog/catalog-util.cc
----------------------------------------------------------------------
diff --git a/be/src/catalog/catalog-util.cc b/be/src/catalog/catalog-util.cc
index 5d1c0fa..ad2024b 100644
--- a/be/src/catalog/catalog-util.cc
+++ b/be/src/catalog/catalog-util.cc
@@ -135,8 +135,8 @@ TCatalogObjectType::type TCatalogObjectTypeFromName(const string& name) {
return TCatalogObjectType::DATA_SOURCE;
} else if (upper == "HDFS_CACHE_POOL") {
return TCatalogObjectType::HDFS_CACHE_POOL;
- } else if (upper == "ROLE") {
- return TCatalogObjectType::ROLE;
+ } else if (upper == "PRINCIPAL") {
+ return TCatalogObjectType::PRINCIPAL;
} else if (upper == "PRIVILEGE") {
return TCatalogObjectType::PRIVILEGE;
}
@@ -196,10 +196,10 @@ Status TCatalogObjectFromObjectName(const TCatalogObjectType::type& object_type,
catalog_object->__set_cache_pool(THdfsCachePool());
catalog_object->cache_pool.__set_pool_name(object_name);
break;
- case TCatalogObjectType::ROLE:
+ case TCatalogObjectType::PRINCIPAL:
catalog_object->__set_type(object_type);
- catalog_object->__set_role(TRole());
- catalog_object->role.__set_role_name(object_name);
+ catalog_object->__set_principal(TPrincipal());
+ catalog_object->principal.__set_principal_name(object_name);
break;
case TCatalogObjectType::PRIVILEGE: {
int pos = object_name.find(".");
@@ -210,7 +210,8 @@ Status TCatalogObjectFromObjectName(const TCatalogObjectType::type& object_type,
}
catalog_object->__set_type(object_type);
catalog_object->__set_privilege(TPrivilege());
- catalog_object->privilege.__set_role_id(atoi(object_name.substr(0, pos).c_str()));
+ catalog_object->privilege.__set_principal_id(
+ atoi(object_name.substr(0, pos).c_str()));
catalog_object->privilege.__set_privilege_name(object_name.substr(pos + 1));
break;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/common/thrift/CatalogObjects.thrift
----------------------------------------------------------------------
diff --git a/common/thrift/CatalogObjects.thrift b/common/thrift/CatalogObjects.thrift
index b73c5e6..29f54e4 100644
--- a/common/thrift/CatalogObjects.thrift
+++ b/common/thrift/CatalogObjects.thrift
@@ -36,7 +36,7 @@ enum TCatalogObjectType {
VIEW,
FUNCTION,
DATA_SOURCE,
- ROLE,
+ PRINCIPAL,
PRIVILEGE,
HDFS_CACHE_POOL,
}
@@ -477,18 +477,28 @@ struct TDatabase {
2: optional hive_metastore.Database metastore_db
}
-// Represents a role in an authorization policy.
-struct TRole {
- // Case-insensitive role name
- 1: required string role_name
+// Represents a principal type that maps to Sentry principal type.
+// https://github.com/apache/sentry/blob/3d062f39ce6a047138660a7b3d0024bde916c5b4/sentry-service/sentry-service-api/src/gen/thrift/gen-javabean/org/apache/sentry/api/service/thrift/TSentryPrincipalType.java
+enum TPrincipalType {
+ ROLE,
+ USER
+}
+
+// Represents a principal in an authorization policy.
+struct TPrincipal {
+ // Case-insensitive principal name
+ 1: required string principal_name
+
+ // Unique ID of this principal, generated by the Catalog Server.
+ 2: required i32 principal_id
- // Unique ID of this role, generated by the Catalog Server.
- 2: required i32 role_id
+ // Type of this principal.
+ 3: required TPrincipalType principal_type
- // List of groups this role has been granted to (group names are case sensitive).
+ // List of groups this principal has been granted to (group names are case sensitive).
// TODO: Keep a list of grant groups globally (in TCatalog?) and reference by ID since
- // the same groups will likely be shared across multiple roles.
- 3: required list<string> grant_groups
+ // the same groups will likely be shared across multiple principals.
+ 4: required list<string> grant_groups
}
// The scope a TPrivilege applies to.
@@ -512,12 +522,12 @@ enum TPrivilegeLevel {
}
// Represents a privilege in an authorization policy. Privileges contain the level
-// of access, the scope and role the privilege applies to, and details on what
+// of access, the scope and principal the privilege applies to, and details on what
// catalog object the privilege is securing. Objects are hierarchical, so a privilege
// corresponding to a table must also specify all the parent objects (database name
// and server name).
struct TPrivilege {
- // A human readable name for this privilege. The combination of role_id +
+ // A human readable name for this privilege. The combination of principal_id +
// privilege_name is guaranteed to be unique. Stored in a form that can be passed
// to Sentry: [ServerName]->[DbName]->[TableName]->[ColumnName]->[Action Granted].
1: required string privilege_name
@@ -529,32 +539,35 @@ struct TPrivilege {
3: required TPrivilegeScope scope
// If true, GRANT OPTION was specified. For a GRANT privilege statement, everyone
- // granted this role should be able to issue GRANT/REVOKE privilege statements even if
- // they are not an admin. For REVOKE privilege statements, the privilege should be
+ // granted this principal should be able to issue GRANT/REVOKE privilege statements even
+ // if they are not an admin. For REVOKE privilege statements, the privilege should be
// retainined and the existing GRANT OPTION (if it was set) on the privilege should be
// removed.
4: required bool has_grant_opt
- // The ID of the role this privilege belongs to.
- 5: optional i32 role_id
+ // The ID of the principal this privilege belongs to.
+ 5: optional i32 principal_id
+
+ // The type of the principal this privilege belongs to.
+ 6: optional TPrincipalType principal_type
// Set if scope is SERVER, URI, DATABASE, or TABLE
- 6: optional string server_name
+ 7: optional string server_name
// Set if scope is DATABASE or TABLE
- 7: optional string db_name
+ 8: optional string db_name
// Unqualified table name. Set if scope is TABLE.
- 8: optional string table_name
+ 9: optional string table_name
// Set if scope is URI
- 9: optional string uri
+ 10: optional string uri
// Time this privilege was created (in milliseconds since epoch).
- 10: optional i64 create_time_ms
+ 11: optional i64 create_time_ms
// Set if scope is COLUMN
- 11: optional string column_name
+ 12: optional string column_name
}
// Thrift representation of an HdfsCachePool.
@@ -595,8 +608,8 @@ struct TCatalogObject {
// Set iff object type is DATA SOURCE
7: optional TDataSource data_source
- // Set iff object type is ROLE
- 8: optional TRole role
+ // Set iff object type is PRINCIPAL
+ 8: optional TPrincipal principal
// Set iff object type is PRIVILEGE
9: optional TPrivilege privilege
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java b/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
index 9ded066..348383c 100644
--- a/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/GrantRevokePrivStmt.java
@@ -60,7 +60,8 @@ public class GrantRevokePrivStmt extends AuthorizationStmt {
params.setIs_grant(isGrantPrivStmt_);
List<TPrivilege> privileges = privilegeSpec_.toThrift();
for (TPrivilege privilege: privileges) {
- privilege.setRole_id(role_.getId());
+ privilege.setPrincipal_id(role_.getId());
+ privilege.setPrincipal_type(role_.getPrincipalType());
privilege.setHas_grant_opt(hasGrantOpt_);
}
params.setHas_grant_opt(hasGrantOpt_);
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
index fb75398..4ee5d49 100644
--- a/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
+++ b/fe/src/main/java/org/apache/impala/analysis/PrivilegeSpec.java
@@ -23,7 +23,7 @@ import org.apache.impala.authorization.Privilege;
import org.apache.impala.catalog.FeDataSourceTable;
import org.apache.impala.catalog.FeTable;
import org.apache.impala.catalog.FeView;
-import org.apache.impala.catalog.RolePrivilege;
+import org.apache.impala.catalog.PrincipalPrivilege;
import org.apache.impala.catalog.TableLoadingException;
import org.apache.impala.common.AnalysisException;
import org.apache.impala.thrift.TPrivilege;
@@ -134,7 +134,7 @@ public class PrivilegeSpec implements ParseNode {
if (uri_ != null) privilege.setUri(uri_.toString());
if (columnName != null) privilege.setColumn_name(columnName);
privilege.setCreate_time_ms(-1);
- privilege.setPrivilege_name(RolePrivilege.buildRolePrivilegeName(privilege));
+ privilege.setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(privilege));
return privilege;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java b/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
index 82c6ed0..749bcc2 100644
--- a/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
+++ b/fe/src/main/java/org/apache/impala/analysis/ShowGrantRoleStmt.java
@@ -49,7 +49,8 @@ public class ShowGrantRoleStmt extends AuthorizationStmt {
params.setRequesting_user(requestingUser_.getShortName());
if (privilegeSpec_ != null) {
params.setPrivilege(privilegeSpec_.toThrift().get(0));
- params.getPrivilege().setRole_id(role_.getId());
+ params.getPrivilege().setPrincipal_id(role_.getId());
+ params.getPrivilege().setPrincipal_type(role_.getPrincipalType());
}
return params;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java b/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
index 8c545fc..a151eb4 100644
--- a/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
+++ b/fe/src/main/java/org/apache/impala/catalog/AuthorizationPolicy.java
@@ -21,16 +21,18 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import com.google.common.base.Preconditions;
import org.apache.commons.net.ntp.TimeStamp;
-import org.apache.log4j.Logger;
-import org.apache.sentry.core.common.ActiveRoleSet;
-import org.apache.sentry.provider.cache.PrivilegeCache;
-
import org.apache.impala.thrift.TColumn;
+import org.apache.impala.thrift.TPrincipalType;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TResultRow;
import org.apache.impala.thrift.TResultSet;
import org.apache.impala.thrift.TResultSetMetadata;
+import org.apache.log4j.Logger;
+import org.apache.sentry.core.common.ActiveRoleSet;
+import org.apache.sentry.provider.cache.PrivilegeCache;
+
import org.apache.impala.util.TResultRowBuilder;
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
@@ -39,17 +41,24 @@ import com.google.common.collect.Sets;
/**
* A thread safe authorization policy cache, consisting of roles, groups that are
- * members of that role, and the privileges associated with the role. The source data
- * this cache is backing is read from the Sentry Policy Service. Writing to the cache
- * will replace any matching items, but will not write back to the Sentry Policy Service.
- * A role can have 0 or more privileges and roles are stored in a map of role name
- * to role object. For example:
- * RoleName -> Role -> [RolePriv1, ..., RolePrivN]
+ * members of that role, the privileges associated with the role, and users and the
+ * privileges associated with the user. The source data this cache is backing is read from
+ * the Sentry Policy Service. Writing to the cache will replace any matching items, but
+ * will not write back to the Sentry Policy Service.
+ * The roleCache_ contains all roles defined in Sentry whereas userCache_ only contains
+ * users that have privileges defined in Sentry and does not represent all users in the
+ * system. A principal type can have 0 or more privileges and principal types are stored
+ * in a map of principal name to principal object. For example:
+ * RoleName -> Role -> [PrincipalPriv1, ..., PrincipalPrivN]
+ * UserName -> User -> [PrincipalPriv1, ..., PrincipalPrivN]
+ * There is a separate cache for users since we cannot guarantee uniqueness between
+ * user names and role names.
* To ensure we can efficiently retrieve the roles that a user is a member of, a map
* of user group name to role name is tracked as grantGroups_.
- * To reduce duplication of metadata, privileges are linked to roles using a "role ID"
- * rather than embedding the role name. When a privilege is added to a role, we do
- * a lookup to get the role ID to using the roleIds_ map.
+ * To reduce duplication of metadata, privileges are linked to roles/users using a
+ * "principal ID" rather than embedding the principal name. When a privilege is added to
+ * a principal type, we do a lookup to get the principal ID to probe the principalIds_
+ * map.
* Acts as the backing cache for the Sentry cached based provider (which is why
* PrivilegeCache is implemented).
* TODO: Instead of calling into Sentry to perform final authorization checks, we
@@ -58,11 +67,16 @@ import com.google.common.collect.Sets;
public class AuthorizationPolicy implements PrivilegeCache {
private static final Logger LOG = Logger.getLogger(AuthorizationPolicy.class);
+ // Need to keep separate caches of role names and user names since there is
+ // no uniqueness guarantee across roles and users
// Cache of role names (case-insensitive) to role objects.
- private final CatalogObjectCache<Role> roleCache_ = new CatalogObjectCache<Role>();
+ private final CatalogObjectCache<Role> roleCache_ = new CatalogObjectCache<>();
+
+ // Cache of user names (case-insensitive) to user objects.
+ private final CatalogObjectCache<User> userCache_ = new CatalogObjectCache<>();
- // Map of role ID -> role name. Used to match privileges to roles.
- Map<Integer, String> roleIds_ = Maps.newHashMap();
+ // Map of principal ID -> user/role name. Used to match privileges to users/roles.
+ private final Map<Integer, String> principalIds_ = Maps.newHashMap();
// Map of group name (case sensitive) to set of role names (case insensitive) that
// have been granted to this group. Kept in sync with roleCache_. Provides efficient
@@ -70,87 +84,100 @@ public class AuthorizationPolicy implements PrivilegeCache {
Map<String, Set<String>> groupsToRoles_ = Maps.newHashMap();
/**
- * Adds a new role to the policy. If a role with the same name already
- * exists and the role ID's are different, it will be overwritten by the new role.
- * If a role exists and the role IDs are the same, the privileges from the old
- * role will be copied to the new role.
+ * Adds a new principal to the policy. If a principal with the same name already
+ * exists and the principal ID's are different, it will be overwritten by the new
+ * principal. If a principal exists and the principal IDs are the same, the privileges
+ * from the old principal will be copied to the new principal.
*/
- public synchronized void addRole(Role role) {
- Role existingRole = roleCache_.get(role.getName());
- // There is already a newer version of this role in the catalog, ignore
+ public synchronized void addPrincipal(Principal principal) {
+ Principal existingPrincipal = getPrincipal(principal.getName(),
+ principal.getPrincipalType());
+ // There is already a newer version of this principal in the catalog, ignore
// just return.
- if (existingRole != null &&
- existingRole.getCatalogVersion() >= role.getCatalogVersion()) return;
-
- // If there was an existing role that was replaced we first need to remove it.
- if (existingRole != null) {
- // Remove the role. This will also clean up the grantGroup mappings.
- removeRole(existingRole.getName());
- CatalogObjectVersionSet.INSTANCE.removeAll(existingRole.getPrivileges());
- if (existingRole.getId() == role.getId()) {
- // Copy the privileges from the existing role.
- for (RolePrivilege p: existingRole.getPrivileges()) {
- role.addPrivilege(p);
+ if (existingPrincipal != null &&
+ existingPrincipal.getCatalogVersion() >= principal.getCatalogVersion()) return;
+
+ // If there was an existing principal that was replaced we first need to remove it.
+ if (existingPrincipal != null) {
+ // Remove the principal. This will also clean up the grantGroup mappings.
+ removePrincipal(existingPrincipal.getName(), existingPrincipal.getPrincipalType());
+ CatalogObjectVersionSet.INSTANCE.removeAll(existingPrincipal.getPrivileges());
+ if (existingPrincipal.getId() == principal.getId()) {
+ // Copy the privileges from the existing principal.
+ for (PrincipalPrivilege p: existingPrincipal.getPrivileges()) {
+ principal.addPrivilege(p);
}
}
}
- roleCache_.add(role);
+ if (principal.getPrincipalType() == TPrincipalType.USER) {
+ Preconditions.checkArgument(principal instanceof User);
+ userCache_.add((User) principal);
+ } else {
+ Preconditions.checkArgument(principal instanceof Role);
+ roleCache_.add((Role) principal);
+ }
// Add new grants
- for (String groupName: role.getGrantGroups()) {
+ for (String groupName: principal.getGrantGroups()) {
Set<String> grantedRoles = groupsToRoles_.get(groupName);
if (grantedRoles == null) {
grantedRoles = Sets.newHashSet();
groupsToRoles_.put(groupName, grantedRoles);
}
- grantedRoles.add(role.getName().toLowerCase());
+ grantedRoles.add(principal.getName().toLowerCase());
}
- // Add this role to the role ID mapping
- roleIds_.put(role.getId(), role.getName());
+ // Add this principal to the principal ID mapping
+ principalIds_.put(principal.getId(), principal.getName());
}
/**
- * Adds a new privilege to the policy mapping to the role specified by the
- * role ID in the privilege.
- * Throws a CatalogException no role with a corresponding ID existing in the catalog.
+ * Adds a new privilege to the policy mapping to the principal specified by the
+ * principal ID in the privilege. Throws a CatalogException no principal with a
+ * corresponding ID existing in the catalog.
*/
- public synchronized void addPrivilege(RolePrivilege privilege)
+ public synchronized void addPrivilege(PrincipalPrivilege privilege)
throws CatalogException {
if (LOG.isTraceEnabled()) {
- LOG.trace("Adding privilege: " + privilege.getName() +
- " role ID: " + privilege.getRoleId());
+ LOG.trace("Adding privilege: " + privilege.getName() + " " +
+ Principal.toString(privilege.getPrincipalType()).toLowerCase() +
+ " ID: " + privilege.getPrincipalId());
}
- Role role = getRole(privilege.getRoleId());
- if (role == null) {
- throw new CatalogException(String.format("Error adding privilege: %s. Role ID " +
- "'%d' does not exist.", privilege.getName(), privilege.getRoleId()));
+ Principal principal = getPrincipal(privilege.getPrincipalId(),
+ privilege.getPrincipalType());
+ if (principal == null) {
+ throw new CatalogException(String.format("Error adding privilege: %s. %s ID " +
+ "'%d' does not exist.", privilege.getName(),
+ Principal.toString(privilege.getPrincipalType()), privilege.getPrincipalId()));
}
if (LOG.isTraceEnabled()) {
- LOG.trace("Adding privilege: " + privilege.getName() + " to role: " +
- role.getName() + "ID: " + role.getId());
+ LOG.trace("Adding privilege: " + privilege.getName() + " to " +
+ Principal.toString(privilege.getPrincipalType()).toLowerCase() + ": " +
+ principal.getName() + " with ID: " + principal.getId());
}
- role.addPrivilege(privilege);
+ principal.addPrivilege(privilege);
}
/**
- * Removes a privilege from the policy mapping to the role specified by the
- * role ID in the privilege.
- * Throws a CatalogException if no role with a corresponding ID exists in the catalog.
- * Returns null if no matching privilege is found in this role.
+ * Removes a privilege from the policy mapping to the role specified by the principal ID
+ * in the privilege. Throws a CatalogException if no role with a corresponding ID exists
+ * in the catalog. Returns null if no matching privilege is found in this principal.
*/
- public synchronized RolePrivilege removePrivilege(RolePrivilege privilege)
+ public synchronized PrincipalPrivilege removePrivilege(PrincipalPrivilege privilege)
throws CatalogException {
- Role role = getRole(privilege.getRoleId());
- if (role == null) {
- throw new CatalogException(String.format("Error removing privilege: %s. Role ID " +
- "'%d' does not exist.", privilege.getName(), privilege.getRoleId()));
+ Principal principal = getPrincipal(privilege.getPrincipalId(),
+ privilege.getPrincipalType());
+ if (principal == null) {
+ throw new CatalogException(String.format("Error removing privilege: %s. %s ID " +
+ "'%d' does not exist.", privilege.getName(),
+ Principal.toString(privilege.getPrincipalType()), privilege.getPrincipalId()));
}
if (LOG.isTraceEnabled()) {
- LOG.trace("Removing privilege: '" + privilege.getName() + "' from Role ID: " +
- privilege.getRoleId() + " Role Name: " + role.getName());
+ LOG.trace("Removing privilege: " + privilege.getName() + " from " +
+ Principal.toString(privilege.getPrincipalType()).toLowerCase() + ": " +
+ principal.getName() + " with ID: " + principal.getId());
}
- return role.removePrivilege(privilege.getName());
+ return principal.removePrivilege(privilege.getName());
}
/**
@@ -161,6 +188,13 @@ public class AuthorizationPolicy implements PrivilegeCache {
}
/**
+ * Returns all users in the policy. Returns an empty list if no users exist.
+ */
+ public synchronized List<User> getAllUsers() {
+ return userCache_.getValues();
+ }
+
+ /**
* Returns all role names in the policy. Returns an empty set if no roles exist.
*/
public synchronized Set<String> getAllRoleNames() {
@@ -168,30 +202,62 @@ public class AuthorizationPolicy implements PrivilegeCache {
}
/**
- * Gets a role given a role name. Returns null if no roles exist with this name.
+ * Gets a role given a role name. Returns null if no role exist with this name.
*/
public synchronized Role getRole(String roleName) {
return roleCache_.get(roleName);
}
/**
- * Gets a role given a role ID. Returns null if no roles exist with this ID.
+ * Gets a role given a role ID. Returns null if no role exists with this ID.
*/
public synchronized Role getRole(int roleId) {
- String roleName = roleIds_.get(roleId);
+ String roleName = principalIds_.get(roleId);
if (roleName == null) return null;
return roleCache_.get(roleName);
}
/**
- * Gets a privilege from the given role ID. Returns null of there are no roles with a
- * matching ID or if no privilege with this name exists for the role.
+ * Returns all user names in the policy. Returns an empty set if no users exist.
*/
- public synchronized RolePrivilege getPrivilege(int roleId, String privilegeName) {
- String roleName = roleIds_.get(roleId);
- if (roleName == null) return null;
- Role role = roleCache_.get(roleName);
- return role.getPrivilege(privilegeName);
+ public synchronized Set<String> getAllUserNames() {
+ return Sets.newHashSet(userCache_.keySet());
+ }
+
+ /**
+ * Gets a user given a user name. Returns null if no user exist with this name.
+ */
+ public synchronized User getUser(String userName) {
+ return userCache_.get(userName);
+ }
+
+ /**
+ * Gets a user given a user ID. Returns null if no user exists with this ID.
+ */
+ public synchronized User getUser(int userId) {
+ String userName = principalIds_.get(userId);
+ if (userName == null) return null;
+ return userCache_.get(userName);
+ }
+
+
+ /**
+ * Gets a principal given a principal name and type. Returns null if no principal exists
+ * with this name and type.
+ */
+ public synchronized Principal getPrincipal(String principalName, TPrincipalType type) {
+ return type == TPrincipalType.ROLE ?
+ roleCache_.get(principalName) : userCache_.get(principalName);
+ }
+
+ /**
+ * Gets a principal given a principal ID and type. Returns null if no principal exists
+ * with this ID and type.
+ */
+ public synchronized Principal getPrincipal(int principalId, TPrincipalType type) {
+ String principalName = principalIds_.get(principalId);
+ if (principalName == null) return null;
+ return getPrincipal(principalName, type);
}
/**
@@ -203,7 +269,7 @@ public class AuthorizationPolicy implements PrivilegeCache {
if (roleNames != null) {
for (String roleName: roleNames) {
// TODO: verify they actually exist.
- Role role = roleCache_.get(roleName);
+ Principal role = roleCache_.get(roleName);
if (role != null) grantedRoles.add(roleCache_.get(roleName));
}
}
@@ -211,6 +277,16 @@ public class AuthorizationPolicy implements PrivilegeCache {
}
/**
+ * Removes a principal for a given principal name and type. Returns the removed
+ * principal or null if no principal with this name and type existed.
+ */
+ public synchronized Principal removePrincipal(String principalName,
+ TPrincipalType type) {
+ return type == TPrincipalType.ROLE ?
+ removeRole(principalName) : removeUser(principalName);
+ }
+
+ /**
* Removes a role. Returns the removed role or null if no role with
* this name existed.
*/
@@ -223,17 +299,29 @@ public class AuthorizationPolicy implements PrivilegeCache {
Set<String> roles = groupsToRoles_.get(grantGroup);
if (roles != null) roles.remove(roleName.toLowerCase());
}
- // Cleanup role id.
- roleIds_.remove(removedRole.getId());
+ // Cleanup role ID.
+ principalIds_.remove(removedRole.getId());
return removedRole;
}
/**
+ * Removes a user. Returns the removed user or null if no user with
+ * this name existed.
+ */
+ public synchronized User removeUser(String userName) {
+ User removedUser = userCache_.remove(userName);
+ if (removedUser == null) return null;
+ // Cleanup user ID.
+ principalIds_.remove(removedUser.getId());
+ return removedUser;
+ }
+
+ /**
* Adds a new grant group to the specified role. Returns the updated
- * Role, if a matching role was found. If the role does not exist a
+ * Principal, if a matching role was found. If the role does not exist a
* CatalogException is thrown.
*/
- public synchronized Role addGrantGroup(String roleName, String groupName)
+ public synchronized Role addRoleGrantGroup(String roleName, String groupName)
throws CatalogException {
Role role = roleCache_.get(roleName);
if (role == null) throw new CatalogException("Role does not exist: " + roleName);
@@ -249,10 +337,10 @@ public class AuthorizationPolicy implements PrivilegeCache {
/**
* Removes a grant group from the specified role. Returns the updated
- * Role, if a matching role was found. If the role does not exist a
+ * Principal, if a matching role was found. If the role does not exist a
* CatalogException is thrown.
*/
- public synchronized Role removeGrantGroup(String roleName, String groupName)
+ public synchronized Role removeRoleGrantGroup(String roleName, String groupName)
throws CatalogException {
Role role = roleCache_.get(roleName);
if (role == null) throw new CatalogException("Role does not exist: " + roleName);
@@ -268,8 +356,8 @@ public class AuthorizationPolicy implements PrivilegeCache {
* Returns a set of privilege strings in Sentry format.
*/
@Override
- public synchronized Set<String>
- listPrivileges(Set<String> groups, ActiveRoleSet roleSet) {
+ public synchronized Set<String> listPrivileges(Set<String> groups,
+ ActiveRoleSet roleSet) {
Set<String> privileges = Sets.newHashSet();
if (roleSet != ActiveRoleSet.ALL) {
throw new UnsupportedOperationException("Impala does not support role subsets.");
@@ -279,7 +367,7 @@ public class AuthorizationPolicy implements PrivilegeCache {
for (String groupName: groups) {
List<Role> grantedRoles = getGrantedRoles(groupName);
for (Role role: grantedRoles) {
- for (RolePrivilege privilege: role.getPrivileges()) {
+ for (PrincipalPrivilege privilege: role.getPrivileges()) {
String authorizeable = privilege.getName();
if (authorizeable == null) {
if (LOG.isTraceEnabled()) {
@@ -294,17 +382,29 @@ public class AuthorizationPolicy implements PrivilegeCache {
return privileges;
}
- /**
+ /**
* Returns a set of privilege strings in Sentry format.
*/
- // This is an override for Sentry 2.1, but not for Sentry 1.x; we
- // avoid annotation to support both.
- // @Override
+ @Override
public Set<String> listPrivileges(Set<String> groups, Set<String> users,
ActiveRoleSet roleSet) {
- /* User based roles and authorization hierarchy is not currently supported.
- Fallback to listing privileges using groups. */
- return listPrivileges(groups, roleSet);
+ Set<String> privileges = listPrivileges(groups, roleSet);
+ for (String userName: users) {
+ User user = getUser(userName);
+ if (user != null) {
+ for (PrincipalPrivilege privilege: user.getPrivileges()) {
+ String authorizeable = privilege.getName();
+ if (authorizeable == null) {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Ignoring invalid privilege: " + privilege.getName());
+ }
+ continue;
+ }
+ privileges.add(authorizeable);
+ }
+ }
+ }
+ return privileges;
}
@Override
@@ -318,6 +418,25 @@ public class AuthorizationPolicy implements PrivilegeCache {
* granted to the role. Used by the SHOW GRANT ROLE statement.
*/
public synchronized TResultSet getRolePrivileges(String roleName, TPrivilege filter) {
+ return getPrincipalPrivileges(roleName, filter, TPrincipalType.ROLE);
+ }
+
+ /**
+ * Returns the privileges that have been granted to a user as a tabular result set.
+ * Allows for filtering based on a specific privilege spec or showing all privileges
+ * granted to the user. Used by the SHOW GRANT USER statement.
+ */
+ public synchronized TResultSet getUserPrivileges(String userName, TPrivilege filter) {
+ return getPrincipalPrivileges(userName, filter, TPrincipalType.USER);
+ }
+
+ /**
+ * Returns the privileges that have been granted to a principal as a tabular result set.
+ * Allows for filtering based on a specific privilege spec or showing all privileges
+ * granted to the principal.
+ */
+ private TResultSet getPrincipalPrivileges(String principalName, TPrivilege filter,
+ TPrincipalType type) {
TResultSet result = new TResultSet();
result.setSchema(new TResultSetMetadata());
result.getSchema().addToColumns(new TColumn("scope", Type.STRING.toThrift()));
@@ -331,14 +450,14 @@ public class AuthorizationPolicy implements PrivilegeCache {
result.getSchema().addToColumns(new TColumn("create_time", Type.STRING.toThrift()));
result.setRows(Lists.<TResultRow>newArrayList());
- Role role = getRole(roleName);
- if (role == null) return result;
- for (RolePrivilege p: role.getPrivileges()) {
+ Principal principal = getPrincipal(principalName, type);
+ if (principal == null) return result;
+ for (PrincipalPrivilege p: principal.getPrivileges()) {
TPrivilege privilege = p.toThrift();
if (filter != null) {
// Check if the privileges are targeting the same object.
filter.setPrivilege_level(privilege.getPrivilege_level());
- String privName = RolePrivilege.buildRolePrivilegeName(filter);
+ String privName = PrincipalPrivilege.buildPrivilegeName(filter);
if (!privName.equalsIgnoreCase(privilege.getPrivilege_name())) continue;
}
TResultRowBuilder rowBuilder = new TResultRowBuilder();
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/Catalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Catalog.java b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
index 7403bc2..a04d6d7 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Catalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Catalog.java
@@ -496,23 +496,28 @@ public abstract class Catalog {
result.setCache_pool(pool.toThrift());
break;
}
- case ROLE:
- Role role = authPolicy_.getRole(objectDesc.getRole().getRole_name());
- if (role == null) {
- throw new CatalogException("Role not found: " +
- objectDesc.getRole().getRole_name());
+ case PRINCIPAL:
+ Principal principal = authPolicy_.getPrincipal(
+ objectDesc.getPrincipal().getPrincipal_name(),
+ objectDesc.getPrincipal().getPrincipal_type());
+ if (principal == null) {
+ throw new CatalogException("Principal not found: " +
+ objectDesc.getPrincipal().getPrincipal_name());
}
- result.setType(role.getCatalogObjectType());
- result.setCatalog_version(role.getCatalogVersion());
- result.setRole(role.toThrift());
+ result.setType(principal.getCatalogObjectType());
+ result.setCatalog_version(principal.getCatalogVersion());
+ result.setPrincipal(principal.toThrift());
break;
case PRIVILEGE:
- Role tmpRole = authPolicy_.getRole(objectDesc.getPrivilege().getRole_id());
- if (tmpRole == null) {
- throw new CatalogException("No role associated with ID: " +
- objectDesc.getPrivilege().getRole_id());
+ Principal tmpPrincipal = authPolicy_.getPrincipal(
+ objectDesc.getPrincipal().getPrincipal_id(),
+ objectDesc.getPrincipal().getPrincipal_type());
+ if (tmpPrincipal == null) {
+ throw new CatalogException(String.format("No %s associated with ID: %d",
+ Principal.toString(objectDesc.getPrincipal().getPrincipal_type())
+ .toLowerCase(), objectDesc.getPrivilege().getPrincipal_id()));
}
- for (RolePrivilege p: tmpRole.getPrivileges()) {
+ for (PrincipalPrivilege p: tmpPrincipal.getPrivileges()) {
if (p.getName().equalsIgnoreCase(
objectDesc.getPrivilege().getPrivilege_name())) {
result.setType(p.getCatalogObjectType());
@@ -521,9 +526,9 @@ public abstract class Catalog {
return result;
}
}
- throw new CatalogException(String.format("Role '%s' does not contain " +
- "privilege: '%s'", tmpRole.getName(),
- objectDesc.getPrivilege().getPrivilege_name()));
+ throw new CatalogException(String.format("%s '%s' does not contain " +
+ "privilege: '%s'", Principal.toString(tmpPrincipal.getPrincipalType()),
+ tmpPrincipal.getName(), objectDesc.getPrivilege().getPrivilege_name()));
default: throw new IllegalStateException(
"Unexpected TCatalogObject type: " + objectDesc.getType());
}
@@ -550,12 +555,13 @@ public abstract class Catalog {
case FUNCTION:
return "FUNCTION:" + catalogObject.getFn().getName() + "(" +
catalogObject.getFn().getSignature() + ")";
- case ROLE:
- return "ROLE:" + catalogObject.getRole().getRole_name().toLowerCase();
+ case PRINCIPAL:
+ return "PRINCIPAL:" + catalogObject.getPrincipal().getPrincipal_name()
+ .toLowerCase();
case PRIVILEGE:
return "PRIVILEGE:" +
catalogObject.getPrivilege().getPrivilege_name().toLowerCase() + "." +
- Integer.toString(catalogObject.getPrivilege().getRole_id());
+ Integer.toString(catalogObject.getPrivilege().getPrincipal_id());
case HDFS_CACHE_POOL:
return "HDFS_CACHE_POOL:" +
catalogObject.getCache_pool().getPool_name().toLowerCase();
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
index a3f5a1e..252bfe1 100644
--- a/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/CatalogServiceCatalog.java
@@ -31,7 +31,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.ReentrantReadWriteLock;
-import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.CachePoolEntry;
@@ -41,8 +40,6 @@ import org.apache.hadoop.hive.metastore.api.UnknownDBException;
import org.apache.impala.authorization.SentryConfig;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.common.FileSystemUtil;
-import org.apache.impala.common.ImpalaException;
-import org.apache.impala.common.JniUtil;
import org.apache.impala.common.Pair;
import org.apache.impala.common.Reference;
import org.apache.impala.service.BackendConfig;
@@ -51,9 +48,9 @@ import org.apache.impala.thrift.TCatalog;
import org.apache.impala.thrift.TCatalogObject;
import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TCatalogUpdateResult;
-import org.apache.impala.thrift.TFunction;
import org.apache.impala.thrift.TGetCatalogUsageResponse;
import org.apache.impala.thrift.TPartitionKeyValue;
+import org.apache.impala.thrift.TPrincipalType;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TTable;
import org.apache.impala.thrift.TTableName;
@@ -66,7 +63,6 @@ import org.apache.log4j.Logger;
import org.apache.thrift.TException;
import org.apache.thrift.TSerializer;
import org.apache.thrift.protocol.TBinaryProtocol;
-import org.apache.thrift.protocol.TCompactProtocol;
import com.codahale.metrics.Timer;
import com.google.common.base.Preconditions;
@@ -444,7 +440,10 @@ public class CatalogServiceCatalog extends Catalog {
addHdfsCachePoolToCatalogDelta(cachePool, ctx);
}
for (Role role: getAllRoles()) {
- addRoleToCatalogDelta(role, ctx);
+ addPrincipalToCatalogDelta(role, ctx);
+ }
+ for (User user: getAllUsers()) {
+ addPrincipalToCatalogDelta(user, ctx);
}
// Identify the catalog objects that were removed from the catalog for which their
// versions are in range ('ctx.fromVersion', 'ctx.toVersion']. We need to make sure
@@ -539,6 +538,18 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
+ * Get a snapshot view of all the users in the catalog.
+ */
+ private List<User> getAllUsers() {
+ versionLock_.readLock().lock();
+ try {
+ return ImmutableList.copyOf(authPolicy_.getAllUsers());
+ } finally {
+ versionLock_.readLock().unlock();
+ }
+ }
+
+ /**
* Adds a database in the topic update if its version is in the range
* ('ctx.fromVersion', 'ctx.toVersion']. It iterates through all the tables and
* functions of this database to determine if they can be included in the topic update.
@@ -702,42 +713,42 @@ public class CatalogServiceCatalog extends Catalog {
/**
- * Adds a role to the topic update if its version is in the range
+ * Adds a principal to the topic update if its version is in the range
* ('ctx.fromVersion', 'ctx.toVersion']. It iterates through all the privileges of
- * this role to determine if they can be inserted in the topic update.
+ * this principal to determine if they can be inserted in the topic update.
*/
- private void addRoleToCatalogDelta(Role role, GetCatalogDeltaContext ctx)
- throws TException {
- long roleVersion = role.getCatalogVersion();
- if (roleVersion > ctx.fromVersion && roleVersion <= ctx.toVersion) {
- TCatalogObject thriftRole =
- new TCatalogObject(TCatalogObjectType.ROLE, roleVersion);
- thriftRole.setRole(role.toThrift());
- ctx.addCatalogObject(thriftRole, false);
+ private void addPrincipalToCatalogDelta(Principal principal, GetCatalogDeltaContext ctx)
+ throws TException {
+ long principalVersion = principal.getCatalogVersion();
+ if (principalVersion > ctx.fromVersion && principalVersion <= ctx.toVersion) {
+ TCatalogObject thriftPrincipal =
+ new TCatalogObject(TCatalogObjectType.PRINCIPAL, principalVersion);
+ thriftPrincipal.setPrincipal(principal.toThrift());
+ ctx.addCatalogObject(thriftPrincipal, false);
}
- for (RolePrivilege p: getAllPrivileges(role)) {
- addRolePrivilegeToCatalogDelta(p, ctx);
+ for (PrincipalPrivilege p: getAllPrivileges(principal)) {
+ addPrincipalPrivilegeToCatalogDelta(p, ctx);
}
}
/**
- * Get a snapshot view of all the privileges in a role.
+ * Get a snapshot view of all the privileges in a principal.
*/
- private List<RolePrivilege> getAllPrivileges(Role role) {
- Preconditions.checkNotNull(role);
+ private List<PrincipalPrivilege> getAllPrivileges(Principal principal) {
+ Preconditions.checkNotNull(principal);
versionLock_.readLock().lock();
try {
- return ImmutableList.copyOf(role.getPrivileges());
+ return ImmutableList.copyOf(principal.getPrivileges());
} finally {
versionLock_.readLock().unlock();
}
}
/**
- * Adds a role privilege to the topic update if its version is in the range
+ * Adds a principal privilege to the topic update if its version is in the range
* ('ctx.fromVersion', 'ctx.toVersion'].
*/
- private void addRolePrivilegeToCatalogDelta(RolePrivilege priv,
+ private void addPrincipalPrivilegeToCatalogDelta(PrincipalPrivilege priv,
GetCatalogDeltaContext ctx) throws TException {
long privVersion = priv.getCatalogVersion();
if (privVersion <= ctx.fromVersion || privVersion > ctx.toVersion) return;
@@ -1465,12 +1476,30 @@ public class CatalogServiceCatalog extends Catalog {
* If a role with the same name already exists it will be overwritten.
*/
public Role addRole(String roleName, Set<String> grantGroups) {
+ Principal role = addPrincipal(roleName, grantGroups, TPrincipalType.ROLE);
+ Preconditions.checkState(role instanceof Role);
+ return (Role) role;
+ }
+
+ /**
+ * Adds a new user with the given name to the AuthorizationPolicy.
+ * If a user with the same name already exists it will be overwritten.
+ */
+ public User addUser(String userName) {
+ Principal user = addPrincipal(userName, Sets.<String>newHashSet(),
+ TPrincipalType.USER);
+ Preconditions.checkState(user instanceof User);
+ return (User) user;
+ }
+
+ private Principal addPrincipal(String principalName, Set<String> grantGroups,
+ TPrincipalType type) {
versionLock_.writeLock().lock();
try {
- Role role = new Role(roleName, grantGroups);
- role.setCatalogVersion(incrementAndGetCatalogVersion());
- authPolicy_.addRole(role);
- return role;
+ Principal principal = Principal.newInstance(principalName, type, grantGroups);
+ principal.setCatalogVersion(incrementAndGetCatalogVersion());
+ authPolicy_.addPrincipal(principal);
+ return principal;
} finally {
versionLock_.writeLock().unlock();
}
@@ -1482,17 +1511,36 @@ public class CatalogServiceCatalog extends Catalog {
* exists.
*/
public Role removeRole(String roleName) {
+ Principal role = removePrincipal(roleName, TPrincipalType.ROLE);
+ if (role == null) return null;
+ Preconditions.checkState(role instanceof Role);
+ return (Role) role;
+ }
+
+ /**
+ * Removes the user with the given name from the AuthorizationPolicy. Returns the
+ * removed user with an incremented catalog version, or null if no user with this name
+ * exists.
+ */
+ public User removeUser(String userName) {
+ Principal user = removePrincipal(userName, TPrincipalType.USER);
+ if (user == null) return null;
+ Preconditions.checkState(user instanceof User);
+ return (User) user;
+ }
+
+ private Principal removePrincipal(String principalName, TPrincipalType type) {
versionLock_.writeLock().lock();
try {
- Role role = authPolicy_.removeRole(roleName);
- if (role == null) return null;
- for (RolePrivilege priv: role.getPrivileges()) {
+ Principal principal = authPolicy_.removePrincipal(principalName, type);
+ if (principal == null) return null;
+ for (PrincipalPrivilege priv: principal.getPrivileges()) {
priv.setCatalogVersion(incrementAndGetCatalogVersion());
deleteLog_.addRemovedObject(priv.toTCatalogObject());
}
- role.setCatalogVersion(incrementAndGetCatalogVersion());
- deleteLog_.addRemovedObject(role.toTCatalogObject());
- return role;
+ principal.setCatalogVersion(incrementAndGetCatalogVersion());
+ deleteLog_.addRemovedObject(principal.toTCatalogObject());
+ return principal;
} finally {
versionLock_.writeLock().unlock();
}
@@ -1506,7 +1554,7 @@ public class CatalogServiceCatalog extends Catalog {
throws CatalogException {
versionLock_.writeLock().lock();
try {
- Role role = authPolicy_.addGrantGroup(roleName, groupName);
+ Role role = authPolicy_.addRoleGrantGroup(roleName, groupName);
Preconditions.checkNotNull(role);
role.setCatalogVersion(incrementAndGetCatalogVersion());
return role;
@@ -1523,7 +1571,7 @@ public class CatalogServiceCatalog extends Catalog {
throws CatalogException {
versionLock_.writeLock().lock();
try {
- Role role = authPolicy_.removeGrantGroup(roleName, groupName);
+ Role role = authPolicy_.removeRoleGrantGroup(roleName, groupName);
Preconditions.checkNotNull(role);
role.setCatalogVersion(incrementAndGetCatalogVersion());
return role;
@@ -1533,17 +1581,35 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
- * Adds a privilege to the given role name. Returns the new RolePrivilege and
+ * Adds a privilege to the given role name. Returns the new PrincipalPrivilege and
* increments the catalog version. If the parent role does not exist a CatalogException
* is thrown.
*/
- public RolePrivilege addRolePrivilege(String roleName, TPrivilege thriftPriv)
+ public PrincipalPrivilege addRolePrivilege(String roleName, TPrivilege thriftPriv)
throws CatalogException {
+ return addPrincipalPrivilege(roleName, thriftPriv, TPrincipalType.ROLE);
+ }
+
+ /**
+ * Adds a privilege to the given user name. Returns the new PrincipalPrivilege and
+ * increments the catalog version. If the user does not exist a CatalogException is
+ * thrown.
+ */
+ public PrincipalPrivilege addUserPrivilege(String userName, TPrivilege thriftPriv)
+ throws CatalogException {
+ return addPrincipalPrivilege(userName, thriftPriv, TPrincipalType.USER);
+ }
+
+ private PrincipalPrivilege addPrincipalPrivilege(String principalName,
+ TPrivilege thriftPriv, TPrincipalType type) throws CatalogException {
versionLock_.writeLock().lock();
try {
- Role role = authPolicy_.getRole(roleName);
- if (role == null) throw new CatalogException("Role does not exist: " + roleName);
- RolePrivilege priv = RolePrivilege.fromThrift(thriftPriv);
+ Principal principal = authPolicy_.getPrincipal(principalName, type);
+ if (principal == null) {
+ throw new CatalogException(String.format("%s does not exist: %s",
+ Principal.toString(type), principalName));
+ }
+ PrincipalPrivilege priv = PrincipalPrivilege.fromThrift(thriftPriv);
priv.setCatalogVersion(incrementAndGetCatalogVersion());
authPolicy_.addPrivilege(priv);
return priv;
@@ -1553,39 +1619,51 @@ public class CatalogServiceCatalog extends Catalog {
}
/**
- * Removes a RolePrivilege from the given role name. Returns the removed
- * RolePrivilege with an incremented catalog version or null if no matching privilege
- * was found. Throws a CatalogException if no role exists with this name.
+ * Removes a PrincipalPrivilege from the given role name. Returns the removed
+ * PrincipalPrivilege with an incremented catalog version or null if no matching
+ * privilege was found. Throws a CatalogException if no role exists with this name.
*/
- public RolePrivilege removeRolePrivilege(String roleName, TPrivilege thriftPriv)
+ public PrincipalPrivilege removeRolePrivilege(String roleName, TPrivilege thriftPriv)
throws CatalogException {
+ return removePrincipalPrivilege(roleName, thriftPriv, TPrincipalType.ROLE);
+ }
+
+ private PrincipalPrivilege removePrincipalPrivilege(String principalName,
+ TPrivilege privilege, TPrincipalType type) throws CatalogException {
versionLock_.writeLock().lock();
try {
- Role role = authPolicy_.getRole(roleName);
- if (role == null) throw new CatalogException("Role does not exist: " + roleName);
- RolePrivilege rolePrivilege =
- role.removePrivilege(thriftPriv.getPrivilege_name());
- if (rolePrivilege == null) return null;
- rolePrivilege.setCatalogVersion(incrementAndGetCatalogVersion());
- deleteLog_.addRemovedObject(rolePrivilege.toTCatalogObject());
- return rolePrivilege;
+ Principal principal = authPolicy_.getPrincipal(principalName, type);
+ if (principal == null) {
+ throw new CatalogException(String.format("%s does not exist: %s",
+ Principal.toString(type), principalName));
+ }
+ PrincipalPrivilege principalPrivilege =
+ principal.removePrivilege(privilege.getPrivilege_name());
+ if (principalPrivilege == null) return null;
+ principalPrivilege.setCatalogVersion(incrementAndGetCatalogVersion());
+ deleteLog_.addRemovedObject(principalPrivilege.toTCatalogObject());
+ return principalPrivilege;
} finally {
versionLock_.writeLock().unlock();
}
}
/**
- * Gets a RolePrivilege from the given role name. Returns the privilege if it exists,
- * or null if no privilege matching the privilege spec exist.
- * Throws a CatalogException if the role does not exist.
+ * Gets a PrincipalPrivilege from the given principal name. Returns the privilege
+ * if it exists, or null if no privilege matching the privilege spec exist.
+ * Throws a CatalogException if the principal does not exist.
*/
- public RolePrivilege getRolePrivilege(String roleName, TPrivilege privSpec)
- throws CatalogException {
+ public PrincipalPrivilege getPrincipalPrivilege(String principalName,
+ TPrivilege privSpec) throws CatalogException {
versionLock_.readLock().lock();
try {
- Role role = authPolicy_.getRole(roleName);
- if (role == null) throw new CatalogException("Role does not exist: " + roleName);
- return role.getPrivilege(privSpec.getPrivilege_name());
+ Principal principal = authPolicy_.getPrincipal(principalName,
+ privSpec.getPrincipal_type());
+ if (principal == null) {
+ throw new CatalogException(Principal.toString(privSpec.getPrincipal_type()) +
+ " does not exist: " + principalName);
+ }
+ return principal.getPrivilege(privSpec.getPrivilege_name());
} finally {
versionLock_.readLock().unlock();
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
index 30c34ad..a259bf0 100644
--- a/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
+++ b/fe/src/main/java/org/apache/impala/catalog/ImpaladCatalog.java
@@ -33,8 +33,8 @@ import org.apache.impala.thrift.TCatalogObjectType;
import org.apache.impala.thrift.TDataSource;
import org.apache.impala.thrift.TDatabase;
import org.apache.impala.thrift.TFunction;
+import org.apache.impala.thrift.TPrincipal;
import org.apache.impala.thrift.TPrivilege;
-import org.apache.impala.thrift.TRole;
import org.apache.impala.thrift.TTable;
import org.apache.impala.thrift.TUniqueId;
import org.apache.impala.thrift.TUpdateCatalogCacheRequest;
@@ -113,7 +113,7 @@ public class ImpaladCatalog extends Catalog implements FeCatalog {
return catalogObject.getType() == TCatalogObjectType.DATABASE ||
catalogObject.getType() == TCatalogObjectType.DATA_SOURCE ||
catalogObject.getType() == TCatalogObjectType.HDFS_CACHE_POOL ||
- catalogObject.getType() == TCatalogObjectType.ROLE;
+ catalogObject.getType() == TCatalogObjectType.PRINCIPAL;
}
/**
@@ -284,14 +284,14 @@ public class ImpaladCatalog extends Catalog implements FeCatalog {
case DATA_SOURCE:
addDataSource(catalogObject.getData_source(), catalogObject.getCatalog_version());
break;
- case ROLE:
- Role role = Role.fromThrift(catalogObject.getRole());
- role.setCatalogVersion(catalogObject.getCatalog_version());
- authPolicy_.addRole(role);
+ case PRINCIPAL:
+ Principal principal = Principal.fromThrift(catalogObject.getPrincipal());
+ principal.setCatalogVersion(catalogObject.getCatalog_version());
+ authPolicy_.addPrincipal(principal);
break;
case PRIVILEGE:
- RolePrivilege privilege =
- RolePrivilege.fromThrift(catalogObject.getPrivilege());
+ PrincipalPrivilege privilege =
+ PrincipalPrivilege.fromThrift(catalogObject.getPrivilege());
privilege.setCatalogVersion(catalogObject.getCatalog_version());
try {
authPolicy_.addPrivilege(privilege);
@@ -331,8 +331,8 @@ public class ImpaladCatalog extends Catalog implements FeCatalog {
case DATA_SOURCE:
removeDataSource(catalogObject.getData_source(), dropCatalogVersion);
break;
- case ROLE:
- removeRole(catalogObject.getRole(), dropCatalogVersion);
+ case PRINCIPAL:
+ removePrincipal(catalogObject.getPrincipal(), dropCatalogVersion);
break;
case PRIVILEGE:
removePrivilege(catalogObject.getPrivilege(), dropCatalogVersion);
@@ -467,24 +467,28 @@ public class ImpaladCatalog extends Catalog implements FeCatalog {
}
}
- private void removeRole(TRole thriftRole, long dropCatalogVersion) {
- Role existingRole = authPolicy_.getRole(thriftRole.getRole_name());
+ private void removePrincipal(TPrincipal thriftPrincipal, long dropCatalogVersion) {
+ Principal existingPrincipal = authPolicy_.getPrincipal(
+ thriftPrincipal.getPrincipal_name(), thriftPrincipal.getPrincipal_type());
// version of the drop, remove the function.
- if (existingRole != null && existingRole.getCatalogVersion() < dropCatalogVersion) {
- authPolicy_.removeRole(thriftRole.getRole_name());
- CatalogObjectVersionSet.INSTANCE.removeAll(existingRole.getPrivileges());
+ if (existingPrincipal != null &&
+ existingPrincipal.getCatalogVersion() < dropCatalogVersion) {
+ authPolicy_.removePrincipal(thriftPrincipal.getPrincipal_name(),
+ thriftPrincipal.getPrincipal_type());
+ CatalogObjectVersionSet.INSTANCE.removeAll(existingPrincipal.getPrivileges());
}
}
private void removePrivilege(TPrivilege thriftPrivilege, long dropCatalogVersion) {
- Role role = authPolicy_.getRole(thriftPrivilege.getRole_id());
- if (role == null) return;
- RolePrivilege existingPrivilege =
- role.getPrivilege(thriftPrivilege.getPrivilege_name());
+ Principal principal = authPolicy_.getPrincipal(thriftPrivilege.getPrincipal_id(),
+ thriftPrivilege.getPrincipal_type());
+ if (principal == null) return;
+ PrincipalPrivilege existingPrivilege =
+ principal.getPrivilege(thriftPrivilege.getPrivilege_name());
// version of the drop, remove the function.
if (existingPrivilege != null &&
existingPrivilege.getCatalogVersion() < dropCatalogVersion) {
- role.removePrivilege(thriftPrivilege.getPrivilege_name());
+ principal.removePrivilege(thriftPrivilege.getPrivilege_name());
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/Principal.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Principal.java b/fe/src/main/java/org/apache/impala/catalog/Principal.java
new file mode 100644
index 0000000..d048d10
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/Principal.java
@@ -0,0 +1,181 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.impala.thrift.TCatalogObject;
+import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.thrift.TPrincipal;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
+import org.apache.impala.thrift.TPrincipalType;
+
+/**
+ * A base class that represents a principal in an authorization policy.
+ * This class is thread safe.
+ */
+public abstract class Principal extends CatalogObjectImpl {
+ private final TPrincipal principal_;
+ // The last principal ID assigned, starts at 0.
+ private static AtomicInteger principalId_ = new AtomicInteger(0);
+
+ private final CatalogObjectCache<PrincipalPrivilege> principalPrivileges_ =
+ new CatalogObjectCache<>();
+
+ protected Principal(String principalName, TPrincipalType type,
+ Set<String> grantGroups) {
+ principal_ = new TPrincipal();
+ principal_.setPrincipal_name(principalName);
+ principal_.setPrincipal_type(type);
+ principal_.setPrincipal_id(principalId_.incrementAndGet());
+ principal_.setGrant_groups(Lists.newArrayList(grantGroups));
+ }
+
+ protected Principal(TPrincipal principal) {
+ principal_ = principal;
+ }
+
+ /**
+ * Adds a privilege to the principal. Returns true if the privilege was added
+ * successfully or false if there was a newer version of the privilege already added
+ * to the principal.
+ */
+ public boolean addPrivilege(PrincipalPrivilege privilege) {
+ return principalPrivileges_.add(privilege);
+ }
+
+ /**
+ * Returns all privileges for this principal. If no privileges have been added to the
+ * principal, an empty list is returned.
+ */
+ public List<PrincipalPrivilege> getPrivileges() {
+ return Lists.newArrayList(principalPrivileges_.getValues());
+ }
+
+ /**
+ * Returns all privilege names for this principal, or an empty set of no privileges are
+ * granted to the principal.
+ */
+ public Set<String> getPrivilegeNames() {
+ return Sets.newHashSet(principalPrivileges_.keySet());
+ }
+
+ /**
+ * Gets a privilege with the given name from this principal. If no privilege exists
+ * with this name null is returned.
+ */
+ public PrincipalPrivilege getPrivilege(String privilegeName) {
+ return principalPrivileges_.get(privilegeName);
+ }
+
+ /**
+ * Removes a privilege with the given name from the principal. Returns the removed
+ * privilege or null if no privilege exists with this name.
+ */
+ public PrincipalPrivilege removePrivilege(String privilegeName) {
+ return principalPrivileges_.remove(privilegeName);
+ }
+
+ /**
+ * Adds a new grant group to this principal.
+ */
+ public synchronized void addGrantGroup(String groupName) {
+ if (principal_.getGrant_groups().contains(groupName)) return;
+ principal_.addToGrant_groups(groupName);
+ }
+
+ /**
+ * Removes a grant group from this principal.
+ */
+ public synchronized void removeGrantGroup(String groupName) {
+ principal_.getGrant_groups().remove(groupName);
+ // Should never have duplicates in the list of groups.
+ Preconditions.checkState(!principal_.getGrant_groups().contains(groupName));
+ }
+
+ /**
+ * Returns the Thrift representation of the principal.
+ */
+ public TPrincipal toThrift() {
+ return principal_;
+ }
+
+ /**
+ * Creates a Principal from a TPrincipal thrift struct.
+ */
+ public static Principal fromThrift(TPrincipal thriftPrincipal) {
+ return thriftPrincipal.getPrincipal_type() == TPrincipalType.ROLE ?
+ new Role(thriftPrincipal) : new User(thriftPrincipal);
+ }
+
+ /**
+ * Creates a new instance of Principal.
+ */
+ public static Principal newInstance(String principalName, TPrincipalType type,
+ Set<String> grantGroups) {
+ return type == TPrincipalType.ROLE ?
+ new Role(principalName, grantGroups) : new User(principalName, grantGroups);
+ }
+
+ /**
+ * Gets the set of group names that have been granted to this principal or an empty
+ * set if no groups have been granted.
+ */
+ public Set<String> getGrantGroups() {
+ return Sets.newHashSet(principal_.getGrant_groups());
+ }
+
+ @Override
+ public TCatalogObjectType getCatalogObjectType() {
+ return TCatalogObjectType.PRINCIPAL;
+ }
+
+ @Override
+ public String getName() { return principal_.getPrincipal_name(); }
+
+ /**
+ * Returns the principal ID.
+ */
+ public int getId() { return principal_.getPrincipal_id(); }
+
+ @Override
+ public String getUniqueName() {
+ return this.getPrincipalType() == TPrincipalType.ROLE ? "ROLE:" : "USER:"
+ + getName().toLowerCase();
+ }
+
+ public TCatalogObject toTCatalogObject() {
+ TCatalogObject catalogObject =
+ new TCatalogObject(getCatalogObjectType(), getCatalogVersion());
+ catalogObject.setPrincipal(toThrift());
+ return catalogObject;
+ }
+
+ /**
+ * Returns the principal type.
+ */
+ public TPrincipalType getPrincipalType() { return principal_.getPrincipal_type(); }
+
+ public static String toString(TPrincipalType type) {
+ return type == TPrincipalType.ROLE ? "Role" : "User";
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java b/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
new file mode 100644
index 0000000..033b9c9
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/PrincipalPrivilege.java
@@ -0,0 +1,154 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+import java.util.List;
+
+import org.apache.impala.thrift.TCatalogObject;
+import org.apache.impala.thrift.TCatalogObjectType;
+import org.apache.impala.thrift.TPrincipalType;
+import org.apache.impala.thrift.TPrivilege;
+import org.apache.impala.thrift.TPrivilegeLevel;
+import org.apache.impala.thrift.TPrivilegeScope;
+import org.apache.log4j.Logger;
+
+import com.google.common.base.Joiner;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+
+/**
+ * Represents a privilege that has been granted to a principal in an authorization policy.
+ * This class is thread safe.
+ */
+public class PrincipalPrivilege extends CatalogObjectImpl {
+ private static final Logger LOG = Logger.getLogger(AuthorizationPolicy.class);
+ // These Joiners are used to build principal names. For simplicity, the principal name
+ // we use can also be sent to the Sentry library to perform authorization checks
+ // so we build them in the same format.
+ private static final Joiner AUTHORIZABLE_JOINER = Joiner.on("->");
+ private static final Joiner KV_JOINER = Joiner.on("=");
+ private final TPrivilege privilege_;
+
+ private PrincipalPrivilege(TPrivilege privilege) {
+ privilege_ = privilege;
+ }
+
+ public TPrivilege toThrift() { return privilege_; }
+ public static PrincipalPrivilege fromThrift(TPrivilege privilege) {
+ return new PrincipalPrivilege(privilege);
+ }
+
+ /**
+ * Builds a privilege name for the given TPrivilege object. For simplicity, this name is
+ * generated in a format that can be sent to the Sentry client to perform authorization
+ * checks.
+ */
+ public static String buildPrivilegeName(TPrivilege privilege) {
+ List<String> authorizable = Lists.newArrayListWithExpectedSize(4);
+ try {
+ Preconditions.checkNotNull(privilege);
+ TPrivilegeScope scope = privilege.getScope();
+ Preconditions.checkNotNull(scope);
+ switch (scope) {
+ case SERVER: {
+ authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
+ toLowerCase()));
+ break;
+ }
+ case URI: {
+ authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
+ toLowerCase()));
+ // (IMPALA-2695) URIs are case sensitive
+ authorizable.add(KV_JOINER.join("uri", privilege.getUri()));
+ break;
+ }
+ case DATABASE: {
+ authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
+ toLowerCase()));
+ authorizable.add(KV_JOINER.join("db", privilege.getDb_name().
+ toLowerCase()));
+ break;
+ }
+ case TABLE: {
+ authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
+ toLowerCase()));
+ authorizable.add(KV_JOINER.join("db", privilege.getDb_name().
+ toLowerCase()));
+ authorizable.add(KV_JOINER.join("table", privilege.getTable_name().
+ toLowerCase()));
+ break;
+ }
+ case COLUMN: {
+ authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
+ toLowerCase()));
+ authorizable.add(KV_JOINER.join("db", privilege.getDb_name().
+ toLowerCase()));
+ authorizable.add(KV_JOINER.join("table", privilege.getTable_name().
+ toLowerCase()));
+ authorizable.add(KV_JOINER.join("column", privilege.getColumn_name().
+ toLowerCase()));
+ break;
+ }
+ default: {
+ throw new UnsupportedOperationException(
+ "Unknown privilege scope: " + scope.toString());
+ }
+ }
+
+ // The ALL privilege is always implied and does not need to be included as part
+ // of the name.
+ if (privilege.getPrivilege_level() != TPrivilegeLevel.ALL) {
+ authorizable.add(KV_JOINER.join("action",
+ privilege.getPrivilege_level().toString()));
+ }
+ return AUTHORIZABLE_JOINER.join(authorizable);
+ } catch (Exception e) {
+ // Should never make it here unless the privilege is malformed.
+ LOG.error("ERROR: ", e);
+ return null;
+ }
+ }
+
+ @Override
+ public TCatalogObjectType getCatalogObjectType() {
+ return TCatalogObjectType.PRIVILEGE;
+ }
+ @Override
+ public String getName() { return privilege_.getPrivilege_name(); }
+ public int getPrincipalId() { return privilege_.getPrincipal_id(); }
+ public TPrincipalType getPrincipalType() { return privilege_.getPrincipal_type(); }
+ @Override
+ public String getUniqueName() {
+ return "PRIVILEGE:" + getName().toLowerCase() + "." + Integer.toString(
+ getPrincipalId());
+ }
+
+ public TCatalogObject toTCatalogObject() {
+ TCatalogObject catalogObject =
+ new TCatalogObject(getCatalogObjectType(), getCatalogVersion());
+ catalogObject.setPrivilege(toThrift());
+ return catalogObject;
+ }
+
+ // The time this principal was created. Used to quickly check if the same privilege
+ // was dropped and re-created. Assumes a principal will not be created + dropped +
+ // created in less than 1ms. Returns -1 if create_time_ms was not set for the privilege.
+ public long getCreateTimeMs() {
+ return privilege_.isSetCreate_time_ms() ? privilege_.getCreate_time_ms() : -1L;
+ }
+}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/Role.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/Role.java b/fe/src/main/java/org/apache/impala/catalog/Role.java
index b45ff22..9cee7d2 100644
--- a/fe/src/main/java/org/apache/impala/catalog/Role.java
+++ b/fe/src/main/java/org/apache/impala/catalog/Role.java
@@ -17,129 +17,23 @@
package org.apache.impala.catalog;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import org.apache.impala.thrift.TCatalogObject;
-import org.apache.impala.thrift.TCatalogObjectType;
-import org.apache.impala.thrift.TRole;
import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
+import org.apache.impala.thrift.TPrincipal;
+import org.apache.impala.thrift.TPrincipalType;
+
+import java.util.Set;
/**
- * Represents a role in an authorization policy. This class is thread safe.
+ * Represents a role in an authorization policy.
*/
-public class Role extends CatalogObjectImpl {
- private final TRole role_;
- // The last role ID assigned, starts at 0.
- private static AtomicInteger roleId_ = new AtomicInteger(0);
-
- private final CatalogObjectCache<RolePrivilege> rolePrivileges_ =
- new CatalogObjectCache<RolePrivilege>();
-
+public class Role extends Principal {
public Role(String roleName, Set<String> grantGroups) {
- role_ = new TRole();
- role_.setRole_name(roleName);
- role_.setRole_id(roleId_.incrementAndGet());
- role_.setGrant_groups(Lists.newArrayList(grantGroups));
- }
-
- private Role(TRole role) {
- role_ = role;
- }
-
- /**
- * Adds a privilege to the role. Returns true if the privilege was added successfully
- * or false if there was a newer version of the privilege already added to the role.
- */
- public boolean addPrivilege(RolePrivilege privilege) {
- return rolePrivileges_.add(privilege);
- }
-
- /**
- * Returns all privileges for this role. If no privileges have been added to the role
- * an empty list will be returned.
- */
- public List<RolePrivilege> getPrivileges() {
- return Lists.newArrayList(rolePrivileges_.getValues());
- }
-
- /**
- * Returns all privilege names for this role, or an empty set of no privileges are
- * granted to the role.
- */
- public Set<String> getPrivilegeNames() {
- return Sets.newHashSet(rolePrivileges_.keySet());
- }
-
- /**
- * Gets a privilege with the given name from this role. If no privilege exists
- * with this name null is returned.
- */
- public RolePrivilege getPrivilege(String privilegeName) {
- return rolePrivileges_.get(privilegeName);
- }
-
- /**
- * Removes a privilege with the given name from the role. Returns the removed
- * privilege or null if no privilege exists with this name.
- */
- public RolePrivilege removePrivilege(String privilegeName) {
- return rolePrivileges_.remove(privilegeName);
- }
-
- /**
- * Adds a new grant group to this role.
- */
- public synchronized void addGrantGroup(String groupName) {
- if (role_.getGrant_groups().contains(groupName)) return;
- role_.addToGrant_groups(groupName);
- }
-
- /**
- * Removes a grant group from this role.
- */
- public synchronized void removeGrantGroup(String groupName) {
- role_.getGrant_groups().remove(groupName);
- // Should never have duplicates in the list of groups.
- Preconditions.checkState(!role_.getGrant_groups().contains(groupName));
- }
-
- /**
- * Returns the Thrift representation of the role.
- */
- public TRole toThrift() {
- return role_;
- }
-
- /**
- * Creates a Role from a TRole thrift struct.
- */
- public static Role fromThrift(TRole thriftRole) {
- return new Role(thriftRole);
- }
-
- /**
- * Gets the set of group names that have been granted this role or an empty
- * Set if no groups have been granted the role.
- */
- public Set<String> getGrantGroups() {
- return Sets.newHashSet(role_.getGrant_groups());
+ super(roleName, TPrincipalType.ROLE, grantGroups);
}
- @Override
- public TCatalogObjectType getCatalogObjectType() { return TCatalogObjectType.ROLE; }
- @Override
- public String getName() { return role_.getRole_name(); }
- public int getId() { return role_.getRole_id(); }
- @Override
- public String getUniqueName() { return "ROLE:" + getName().toLowerCase(); }
- public TCatalogObject toTCatalogObject() {
- TCatalogObject catalogObject =
- new TCatalogObject(getCatalogObjectType(), getCatalogVersion());
- catalogObject.setRole(toThrift());
- return catalogObject;
+ public Role(TPrincipal thriftPrincipal) {
+ super(thriftPrincipal);
+ Preconditions.checkArgument(
+ thriftPrincipal.getPrincipal_type() == TPrincipalType.ROLE);
}
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java b/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
deleted file mode 100644
index ef3717c..0000000
--- a/fe/src/main/java/org/apache/impala/catalog/RolePrivilege.java
+++ /dev/null
@@ -1,151 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-package org.apache.impala.catalog;
-
-import java.util.List;
-
-import org.apache.log4j.Logger;
-
-import org.apache.impala.thrift.TCatalogObject;
-import org.apache.impala.thrift.TCatalogObjectType;
-import org.apache.impala.thrift.TPrivilege;
-import org.apache.impala.thrift.TPrivilegeLevel;
-import org.apache.impala.thrift.TPrivilegeScope;
-import com.google.common.base.Joiner;
-import com.google.common.base.Preconditions;
-import com.google.common.collect.Lists;
-
-/**
- * Represents a privilege that has been granted to a role in an authorization policy.
- * This class is thread safe.
- */
-public class RolePrivilege extends CatalogObjectImpl {
- private static final Logger LOG = Logger.getLogger(AuthorizationPolicy.class);
- // These Joiners are used to build role names. For simplicity, the role name we
- // use can also be sent to the Sentry library to perform authorization checks
- // so we build them in the same format.
- private static final Joiner AUTHORIZABLE_JOINER = Joiner.on("->");
- private static final Joiner KV_JOINER = Joiner.on("=");
- private final TPrivilege privilege_;
-
- private RolePrivilege(TPrivilege privilege) {
- privilege_ = privilege;
- }
-
- public TPrivilege toThrift() { return privilege_; }
- public static RolePrivilege fromThrift(TPrivilege privilege) {
- return new RolePrivilege(privilege);
- }
-
- /**
- * Builds a privilege name for the given TPrivilege object. For simplicity, this name is
- * generated in a format that can be sent to the Sentry client to perform authorization
- * checks.
- */
- public static String buildRolePrivilegeName(TPrivilege privilege) {
- List<String> authorizable = Lists.newArrayListWithExpectedSize(4);
- try {
- Preconditions.checkNotNull(privilege);
- TPrivilegeScope scope = privilege.getScope();
- Preconditions.checkNotNull(scope);
- switch (scope) {
- case SERVER: {
- authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
- toLowerCase()));
- break;
- }
- case URI: {
- authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
- toLowerCase()));
- // (IMPALA-2695) URIs are case sensitive
- authorizable.add(KV_JOINER.join("uri", privilege.getUri()));
- break;
- }
- case DATABASE: {
- authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
- toLowerCase()));
- authorizable.add(KV_JOINER.join("db", privilege.getDb_name().
- toLowerCase()));
- break;
- }
- case TABLE: {
- authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
- toLowerCase()));
- authorizable.add(KV_JOINER.join("db", privilege.getDb_name().
- toLowerCase()));
- authorizable.add(KV_JOINER.join("table", privilege.getTable_name().
- toLowerCase()));
- break;
- }
- case COLUMN: {
- authorizable.add(KV_JOINER.join("server", privilege.getServer_name().
- toLowerCase()));
- authorizable.add(KV_JOINER.join("db", privilege.getDb_name().
- toLowerCase()));
- authorizable.add(KV_JOINER.join("table", privilege.getTable_name().
- toLowerCase()));
- authorizable.add(KV_JOINER.join("column", privilege.getColumn_name().
- toLowerCase()));
- break;
- }
- default: {
- throw new UnsupportedOperationException(
- "Unknown privilege scope: " + scope.toString());
- }
- }
-
- // The ALL privilege is always implied and does not need to be included as part
- // of the name.
- if (privilege.getPrivilege_level() != TPrivilegeLevel.ALL) {
- authorizable.add(KV_JOINER.join("action",
- privilege.getPrivilege_level().toString()));
- }
- return AUTHORIZABLE_JOINER.join(authorizable);
- } catch (Exception e) {
- // Should never make it here unless the privilege is malformed.
- LOG.error("ERROR: ", e);
- return null;
- }
- }
-
- @Override
- public TCatalogObjectType getCatalogObjectType() {
- return TCatalogObjectType.PRIVILEGE;
- }
- @Override
- public String getName() { return privilege_.getPrivilege_name(); }
- public int getRoleId() { return privilege_.getRole_id(); }
- @Override
- public String getUniqueName() {
- return "PRIVILEGE:" + getName().toLowerCase() + "." + Integer.toString(getRoleId());
- }
-
- public TCatalogObject toTCatalogObject() {
- TCatalogObject catalogObject =
- new TCatalogObject(getCatalogObjectType(), getCatalogVersion());
- catalogObject.setPrivilege(toThrift());
- return catalogObject;
- }
-
- // The time this role was created. Used to quickly check if the same privilege
- // was dropped and re-created. Assumes a role will not be created + dropped + created
- // in less than 1ms. Returns -1 if create_time_ms was not set for the privilege.
- public long getCreateTimeMs() {
- return privilege_.isSetCreate_time_ms() ? privilege_.getCreate_time_ms() : -1L;
- }
-}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/catalog/User.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/catalog/User.java b/fe/src/main/java/org/apache/impala/catalog/User.java
new file mode 100644
index 0000000..2845670
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/catalog/User.java
@@ -0,0 +1,39 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.catalog;
+
+import com.google.common.base.Preconditions;
+import org.apache.impala.thrift.TPrincipal;
+import org.apache.impala.thrift.TPrincipalType;
+
+import java.util.Set;
+
+/**
+ * Represents a role in an authorization policy.
+ */
+public class User extends Principal {
+ public User(String userName, Set<String> grantGroups) {
+ super(userName, TPrincipalType.USER, grantGroups);
+ }
+
+ public User(TPrincipal thriftPrincipal) {
+ super(thriftPrincipal);
+ Preconditions.checkArgument(
+ thriftPrincipal.getPrincipal_type() == TPrincipalType.USER);
+ }
+}
[5/6] impala git commit: IMPALA-7392: [DOCS] SCAN_BYTES_LIMIT query
option documented
Posted by ta...@apache.org.
IMPALA-7392: [DOCS] SCAN_BYTES_LIMIT query option documented
Change-Id: I6430e06cabe21b8080239f3225d3bfdd5cc502cb
Reviewed-on: http://gerrit.cloudera.org:8080/11240
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/48fdd0b0
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/48fdd0b0
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/48fdd0b0
Branch: refs/heads/master
Commit: 48fdd0b0a89a2949d81b6d3486c202ceb4c5c1c9
Parents: a23e6f2
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Wed Aug 15 14:50:08 2018 -0700
Committer: Alex Rodoni <ar...@cloudera.com>
Committed: Wed Aug 15 23:04:02 2018 +0000
----------------------------------------------------------------------
docs/impala.ditamap | 1 +
docs/topics/impala_scan_bytes_limit.xml | 131 +++++++++++++++++++++++++++
2 files changed, 132 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/48fdd0b0/docs/impala.ditamap
----------------------------------------------------------------------
diff --git a/docs/impala.ditamap b/docs/impala.ditamap
index 1ea0e6d..9260a9b 100644
--- a/docs/impala.ditamap
+++ b/docs/impala.ditamap
@@ -221,6 +221,7 @@ under the License.
<topicref rev="2.5.0" href="topics/impala_runtime_filter_mode.xml"/>
<topicref rev="2.5.0" href="topics/impala_runtime_filter_wait_time_ms.xml"/>
<topicref rev="2.6.0" href="topics/impala_s3_skip_insert_staging.xml"/>
+ <topicref rev="3.1" href="topics/impala_scan_bytes_limit.xml"/>
<topicref rev="2.5.0" href="topics/impala_schedule_random_replica.xml"/>
<topicref rev="2.8.0 IMPALA-3671" href="topics/impala_scratch_limit.xml"/>
<!-- This option is for internal use only and might go away without ever being documented. -->
http://git-wip-us.apache.org/repos/asf/impala/blob/48fdd0b0/docs/topics/impala_scan_bytes_limit.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_scan_bytes_limit.xml b/docs/topics/impala_scan_bytes_limit.xml
new file mode 100644
index 0000000..5fc4a8a
--- /dev/null
+++ b/docs/topics/impala_scan_bytes_limit.xml
@@ -0,0 +1,131 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements. See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership. The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied. See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<!DOCTYPE concept PUBLIC "-//OASIS//DTD DITA Concept//EN" "concept.dtd">
+<concept id="scan_bytes_limit">
+
+ <title>SCAN_BYTES_LIMIT Query Option (<keyword keyref="impala31"/> or higher
+ only)</title>
+
+ <titlealts audience="PDF">
+
+ <navtitle>SCAN_BYTES_LIMIT</navtitle>
+
+ </titlealts>
+
+ <prolog>
+ <metadata>
+ <data name="Category" value="Impala"/>
+ <data name="Category" value="Impala Query Options"/>
+ <data name="Category" value="Scalability"/>
+ <data name="Category" value="Memory"/>
+ <data name="Category" value="Troubleshooting"/>
+ <data name="Category" value="Developers"/>
+ <data name="Category" value="Data Analysts"/>
+ </metadata>
+ </prolog>
+
+ <conbody>
+
+ <p>
+ The <codeph>SCAN_BYTES_LIMIT</codeph> query option sets a time limit on the bytes scanned
+ by HDFS and HBase SCAN operations. If a query is still executing when the query’s
+ coordinator detects that it has exceeded the limit, the query is terminated with an error.
+ The option is intended to prevent runaway queries that scan more data than is intended.
+ </p>
+
+ <p>
+ For example, an Impala administrator could set a default value of
+ <codeph>SCAN_BYTES_LIMIT=100GB</codeph> for a resource pool to automatically kill queries
+ that scan more than 100 GB of data (see
+ <xref
+ href="https://impala.apache.org/docs/build/html/topics/impala_admission.html"
+ format="html" scope="external">Impala
+ Admission Control and Query Queuing</xref> for information about default query options).
+ If a user accidentally omits a partition filter in a <codeph>WHERE</codeph> clause and
+ runs a large query that scans a lot of data, the query will be automatically terminated
+ after the time limit expires to free up resources.
+ </p>
+
+ <p>
+ You can override the default value per-query or per-session, in the same way as other
+ query options, if you do not want the default <codeph>SCAN_BYTES_LIMIT</codeph> value to
+ apply to a specific query or session.
+ <note>
+ <ul>
+ <li dir="ltr">
+ <p dir="ltr">
+ Only data actually read from the underlying storage layer is counted towards the
+ limit. E.g. Impala’s Parquet scanner employs several techniques to skip over
+ data in a file that is not relevant to a specific query, so often only a fraction
+ of the file size is counted towards <codeph>SCAN_BYTES_LIMIT</codeph>.
+ </p>
+ </li>
+
+ <li dir="ltr">
+ <p dir="ltr">
+ As of Impala 3.1, bytes scanned by Kudu tablet servers are not counted towards the
+ limit.
+ </p>
+ </li>
+ </ul>
+ </note>
+ </p>
+
+ <p>
+ <b>Syntax:</b> <codeph>SET SCAN_BYTES_LIMIT=bytes;</codeph>
+ </p>
+
+ <p>
+ <b>Type:</b> numeric
+ </p>
+
+ <p>
+ <b>Units:</b>
+ <ul>
+ <li>
+ A numeric argument represents memory size in bytes.
+ </li>
+
+ <li>
+ Specify a suffix of <codeph>m</codeph> or <codeph>mb</codeph> for megabytes.
+ </li>
+
+ <li>
+ Specify a suffix of <codeph>g</codeph> or <codeph>gb</codeph> for gigabytes.
+ </li>
+
+ <li>
+ If you specify a suffix with unrecognized formats, subsequent queries fail with an
+ error.
+ </li>
+ </ul>
+ </p>
+
+ <p>
+ <b>Default:</b> <codeph>0</codeph> (no limit)
+ </p>
+
+ <p>
+ <b>Added in:</b> <keyword keyref="impala31"/>
+ </p>
+
+ </conbody>
+
+</concept>
[3/6] impala git commit: IMPALA-7342: Add initial support for
user-level permissions
Posted by ta...@apache.org.
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
index 40f604d..cdf85ed 100644
--- a/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
+++ b/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hive.metastore.api.SerDeInfo;
import org.apache.hadoop.hive.metastore.api.StorageDescriptor;
import org.apache.hadoop.hive.metastore.api.StringColumnStatsData;
import org.apache.impala.analysis.AlterTableSortByStmt;
-import org.apache.impala.analysis.ColumnName;
import org.apache.impala.analysis.FunctionName;
import org.apache.impala.analysis.TableName;
import org.apache.impala.authorization.User;
@@ -75,8 +74,8 @@ import org.apache.impala.catalog.KuduTable;
import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.catalog.PartitionNotFoundException;
import org.apache.impala.catalog.PartitionStatsUtil;
+import org.apache.impala.catalog.PrincipalPrivilege;
import org.apache.impala.catalog.Role;
-import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.catalog.RowFormat;
import org.apache.impala.catalog.ScalarFunction;
import org.apache.impala.catalog.Table;
@@ -2989,7 +2988,7 @@ public class CatalogOpExecutor {
TCatalogObject catalogObject = new TCatalogObject();
catalogObject.setType(role.getCatalogObjectType());
- catalogObject.setRole(role.toThrift());
+ catalogObject.setPrincipal(role.toThrift());
catalogObject.setCatalog_version(role.getCatalogVersion());
if (createDropRoleParams.isIs_drop()) {
resp.result.addToRemoved_catalog_objects(catalogObject);
@@ -3024,7 +3023,7 @@ public class CatalogOpExecutor {
Preconditions.checkNotNull(role);
TCatalogObject catalogObject = new TCatalogObject();
catalogObject.setType(role.getCatalogObjectType());
- catalogObject.setRole(role.toThrift());
+ catalogObject.setPrincipal(role.toThrift());
catalogObject.setCatalog_version(role.getCatalogVersion());
resp.result.addToUpdated_catalog_objects(catalogObject);
if (grantRevokeRoleParams.isIs_grant()) {
@@ -3046,7 +3045,7 @@ public class CatalogOpExecutor {
verifySentryServiceEnabled();
String roleName = grantRevokePrivParams.getRole_name();
List<TPrivilege> privileges = grantRevokePrivParams.getPrivileges();
- List<RolePrivilege> rolePrivileges = null;
+ List<PrincipalPrivilege> rolePrivileges = null;
if (grantRevokePrivParams.isIs_grant()) {
rolePrivileges = catalog_.getSentryProxy().grantRolePrivileges(requestingUser,
roleName, privileges);
@@ -3058,7 +3057,7 @@ public class CatalogOpExecutor {
}
Preconditions.checkNotNull(rolePrivileges);
List<TCatalogObject> updatedPrivs = Lists.newArrayList();
- for (RolePrivilege rolePriv: rolePrivileges) {
+ for (PrincipalPrivilege rolePriv: rolePrivileges) {
updatedPrivs.add(rolePriv.toTCatalogObject());
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/service/JniFrontend.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/service/JniFrontend.java b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
index f02db61..ce16b51 100644
--- a/fe/src/main/java/org/apache/impala/service/JniFrontend.java
+++ b/fe/src/main/java/org/apache/impala/service/JniFrontend.java
@@ -45,7 +45,6 @@ import org.apache.impala.analysis.ToSqlUtils;
import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.ImpalaInternalAdminUser;
import org.apache.impala.authorization.User;
-import org.apache.impala.catalog.DataSource;
import org.apache.impala.catalog.FeDataSource;
import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.Function;
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java b/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
index 8fc72c9..f039ee7 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryPolicyService.java
@@ -19,6 +19,7 @@ package org.apache.impala.util;
import java.util.List;
+import org.apache.impala.catalog.PrincipalPrivilege;
import org.apache.sentry.api.service.thrift.SentryPolicyServiceClient;
import org.apache.sentry.api.service.thrift.TSentryGrantOption;
import org.apache.sentry.api.service.thrift.TSentryPrivilege;
@@ -31,7 +32,6 @@ import org.apache.impala.analysis.PrivilegeSpec;
import org.apache.impala.authorization.SentryConfig;
import org.apache.impala.authorization.User;
import org.apache.impala.catalog.AuthorizationException;
-import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.InternalException;
import org.apache.impala.thrift.TPrivilege;
@@ -236,7 +236,7 @@ public class SentryPolicyService {
*
* @param requestingUser - The requesting user.
* @param roleName - The role to grant privileges to (case insensitive).
- * @param privilege - The privilege to grant.
+ * @param privileges - The privileges to grant.
* @throws ImpalaException - On any error
*/
public void grantRolePrivileges(User requestingUser, String roleName,
@@ -307,7 +307,7 @@ public class SentryPolicyService {
*
* @param requestingUser - The requesting user.
* @param roleName - The role to revoke privileges from (case insensitive).
- * @param privilege - The privilege to revoke.
+ * @param privileges - The privileges to revoke.
* @throws ImpalaException - On any error
*/
public void revokeRolePrivileges(User requestingUser, String roleName,
@@ -448,7 +448,7 @@ public class SentryPolicyService {
privilege.setPrivilege_level(Enum.valueOf(TPrivilegeLevel.class,
sentryPriv.getAction().toUpperCase()));
}
- privilege.setPrivilege_name(RolePrivilege.buildRolePrivilegeName(privilege));
+ privilege.setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(privilege));
privilege.setCreate_time_ms(sentryPriv.getCreateTime());
if (sentryPriv.isSetGrantOption() &&
sentryPriv.getGrantOption() == TSentryGrantOption.TRUE) {
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/main/java/org/apache/impala/util/SentryProxy.java
----------------------------------------------------------------------
diff --git a/fe/src/main/java/org/apache/impala/util/SentryProxy.java b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
index 7863923..fa769b6 100644
--- a/fe/src/main/java/org/apache/impala/util/SentryProxy.java
+++ b/fe/src/main/java/org/apache/impala/util/SentryProxy.java
@@ -18,11 +18,18 @@
package org.apache.impala.util;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
+import org.apache.impala.catalog.AuthorizationException;
+import org.apache.impala.catalog.CatalogException;
+import org.apache.impala.catalog.CatalogServiceCatalog;
+import org.apache.impala.catalog.PrincipalPrivilege;
+import org.apache.impala.catalog.Role;
+import org.apache.impala.thrift.TPrincipalType;
import org.apache.log4j.Logger;
import org.apache.sentry.api.service.thrift.TSentryGroup;
import org.apache.sentry.api.service.thrift.TSentryPrivilege;
@@ -30,11 +37,6 @@ import org.apache.sentry.api.service.thrift.TSentryRole;
import org.apache.impala.authorization.SentryConfig;
import org.apache.impala.authorization.User;
-import org.apache.impala.catalog.AuthorizationException;
-import org.apache.impala.catalog.CatalogException;
-import org.apache.impala.catalog.CatalogServiceCatalog;
-import org.apache.impala.catalog.Role;
-import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.common.ImpalaException;
import org.apache.impala.common.ImpalaRuntimeException;
import org.apache.impala.service.BackendConfig;
@@ -97,10 +99,10 @@ public class SentryProxy {
* There is currently no way to get a snapshot of the policy from the Sentry Service,
* so it is possible that Impala will end up in a state that is not consistent with a
* state the Sentry Service has ever been in. For example, consider the case where a
- * refresh is running and all privileges for Role A have been processed. Before moving
- * to Role B, the user revokes a privilege from Role A and grants it to Role B.
- * Impala will temporarily (until the next refresh) think the privilege is granted to
- * Role A AND to Role B.
+ * refresh is running and all privileges for Principal A have been processed. Before
+ * moving to Principal B, the user revokes a privilege from Principal A and grants it to
+ * Principal B. Impala will temporarily (until the next refresh) think the privilege is
+ * granted to Principal A AND to Principal B.
* TODO: Think more about consistency as well as how to recover from errors that leave
* the policy in a potentially inconsistent state (an RPC fails part-way through a
* refresh). We should also consider applying this entire update to the catalog
@@ -154,23 +156,25 @@ public class SentryProxy {
sentryPolicyService_.listRolePrivileges(processUser_, role.getName());
} catch (ImpalaException e) {
String roleName = role.getName() != null ? role.getName(): "null";
- LOG.error("Error listing the Role name: " + roleName, e);
+ LOG.error("Error listing the role name: " + roleName, e);
}
// Check all the privileges that are part of this role.
for (TSentryPrivilege sentryPriv: sentryPrivlist) {
TPrivilege thriftPriv =
SentryPolicyService.sentryPrivilegeToTPrivilege(sentryPriv);
- thriftPriv.setRole_id(role.getId());
+ thriftPriv.setPrincipal_id(role.getId());
+ thriftPriv.setPrincipal_type(TPrincipalType.ROLE);
+
privilegesToRemove.remove(thriftPriv.getPrivilege_name().toLowerCase());
- RolePrivilege existingPriv =
+ PrincipalPrivilege existingRolePriv =
role.getPrivilege(thriftPriv.getPrivilege_name());
// We already know about this privilege (privileges cannot be modified).
- if (existingPriv != null &&
- existingPriv.getCreateTimeMs() == sentryPriv.getCreateTime()) {
+ if (existingRolePriv != null &&
+ existingRolePriv.getCreateTimeMs() == sentryPriv.getCreateTime()) {
if (resetVersions_) {
- existingPriv.setCatalogVersion(
+ existingRolePriv.setCatalogVersion(
catalog_.incrementAndGetCatalogVersion());
}
continue;
@@ -288,11 +292,11 @@ public class SentryProxy {
* Throws exception if there was any error updating the Sentry Service or if the Impala
* catalog does not contain the given role name.
*/
- public synchronized List<RolePrivilege> grantRolePrivileges(User user,
+ public synchronized List<PrincipalPrivilege> grantRolePrivileges(User user,
String roleName, List<TPrivilege> privileges) throws ImpalaException {
sentryPolicyService_.grantRolePrivileges(user, roleName, privileges);
// Update the catalog
- List<RolePrivilege> rolePrivileges = Lists.newArrayList();
+ List<PrincipalPrivilege> rolePrivileges = Lists.newArrayList();
for (TPrivilege privilege: privileges) {
rolePrivileges.add(catalog_.addRolePrivilege(roleName, privilege));
}
@@ -306,15 +310,15 @@ public class SentryProxy {
* updating the Sentry Service or if the Impala catalog does not contain the given role
* name.
*/
- public synchronized List<RolePrivilege> revokeRolePrivileges(User user,
+ public synchronized List<PrincipalPrivilege> revokeRolePrivileges(User user,
String roleName, List<TPrivilege> privileges, boolean hasGrantOption)
throws ImpalaException {
- List<RolePrivilege> rolePrivileges = Lists.newArrayList();
+ List<PrincipalPrivilege> rolePrivileges = Lists.newArrayList();
if (!hasGrantOption) {
sentryPolicyService_.revokeRolePrivileges(user, roleName, privileges);
// Update the catalog
for (TPrivilege privilege: privileges) {
- RolePrivilege rolePriv = catalog_.removeRolePrivilege(roleName, privilege);
+ PrincipalPrivilege rolePriv = catalog_.removeRolePrivilege(roleName, privilege);
if (rolePriv == null) continue;
rolePrivileges.add(rolePriv);
}
@@ -326,7 +330,8 @@ public class SentryProxy {
sentryPolicyService_.revokeRolePrivileges(user, roleName, privileges);
List<TPrivilege> updatedPrivileges = Lists.newArrayList();
for (TPrivilege privilege: privileges) {
- RolePrivilege existingPriv = catalog_.getRolePrivilege(roleName, privilege);
+ PrincipalPrivilege existingPriv = catalog_.getPrincipalPrivilege(roleName,
+ privilege);
if (existingPriv == null) continue;
TPrivilege updatedPriv = existingPriv.toThrift();
updatedPriv.setHas_grant_opt(false);
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java b/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
index 2741e5d..dd58131 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AnalyzeAuthStmtsTest.java
@@ -29,8 +29,8 @@ import org.apache.impala.util.EventSequence;
import org.junit.Test;
public class AnalyzeAuthStmtsTest extends AnalyzerTest {
- public AnalyzeAuthStmtsTest() throws AnalysisException {
- catalog_.getAuthPolicy().addRole(
+ public AnalyzeAuthStmtsTest() {
+ catalog_.getAuthPolicy().addPrincipal(
new Role("myRole", new HashSet<String>()));
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
index 9a38300..abfff91 100644
--- a/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
+++ b/fe/src/test/java/org/apache/impala/analysis/AuthorizationStmtTest.java
@@ -25,8 +25,8 @@ import org.apache.impala.authorization.AuthorizationConfig;
import org.apache.impala.authorization.PrivilegeRequest;
import org.apache.impala.authorization.User;
import org.apache.impala.catalog.AuthorizationException;
+import org.apache.impala.catalog.PrincipalPrivilege;
import org.apache.impala.catalog.Role;
-import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.catalog.ScalarFunction;
import org.apache.impala.catalog.Type;
import org.apache.impala.common.FrontendTestBase;
@@ -38,6 +38,7 @@ import org.apache.impala.thrift.TColumnValue;
import org.apache.impala.thrift.TDescribeOutputStyle;
import org.apache.impala.thrift.TDescribeResult;
import org.apache.impala.thrift.TFunctionBinaryType;
+import org.apache.impala.thrift.TPrincipalType;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.thrift.TPrivilegeLevel;
import org.apache.impala.thrift.TPrivilegeScope;
@@ -66,7 +67,7 @@ import static org.junit.Assert.fail;
*/
public class AuthorizationStmtTest extends FrontendTestBase {
private static final String SENTRY_SERVER = "server1";
- private final static User USER = new User(System.getProperty("user.name"));
+ private static final User USER = new User(System.getProperty("user.name"));
private final AnalysisContext analysisContext_;
private final SentryPolicyService sentryService_;
private final ImpaladTestCatalog authzCatalog_;
@@ -963,7 +964,16 @@ public class AuthorizationStmtTest extends FrontendTestBase {
authorize(String.format("show role grant group `%s`", USER.getName())).ok();
// Show grant role should always be allowed.
- authorize(String.format("show grant role authz_test_role")).ok();
+ try {
+ authzCatalog_.addRole("test_role");
+ authorize("show grant role test_role").ok();
+ authorize("show grant role test_role on server").ok();
+ authorize("show grant role test_role on database functional").ok();
+ authorize("show grant role test_role on table functional.alltypes").ok();
+ authorize("show grant role test_role on uri '/test-warehouse'").ok();
+ } finally {
+ authzCatalog_.removeRole("test_role");
+ }
// Show create table.
test = authorize("show create table functional.alltypes");
@@ -2299,10 +2309,51 @@ public class AuthorizationStmtTest extends FrontendTestBase {
return privLevels.toArray(new TPrivilegeLevel[0]);
}
+ private static abstract class WithPrincipal {
+ protected final AuthzTest test_;
+
+ public WithPrincipal(AuthzTest test) { test_ = test; }
+
+ public abstract void create(TPrivilege[]... privileges) throws ImpalaException;
+ public abstract void drop() throws ImpalaException;
+ public abstract String getName();
+ }
+
+ private static class WithUser extends WithPrincipal {
+ public WithUser(AuthzTest test) { super(test); }
+
+ @Override
+ public void create(TPrivilege[]... privileges) throws ImpalaException {
+ test_.createUser(privileges);
+ }
+
+ @Override
+ public void drop() throws ImpalaException { test_.dropUser(); }
+
+ @Override
+ public String getName() { return test_.user_; }
+ }
+
+ private static class WithRole extends WithPrincipal {
+ public WithRole(AuthzTest test) { super(test); }
+
+ @Override
+ public void create(TPrivilege[]... privileges) throws ImpalaException {
+ test_.createRole(privileges);
+ }
+
+ @Override
+ public void drop() throws ImpalaException { test_.dropRole(); }
+
+ @Override
+ public String getName() { return test_.role_; }
+ }
+
private class AuthzTest {
private final AnalysisContext context_;
private final String stmt_;
private final String role_ = "authz_test_role";
+ private final String user_ = USER.getName();
public AuthzTest(String stmt) {
this(null, stmt);
@@ -2319,95 +2370,121 @@ public class AuthorizationStmtTest extends FrontendTestBase {
authzCatalog_.addRoleGrantGroup(role_, USER.getName());
for (TPrivilege[] privs: privileges) {
for (TPrivilege privilege: privs) {
- privilege.setRole_id(role.getId());
+ privilege.setPrincipal_id(role.getId());
+ privilege.setPrincipal_type(TPrincipalType.ROLE);
authzCatalog_.addRolePrivilege(role_, privilege);
}
}
}
+ private void createUser(TPrivilege[]... privileges) throws ImpalaException {
+ org.apache.impala.catalog.User user = authzCatalog_.addUser(user_);
+ for (TPrivilege[] privs: privileges) {
+ for (TPrivilege privilege: privs) {
+ privilege.setPrincipal_id(user.getId());
+ privilege.setPrincipal_type(TPrincipalType.USER);
+ authzCatalog_.addUserPrivilege(user_, privilege);
+ }
+ }
+ }
+
private void dropRole() throws ImpalaException {
authzCatalog_.removeRole(role_);
}
+ private void dropUser() throws ImpalaException {
+ authzCatalog_.removeUser(user_);
+ }
+
/**
- * This method runs with the specified privileges.
+ * This method runs with the specified privileges for the role and then for the user.
*
- * A new temporary role will be created and assigned to the specified privileges
- * into the new role. The new role will be dropped once this method finishes.
+ * A new temporary role/user will be created and assigned to the specified privileges
+ * into the new role/user. The new role/user will be dropped once this method
+ * finishes.
*/
public AuthzTest ok(TPrivilege[]... privileges) throws ImpalaException {
- try {
- createRole(privileges);
- if (context_ != null) {
- authzOk(context_, stmt_);
- } else {
- authzOk(stmt_);
+ for (WithPrincipal withPrincipal: new WithPrincipal[]{
+ new WithRole(this), new WithUser(this)}) {
+ try {
+ withPrincipal.create(privileges);
+ if (context_ != null) {
+ authzOk(context_, stmt_, withPrincipal);
+ } else {
+ authzOk(stmt_, withPrincipal);
+ }
+ } finally {
+ withPrincipal.drop();
}
- } catch (AuthorizationException ae) {
- // Because the same test can be called from multiple statements
- // it is useful to know which statement caused the exception.
- throw new AuthorizationException(stmt_ + ": " + ae.getMessage(), ae);
- } finally {
- dropRole();
}
return this;
}
/**
- * This method runs with the specified privileges and checks describe output.
+ * This method runs with the specified privileges and checks describe output for the
+ * role and then the user.
*
- * A new temporary role will be created and assigned to the specified privileges
- * into the new role. The new role will be dropped once this method finishes.
+ * A new temporary role/user will be created and assigned to the specified privileges
+ * into the new role/user. The new role/user will be dropped once this method
+ * finishes.
*/
public AuthzTest okDescribe(TTableName table, TDescribeOutputStyle style,
String[] requiredStrings, String[] excludedStrings, TPrivilege[]... privileges)
throws ImpalaException {
- try {
- createRole(privileges);
- if (context_ != null) {
- authzOk(context_, stmt_);
- } else {
- authzOk(stmt_);
- }
- List<String> result = resultToStringList(authzFrontend_.describeTable(table,
- style, USER));
- if (requiredStrings != null) {
- for (String str: requiredStrings) {
- assertTrue(String.format("\"%s\" is not in the describe output.\n" +
- "Expected : %s\n" +
- "Actual : %s", str, Arrays.toString(requiredStrings), result),
- result.contains(str));
+ for (WithPrincipal withPrincipal: new WithPrincipal[]{
+ new WithRole(this), new WithUser(this)}) {
+ try {
+ withPrincipal.create(privileges);
+ if (context_ != null) {
+ authzOk(context_, stmt_, withPrincipal);
+ } else {
+ authzOk(stmt_, withPrincipal);
}
- }
- if (excludedStrings != null) {
- for (String str: excludedStrings) {
- assertTrue(String.format("\"%s\" should not be in the describe output.", str),
- !result.contains(str));
+ List<String> result = resultToStringList(authzFrontend_.describeTable(table,
+ style, USER));
+ if (requiredStrings != null) {
+ for (String str : requiredStrings) {
+ assertTrue(String.format("\"%s\" is not in the describe output.\n" +
+ "Expected : %s\n" +
+ "Actual : %s", str, Arrays.toString(requiredStrings), result),
+ result.contains(str));
+ }
}
+ if (excludedStrings != null) {
+ for (String str : excludedStrings) {
+ assertTrue(String.format(
+ "\"%s\" should not be in the describe output.", str),
+ !result.contains(str));
+ }
+ }
+ } finally {
+ withPrincipal.drop();
}
- } finally {
- dropRole();
}
return this;
}
/**
- * This method runs with the specified privileges.
+ * This method runs with the specified privileges for the user and then the role.
*
- * A new temporary role will be created and assigned to the specified privileges
- * into the new role. The new role will be dropped once this method finishes.
+ * A new temporary role/user will be created and assigned to the specified privileges
+ * into the new role/user. The new role/user will be dropped once this method
+ * finishes.
*/
public AuthzTest error(String expectedError, TPrivilege[]... privileges)
throws ImpalaException {
- try {
- createRole(privileges);
- if (context_ != null) {
- authzError(context_, stmt_, expectedError);
- } else {
- authzError(stmt_, expectedError);
+ for (WithPrincipal withPrincipal: new WithPrincipal[]{
+ new WithRole(this), new WithUser(this)}) {
+ try {
+ withPrincipal.create(privileges);
+ if (context_ != null) {
+ authzError(context_, stmt_, expectedError, withPrincipal);
+ } else {
+ authzError(stmt_, expectedError, withPrincipal);
+ }
+ } finally {
+ withPrincipal.drop();
}
- } finally {
- dropRole();
}
return this;
}
@@ -2426,7 +2503,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
for (int i = 0; i < levels.length; i++) {
privileges[i] = new TPrivilege("", levels[i], TPrivilegeScope.SERVER, false);
privileges[i].setServer_name(SENTRY_SERVER);
- privileges[i].setPrivilege_name(RolePrivilege.buildRolePrivilegeName(
+ privileges[i].setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(
privileges[i]));
}
return privileges;
@@ -2438,7 +2515,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
privileges[i] = new TPrivilege("", levels[i], TPrivilegeScope.DATABASE, false);
privileges[i].setServer_name(SENTRY_SERVER);
privileges[i].setDb_name(db);
- privileges[i].setPrivilege_name(RolePrivilege.buildRolePrivilegeName(
+ privileges[i].setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(
privileges[i]));
}
return privileges;
@@ -2451,7 +2528,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
privileges[i].setServer_name(SENTRY_SERVER);
privileges[i].setDb_name(db);
privileges[i].setTable_name(table);
- privileges[i].setPrivilege_name(RolePrivilege.buildRolePrivilegeName(
+ privileges[i].setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(
privileges[i]));
}
return privileges;
@@ -2474,7 +2551,7 @@ public class AuthorizationStmtTest extends FrontendTestBase {
privileges[idx].setDb_name(db);
privileges[idx].setTable_name(table);
privileges[idx].setColumn_name(column);
- privileges[idx].setPrivilege_name(RolePrivilege.buildRolePrivilegeName(
+ privileges[idx].setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(
privileges[idx]));
idx++;
}
@@ -2488,47 +2565,56 @@ public class AuthorizationStmtTest extends FrontendTestBase {
privileges[i] = new TPrivilege("", levels[i], TPrivilegeScope.URI, false);
privileges[i].setServer_name(SENTRY_SERVER);
privileges[i].setUri(uri);
- privileges[i].setPrivilege_name(RolePrivilege.buildRolePrivilegeName(
+ privileges[i].setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(
privileges[i]));
}
return privileges;
}
- private void authzOk(String stmt) throws ImpalaException {
- authzOk(analysisContext_, stmt);
+ private void authzOk(String stmt, WithPrincipal withPrincipal) throws ImpalaException {
+ authzOk(analysisContext_, stmt, withPrincipal);
}
- private void authzOk(AnalysisContext context, String stmt) throws ImpalaException {
- authzOk(authzFrontend_, context, stmt);
+ private void authzOk(AnalysisContext context, String stmt, WithPrincipal withPrincipal)
+ throws ImpalaException {
+ authzOk(authzFrontend_, context, stmt, withPrincipal);
}
- private void authzOk(Frontend fe, AnalysisContext context, String stmt)
- throws ImpalaException {
- parseAndAnalyze(stmt, context, fe);
+ private void authzOk(Frontend fe, AnalysisContext context, String stmt,
+ WithPrincipal withPrincipal) throws ImpalaException {
+ try {
+ parseAndAnalyze(stmt, context, fe);
+ } catch (AuthorizationException e) {
+ // Because the same test can be called from multiple statements
+ // it is useful to know which statement caused the exception.
+ throw new AuthorizationException(String.format(
+ "\nPrincipal: %s\nStatement: %s\nError: %s", withPrincipal.getName(),
+ stmt, e.getMessage(), e));
+ }
}
/**
* Verifies that a given statement fails authorization and the expected error
* string matches.
*/
- private void authzError(String stmt, String expectedError, Matcher matcher)
- throws ImpalaException {
- authzError(analysisContext_, stmt, expectedError, matcher);
+ private void authzError(String stmt, String expectedError, Matcher matcher,
+ WithPrincipal withPrincipal) throws ImpalaException {
+ authzError(analysisContext_, stmt, expectedError, matcher, withPrincipal);
}
- private void authzError(String stmt, String expectedError)
+ private void authzError(String stmt, String expectedError, WithPrincipal withPrincipal)
throws ImpalaException {
- authzError(analysisContext_, stmt, expectedError, startsWith());
+ authzError(analysisContext_, stmt, expectedError, startsWith(), withPrincipal);
}
private void authzError(AnalysisContext ctx, String stmt, String expectedError,
- Matcher matcher) throws ImpalaException {
- authzError(authzFrontend_, ctx, stmt, expectedError, matcher);
+ Matcher matcher, WithPrincipal withPrincipal) throws ImpalaException {
+ authzError(authzFrontend_, ctx, stmt, expectedError, matcher, withPrincipal);
}
- private void authzError(AnalysisContext ctx, String stmt, String expectedError)
- throws ImpalaException {
- authzError(authzFrontend_, ctx, stmt, expectedError, startsWith());
+ private void authzError(AnalysisContext ctx, String stmt, String expectedError,
+ WithPrincipal withPrincipal) throws ImpalaException {
+ authzError(authzFrontend_, ctx, stmt, expectedError, startsWith(), withPrincipal);
}
private interface Matcher {
@@ -2553,8 +2639,8 @@ public class AuthorizationStmtTest extends FrontendTestBase {
};
}
- private void authzError(Frontend fe, AnalysisContext ctx,
- String stmt, String expectedErrorString, Matcher matcher)
+ private void authzError(Frontend fe, AnalysisContext ctx, String stmt,
+ String expectedErrorString, Matcher matcher, WithPrincipal withPrincipal)
throws ImpalaException {
Preconditions.checkNotNull(expectedErrorString);
try {
@@ -2568,7 +2654,8 @@ public class AuthorizationStmtTest extends FrontendTestBase {
matcher.match(errorString, expectedErrorString));
return;
}
- fail("Stmt didn't result in authorization error: " + stmt);
+ fail(String.format("Statement did not result in authorization error.\n" +
+ "Principal: %s\nStatement: %s", withPrincipal.getName(), stmt));
}
private void verifyPrivilegeReqs(String stmt, Set<String> expectedPrivilegeNames)
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
index 7ff5054..ad1595b 100644
--- a/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
+++ b/fe/src/test/java/org/apache/impala/catalog/CatalogTest.java
@@ -18,14 +18,17 @@
package org.apache.impala.catalog;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -44,6 +47,10 @@ import org.apache.impala.catalog.MetaStoreClientPool.MetaStoreClient;
import org.apache.impala.common.Reference;
import org.apache.impala.testutil.CatalogServiceTestCatalog;
import org.apache.impala.thrift.TFunctionBinaryType;
+import org.apache.impala.thrift.TPrincipalType;
+import org.apache.impala.thrift.TPrivilege;
+import org.apache.impala.thrift.TPrivilegeLevel;
+import org.apache.impala.thrift.TPrivilegeScope;
import org.apache.impala.thrift.TTableName;
import org.junit.Test;
@@ -699,4 +706,84 @@ public class CatalogTest {
fnNames = getFunctionSignatures("default");
assertEquals(fnNames.size(), 0);
}
+
+ @Test
+ public void testSentryCatalog() throws CatalogException {
+ AuthorizationPolicy authPolicy = catalog_.getAuthPolicy();
+
+ User user = catalog_.addUser("user1");
+ TPrivilege userPrivilege = new TPrivilege();
+ userPrivilege.setPrincipal_type(TPrincipalType.USER);
+ userPrivilege.setPrincipal_id(user.getId());
+ userPrivilege.setCreate_time_ms(-1);
+ userPrivilege.setServer_name("server1");
+ userPrivilege.setScope(TPrivilegeScope.SERVER);
+ userPrivilege.setPrivilege_level(TPrivilegeLevel.ALL);
+ userPrivilege.setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(userPrivilege));
+ catalog_.addUserPrivilege("user1", userPrivilege);
+ assertSame(user, authPolicy.getPrincipal("user1", TPrincipalType.USER));
+ assertNull(authPolicy.getPrincipal("user2", TPrincipalType.USER));
+ assertNull(authPolicy.getPrincipal("user1", TPrincipalType.ROLE));
+ // Add the same user, the old user will be deleted.
+ user = catalog_.addUser("user1");
+ assertSame(user, authPolicy.getPrincipal("user1", TPrincipalType.USER));
+ // Delete the user.
+ assertSame(user, catalog_.removeUser("user1"));
+ assertNull(authPolicy.getPrincipal("user1", TPrincipalType.USER));
+
+ Role role = catalog_.addRole("role1", Sets.newHashSet("group1", "group2"));
+ TPrivilege rolePrivilege = new TPrivilege();
+ rolePrivilege.setPrincipal_type(TPrincipalType.ROLE);
+ rolePrivilege.setPrincipal_id(role.getId());
+ rolePrivilege.setCreate_time_ms(-1);
+ rolePrivilege.setServer_name("server1");
+ rolePrivilege.setScope(TPrivilegeScope.SERVER);
+ rolePrivilege.setPrivilege_level(TPrivilegeLevel.ALL);
+ rolePrivilege.setPrivilege_name(PrincipalPrivilege.buildPrivilegeName(rolePrivilege));
+ catalog_.addRolePrivilege("role1", rolePrivilege);
+ assertSame(role, catalog_.getAuthPolicy().getPrincipal("role1", TPrincipalType.ROLE));
+ assertNull(catalog_.getAuthPolicy().getPrincipal("role1", TPrincipalType.USER));
+ assertNull(catalog_.getAuthPolicy().getPrincipal("role2", TPrincipalType.ROLE));
+ // Add the same role, the old role will be deleted.
+ role = catalog_.addRole("role1", new HashSet<String>());
+ assertSame(role, authPolicy.getPrincipal("role1", TPrincipalType.ROLE));
+ // Delete the role.
+ assertSame(role, catalog_.removeRole("role1"));
+ assertNull(authPolicy.getPrincipal("role1", TPrincipalType.ROLE));
+
+ // Assert that principal IDs will be unique between roles and users, e.g. no user and
+ // role with the same principal ID. The same name can be used for both user and role.
+ int size = 10;
+ String prefix = "foo";
+ for (int i = 0; i < size; i++) {
+ String name = prefix + i;
+ catalog_.addUser(name);
+ catalog_.addRole(name, new HashSet<String>());
+ }
+
+ for (int i = 0; i < size; i++) {
+ String name = prefix + i;
+ Principal u = authPolicy.getPrincipal(name, TPrincipalType.USER);
+ Principal r = authPolicy.getPrincipal(name, TPrincipalType.ROLE);
+ assertEquals(name, u.getName());
+ assertEquals(name, r.getName());
+ assertNotEquals(u.getId(), r.getId());
+ }
+
+ // Validate getAllUsers vs getAllUserNames
+ List<User> allUsers = authPolicy.getAllUsers();
+ Set<String> allUserNames = authPolicy.getAllUserNames();
+ assertEquals(allUsers.size(), allUserNames.size());
+ for (Principal principal: allUsers) {
+ assertTrue(allUserNames.contains(principal.getName()));
+ }
+
+ // Validate getAllRoles and getAllRoleNames work as expected.
+ List<Role> allRoles = authPolicy.getAllRoles();
+ Set<String> allRoleNames = authPolicy.getAllRoleNames();
+ assertEquals(allRoles.size(), allRoleNames.size());
+ for (Principal principal: allRoles) {
+ assertTrue(allRoleNames.contains(principal.getName()));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/impala/blob/a23e6f29/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
----------------------------------------------------------------------
diff --git a/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java b/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
index 2b2fdf8..3186113 100644
--- a/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
+++ b/fe/src/test/java/org/apache/impala/testutil/ImpaladTestCatalog.java
@@ -27,9 +27,11 @@ import org.apache.impala.catalog.FeDb;
import org.apache.impala.catalog.HdfsCachePool;
import org.apache.impala.catalog.HdfsTable;
import org.apache.impala.catalog.ImpaladCatalog;
+import org.apache.impala.catalog.Principal;
+import org.apache.impala.catalog.PrincipalPrivilege;
import org.apache.impala.catalog.Role;
-import org.apache.impala.catalog.RolePrivilege;
import org.apache.impala.catalog.Table;
+import org.apache.impala.catalog.User;
import org.apache.impala.thrift.TPrivilege;
import org.apache.impala.util.PatternMatcher;
@@ -123,14 +125,26 @@ public class ImpaladTestCatalog extends ImpaladCatalog {
return srcCatalog_.addRole(roleName, new HashSet<String>());
}
- public Role addRoleGrantGroup(String roleName, String groupName) throws CatalogException {
+ public Role addRoleGrantGroup(String roleName, String groupName)
+ throws CatalogException {
return srcCatalog_.addRoleGrantGroup(roleName, groupName);
}
- public RolePrivilege addRolePrivilege(String roleName, TPrivilege privilege)
+ public PrincipalPrivilege addRolePrivilege(String roleName, TPrivilege privilege)
throws CatalogException {
return srcCatalog_.addRolePrivilege(roleName, privilege);
}
public void removeRole(String roleName) { srcCatalog_.removeRole(roleName); }
+
+ public User addUser(String userName) {
+ return srcCatalog_.addUser(userName);
+ }
+
+ public PrincipalPrivilege addUserPrivilege(String userName, TPrivilege privilege)
+ throws CatalogException {
+ return srcCatalog_.addUserPrivilege(userName, privilege);
+ }
+
+ public void removeUser(String userName) { srcCatalog_.removeUser(userName); }
}
[6/6] impala git commit: IMPALA-7442: reduce mem requirement of
semi-joins-exhaustive
Posted by ta...@apache.org.
IMPALA-7442: reduce mem requirement of semi-joins-exhaustive
The test started running into IMPALA-7446, maybe because of
a timing change. This appears to have always been possible.
The fix is to reduce the memory requirement of the test.
IMPALA-2256 is no longer really possible because the
BufferedTupleStream code was simplified to avoid the 32-bit
row index limitation, so we're not losing important coverage
on the current code with this change.
Testing:
Ran test in a loop to confirm it did not OOM.
Change-Id: I9d9480cad6bf8222abe990e7046498a0531e2849
Reviewed-on: http://gerrit.cloudera.org:8080/11223
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/29905d1e
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/29905d1e
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/29905d1e
Branch: refs/heads/master
Commit: 29905d1ea41b3c0e7ceebf913981173292dfffbf
Parents: 48fdd0b
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Aug 14 16:22:55 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Aug 16 00:21:54 2018 +0000
----------------------------------------------------------------------
.../queries/QueryTest/semi-joins-exhaustive.test | 15 +++++++--------
1 file changed, 7 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/29905d1e/testdata/workloads/functional-query/queries/QueryTest/semi-joins-exhaustive.test
----------------------------------------------------------------------
diff --git a/testdata/workloads/functional-query/queries/QueryTest/semi-joins-exhaustive.test b/testdata/workloads/functional-query/queries/QueryTest/semi-joins-exhaustive.test
index a73b0ec..2a7f101 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/semi-joins-exhaustive.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/semi-joins-exhaustive.test
@@ -1,18 +1,17 @@
====
---- QUERY
-# Regression test for IMPALA-2256. Join whose right side has very high
-# cardinality (60M) and zero materialized slots.
-# Because the right side of the join here is always
-# the same key, this query can run out of memory and fail to spill; see
-# IMPALA-4857. The cardinality (60M) is chosen so that the test
-# runs when impalad has a 7.8GB memlimit. (The peak memory usage
-# of the relevant fragment is 3.6GB when tested.)
+# Regression test for IMPALA-2256. Join whose right side has high cardinality and
+# zero materialized slots. Because the right side of the join here is always the
+# same key, this query can run out of memory and fail to spill; see IMPALA-4856.
+# The cardinality (~12M) is chosen so that the test run successfully in parallel
+# with other tests when impalad has a 7.8GB memlimit. (The peak memory usage of
+# the relevant fragment is ~850MB when tested.)
SELECT straight_join
COUNT(*) FROM alltypesagg t1
WHERE t1.int_col IN (
SELECT 1 FROM alltypesagg t1
CROSS JOIN alltypesagg t2
- WHERE t1.int_col < 500)
+ WHERE t1.int_col < 100)
---- RESULTS
10
---- TYPES