You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by jo...@apache.org on 2019/07/18 17:57:20 UTC
[impala] branch master updated (4dfe19c -> 6907528)
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 4dfe19c Bump CDP_BUILD_NUMBER to 1268086
new 7364912 IMPALA-8750: Fix profile observability tests
new 6907528 IMPALA-8493 IMPALA-8494: [DOCS] GRANT/REVOKE User and Group
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
docs/topics/impala_grant.xml | 34 ++++++++++++++++-----
docs/topics/impala_revoke.xml | 30 ++++++++++++++++---
tests/common/impala_service.py | 9 ++++++
tests/query_test/test_observability.py | 54 ++++++++++++++++++++++++++++++----
4 files changed, 110 insertions(+), 17 deletions(-)
[impala] 01/02: IMPALA-8750: Fix profile observability tests
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 7364912088f66d2fb329d40b884763d1d1d63619
Author: Tamas Mate <tm...@cloudera.com>
AuthorDate: Fri Jul 12 19:11:43 2019 +0200
IMPALA-8750: Fix profile observability tests
IMPALA-8443 adds a test 'test_query_profile_contains_query_compilation_events'
which can fail because there are two parts of the 'Query Compilation' events
that can change based on the build environment.
1) Whether the metadata is cached or not before the test starts.
2) Whether 'lineage_event_log_dir' is configured on the cluster.
This change covers these scenarios by splitting the tests into sepearate ones
where the catalog cache is pre-evicted/pre-loaded and taking into consideration
the current cluster configuration.
Change-Id: I65a1e81870e808f0e261f8a6097efdcc6903912a
Reviewed-on: http://gerrit.cloudera.org:8080/13851
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/common/impala_service.py | 9 ++++++
tests/query_test/test_observability.py | 54 ++++++++++++++++++++++++++++++----
2 files changed, 57 insertions(+), 6 deletions(-)
diff --git a/tests/common/impala_service.py b/tests/common/impala_service.py
index 7697724..e5aca37 100644
--- a/tests/common/impala_service.py
+++ b/tests/common/impala_service.py
@@ -84,6 +84,15 @@ class BaseImpalaService(object):
return [metrics.get(metric_name, default_value)
for metric_name, default_value in zip(metric_names, default_values)]
+ def get_flag_current_value(self, flag):
+ """Returns the value of the the given flag name from the Impala /varz debug webpage.
+ If the flag does not exist it returns None."""
+ varz = json.loads(self.read_debug_webpage("varz?json"))
+ for var in varz.get("flags"):
+ if var["name"] == flag:
+ return var["current"]
+ return None
+
def wait_for_metric_value(self, metric_name, expected_value, timeout=10, interval=1):
start_time = time()
while (time() - start_time < timeout):
diff --git a/tests/query_test/test_observability.py b/tests/query_test/test_observability.py
index 6f397df..3d12033 100644
--- a/tests/query_test/test_observability.py
+++ b/tests/query_test/test_observability.py
@@ -279,18 +279,60 @@ class TestObservability(ImpalaTestSuite):
assert results.runtime_profile.count("AGGREGATION_NODE") == 2
assert results.runtime_profile.count("PLAN_ROOT_SINK") == 2
- def test_query_profile_contains_query_compilation_events(self):
- """Test that the expected events show up in a query profile.
- If the table metadata is not cached this test will fail, as the metadata load
- creates lines dynamically."""
- event_regexes = [r'Query Compilation:',
- r'Metadata of all .* tables cached:',
+ def test_query_profile_contains_query_compilation_static_events(self):
+ """Test that the expected events show up in a query profile. These lines are static
+ and should appear in this exact order."""
+ event_regexes = [
r'Analysis finished:',
r'Authorization finished (.*):',
r'Value transfer graph computed:',
r'Single node plan created:',
r'Runtime filters computed:',
+ r'Distributed plan created:']
+ query = "select * from functional.alltypes"
+ runtime_profile = self.execute_query(query).runtime_profile
+ self.__verify_profile_event_sequence(event_regexes, runtime_profile)
+
+ def test_query_profile_contains_query_compilation_metadata_load_events(self):
+ """Test that the Metadata load started and finished events appear in the query
+ profile when Catalog cache is evicted."""
+ invalidate_query = "invalidate metadata functional.alltypes"
+ select_query = "select * from functional.alltypes"
+ self.execute_query(invalidate_query).runtime_profile
+ runtime_profile = self.execute_query(select_query).runtime_profile
+ event_regexes = [r'Query Compilation:',
+ r'Metadata load started:',
+ r'Metadata load finished. loaded-tables=.*/.* load-requests=.* '
+ r'catalog-updates=.*:',
+ r'Analysis finished:']
+ self.__verify_profile_event_sequence(event_regexes, runtime_profile)
+
+ def test_query_profile_contains_query_compilation_metadata_cached_event(self):
+ """Test that the Metadata cache available event appears in the query profile when
+ the table is cached."""
+ refresh_query = "refresh functional.alltypes"
+ select_query = "select * from functional.alltypes"
+ self.execute_query(refresh_query).runtime_profile
+ runtime_profile = self.execute_query(select_query).runtime_profile
+ event_regexes = [r'Query Compilation:',
+ r'Metadata of all .* tables cached:',
+ r'Analysis finished:']
+ self.__verify_profile_event_sequence(event_regexes, runtime_profile)
+
+ def test_query_profile_contains_query_compilation_lineage_event(self):
+ """Test that the lineage information appears in the profile in the right place. This
+ event depends on whether the lineage_event_log_dir is configured."""
+ impalad = self.impalad_test_service
+ lineage_event_log_dir_value = impalad.get_flag_current_value("lineage_event_log_dir")
+ assert lineage_event_log_dir_value is not None
+ if lineage_event_log_dir_value == "":
+ event_regexes = [
+ r'Distributed plan created:',
+ r'Planning finished:']
+ else:
+ event_regexes = [
r'Distributed plan created:',
+ r'Lineage info computed:',
r'Planning finished:']
query = "select * from functional.alltypes"
runtime_profile = self.execute_query(query).runtime_profile
[impala] 02/02: IMPALA-8493 IMPALA-8494: [DOCS] GRANT/REVOKE User
and Group
Posted by jo...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
joemcdonnell pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 6907528089abab78e7f07b3327eb7a9c5268db50
Author: Alex Rodoni <ar...@cloudera.com>
AuthorDate: Sat Jul 13 13:39:51 2019 -1000
IMPALA-8493 IMPALA-8494: [DOCS] GRANT/REVOKE User and Group
- Only when Ranger is used.
Change-Id: I99165bcffa76b8da983fdbf905df4035f4afafe9
Reviewed-on: http://gerrit.cloudera.org:8080/13861
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Austin Nobis <an...@cloudera.com>
Reviewed-by: Alex Rodoni <ar...@cloudera.com>
---
docs/topics/impala_grant.xml | 34 +++++++++++++++++++++++++++-------
docs/topics/impala_revoke.xml | 30 ++++++++++++++++++++++++++----
2 files changed, 53 insertions(+), 11 deletions(-)
diff --git a/docs/topics/impala_grant.xml b/docs/topics/impala_grant.xml
index 2540767..669e112 100644
--- a/docs/topics/impala_grant.xml
+++ b/docs/topics/impala_grant.xml
@@ -46,16 +46,20 @@ under the License.
<conbody>
<p rev="2.0.0">
- The <codeph>GRANT</codeph> statement grants a privilege on a specified object to a role or
- grants a role to a group.
+ The <codeph>GRANT</codeph> statement grants a privilege on a specified object to a role,
+ to a user, or to a group. The statement is also used to grant a role to a group.
</p>
<p conref="../shared/impala_common.xml#common/syntax_blurb"/>
+ <p>
+ The following syntax is supported when Impala is using Sentry to manage authorization.
+ </p>
+
<codeblock rev="2.3.0 collevelauth">GRANT ROLE <varname>role_name</varname> TO GROUP <varname>group_name</varname>
GRANT <varname>privilege</varname> ON <varname>object_type</varname> <varname>object_name</varname>
- TO [ROLE] <varname>roleName</varname>
+ TO [ROLE] <varname>role_name</varname>
[WITH GRANT OPTION]
<ph id="privileges" rev="3.0"><varname>privilege</varname> ::= ALL | ALTER | CREATE | DROP | INSERT | REFRESH | SELECT | SELECT(<varname>column_name</varname>)</ph>
@@ -63,6 +67,18 @@ GRANT <varname>privilege</varname> ON <varname>object_type</varname> <varname>ob
</codeblock>
<p>
+ The following syntax is supported when Impala is using Ranger to manage authorization.
+ </p>
+
+<codeblock>GRANT <varname>privilege</varname> ON <varname>object_type</varname> <varname>object_name</varname>
+ TO USER <varname>user_name</varname>
+GRANT <varname>privilege</varname> ON <varname>object_type</varname> <varname>object_name</varname>
+ TO GROUP <varname>group_name</varname>
+
+<ph rev="3.0">privilege ::= ALL | ALTER | CREATE | DROP | INSERT | REFRESH | SELECT | SELECT(<varname>column_name</varname>)</ph>
+<ph rev="3.0">object_type ::= SERVER | URI | DATABASE | TABLE</ph></codeblock>
+
+ <p>
Typically, the <codeph><varname>object_name</varname></codeph> is an identifier. For URIs,
it is a string literal.
</p>
@@ -70,9 +86,12 @@ GRANT <varname>privilege</varname> ON <varname>object_type</varname> <varname>ob
<p conref="../shared/impala_common.xml#common/privileges_blurb"/>
<p>
- Only Sentry administrative users, users who belong to the groups defined in
- <codeph>sentry.service.admin.group</codeph> of the Sentry configuration can grant roles to
- a group.
+ Only administrative users (initially, a predefined set of users specified in the Sentry
+ service configuration file) can use this statement.
+ </p>
+
+ <p>
+ Only Sentry administrative users can grant roles to a group.
</p>
<p>
@@ -88,7 +107,8 @@ GRANT <varname>privilege</varname> ON <varname>object_type</varname> <varname>ob
<p rev="2.3.0 collevelauth">
The ability to grant or revoke <codeph>SELECT</codeph> privilege on specific columns is
available in <keyword keyref="impala23_full"/> and higher. See
- <xref keyref="sg_hive_sql"/> for details.
+ <xref
+ keyref="sg_hive_sql"/> for details.
</p>
<p>
diff --git a/docs/topics/impala_revoke.xml b/docs/topics/impala_revoke.xml
index c4f654e..bf6d0f0 100644
--- a/docs/topics/impala_revoke.xml
+++ b/docs/topics/impala_revoke.xml
@@ -47,11 +47,15 @@ under the License.
<p rev="2.0.0">
The <codeph>REVOKE</codeph> statement revokes roles or privileges on a specified object
- from groups.
+ from groups, roles, or users.
</p>
<p conref="../shared/impala_common.xml#common/syntax_blurb"/>
+ <p>
+ The following syntax is supported when Impala is using Senty to manage authorization.
+ </p>
+
<codeblock rev="2.3.0 collevelauth">REVOKE ROLE <varname>role_name</varname> FROM GROUP <varname>group_name</varname>
REVOKE [GRANT OPTION FOR] <varname>privilege</varname> ON <varname>object_type</varname> <varname>object_name</varname>
@@ -63,6 +67,21 @@ REVOKE [GRANT OPTION FOR] <varname>privilege</varname> ON <varname>object_type</
<ph id="priv_objs" rev="3.0"><varname>object_type</varname> ::= SERVER | URI | DATABASE | TABLE</ph></codeblock>
<p>
+ The following syntax is supported when Impala is using Ranger to manage authorization.
+ </p>
+
+<codeblock>REVOKE <varname>privilege</varname> ON <varname>object_type</varname> <varname>object_name</varname>
+ FROM USER <varname>user_name</varname>
+
+REVOKE <varname>privilege</varname> ON <varname>object_type</varname> <varname>object_name</varname>
+ FROM GROUP <varname>group_name</varname>
+<ph rev="3.0">
+
+ privilege ::= ALL | ALTER | CREATE | DROP | INSERT | REFRESH | SELECT | SELECT(<varname>column_name</varname>)
+</ph>
+<ph rev="3.0">object_type ::= SERVER | URI | DATABASE | TABLE</ph></codeblock>
+
+ <p>
<b>Usage notes:</b>
</p>
@@ -106,9 +125,12 @@ REVOKE ALL ON SERVER FROM ROLE foo_role;</codeblock>
<p conref="../shared/impala_common.xml#common/privileges_blurb"/>
<p>
- Only Sentry administrative users, users who belong to the groups defined in
- <codeph>sentry.service.admin.group</codeph> of the Sentry configuration can revoke a role
- from a group.
+ Only administrative users (those with <codeph>ALL</codeph> privileges on the server,
+ defined in the Sentry policy file) can use this statement.
+ </p>
+
+ <p>
+ Only Sentry administrative users can revoke the role from a group.
</p>
<p conref="../shared/impala_common.xml#common/compatibility_blurb"/>