You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by cs...@apache.org on 2019/09/11 13:06:41 UTC
[impala] branch master updated (de77c61 -> b37dd05)
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from de77c61 IMPALA-8917: Remove hostname from webui links if Knox isn't being used
new b11a985 IMPALA-8897 (part 3): Fix some additional javascript urls
new 576f205 IMPALA-8936: Improve queuing reason for unhealthy executor groups
new b37dd05 IMPALA-8933: Enforce ranger deny policy
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/scheduling/admission-controller.cc | 5 +++--
.../apache/impala/authorization/BaseAuthorizationChecker.java | 7 ++++++-
.../impala/authorization/ranger/RangerAuditLogTest.java | 4 ++--
tests/custom_cluster/test_coordinators.py | 4 ++--
tests/custom_cluster/test_executor_groups.py | 11 +++++------
tests/webserver/test_web_pages.py | 5 ++++-
www/query_backends.tmpl | 2 +-
www/query_finstances.tmpl | 2 +-
8 files changed, 24 insertions(+), 16 deletions(-)
[impala] 01/03: IMPALA-8897 (part 3): Fix some additional
javascript urls
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit b11a985677ab49f1db114ee39719fce1d19f5747
Author: Thomas Tauber-Marshall <tm...@cloudera.com>
AuthorDate: Tue Sep 10 12:59:49 2019 -0700
IMPALA-8897 (part 3): Fix some additional javascript urls
Part 2 of this patch missed rewriting two urls in javascript.
This patch fixes it by adding 'make_url()' around those urls.
Testing:
- Deployed to a real cluster and verified that Knox integreation
works correctly on the relevant pages now.
- Extended test_knox_integration to check for this case.
Change-Id: I07a099b3a23324ad56133c5584d6a9bc88541582
Reviewed-on: http://gerrit.cloudera.org:8080/14206
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
tests/webserver/test_web_pages.py | 5 ++++-
www/query_backends.tmpl | 2 +-
www/query_finstances.tmpl | 2 +-
3 files changed, 6 insertions(+), 3 deletions(-)
diff --git a/tests/webserver/test_web_pages.py b/tests/webserver/test_web_pages.py
index 98d4801..771a67c 100644
--- a/tests/webserver/test_web_pages.py
+++ b/tests/webserver/test_web_pages.py
@@ -675,7 +675,10 @@ class TestWebPage(ImpalaTestSuite):
form_regex = "<form [^{]*?>(?!{{>www/form-hidden-inputs.tmpl}})"
# Matches XMLHttpRequest.open() in javascript that are not followed with make_url().
javascript_regex = "open\(['\"]GET['\"], (?!make_url)"
- regex = "(%s)|(%s)|(%s)" % (href_regex, form_regex, javascript_regex)
+ # Matches urls in json parameters passed to DataTables.
+ datatables_regex = "url: ['\"](?!make_url)"
+ regex = "(%s)|(%s)|(%s)|(%s)" % \
+ (href_regex, form_regex, javascript_regex, datatables_regex)
results = grep_dir(os.path.join(os.environ['IMPALA_HOME'], "www"), regex, ".*\.tmpl")
assert len(results) == 0, \
"All links on the webui must include the webserver host: %s" % results
diff --git a/www/query_backends.tmpl b/www/query_backends.tmpl
index fedae37..0ff32cd 100644
--- a/www/query_backends.tmpl
+++ b/www/query_backends.tmpl
@@ -60,7 +60,7 @@ var refresh = function () {
$(document).ready(function() {
table = $('#backends').DataTable({
- ajax: { url: "/query_backends?query_id={{query_id}}&json",
+ ajax: { url: make_url("/query_backends?query_id={{query_id}}&json"),
dataSrc: "backend_states",
},
"columns": [ {data: 'host'},
diff --git a/www/query_finstances.tmpl b/www/query_finstances.tmpl
index 5d8ba33..806235d 100644
--- a/www/query_finstances.tmpl
+++ b/www/query_finstances.tmpl
@@ -90,7 +90,7 @@ function unpackJson(json) {
$(document).ready(function() {
table = $('#finstances').DataTable({
- ajax: { url: "/query_finstances?query_id={{query_id}}&json",
+ ajax: { url: make_url("/query_finstances?query_id={{query_id}}&json"),
dataSrc: unpackJson,
},
"columns": [ {data: 'host'},
[impala] 03/03: IMPALA-8933: Enforce ranger deny policy
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit b37dd05e8f35bba5d9126ef79b35c1831f966f1b
Author: Kurt Deschler <kd...@cloudera.com>
AuthorDate: Mon Sep 9 19:32:18 2019 -0500
IMPALA-8933: Enforce ranger deny policy
This patch fixes a case where access to a given column is allowed at the
table level by a ranger policy and denied at the column level by a
second ranger policy. The code previously skipped evaluating column
level policies when a table level policy allowed access but that
optimization can only be applied when the column level policy does not
deny access.
Testing:
- Manually tested with table level allow and column level deny policies
in ranger
- Ran ranger-specific authorization funcional and unit tests
Steps to Repro:
Connect impala-shell as admin:
CREATE table(c1 int, c2 int);
INSERT INTO T1 VALUES(1,1);
In Ranger:
Add policies:
1) Name t1allow, Database *, Table t1,
Allow conditions user: <unix login>, Permissions: select
2) Name t1deny, Database *, Table t1,
Deny conditions user: <unix login>, Permissions: select
Connect impala-shell as <unix login>:
SELECT c1 from t1; -- Not allowed
SELECT c2 from t1; -- Allowed
Change-Id: Ic60786cd81080feeb0bfcd92aa2be646ee8cb7da
Reviewed-on: http://gerrit.cloudera.org:8080/14203
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
.../org/apache/impala/authorization/BaseAuthorizationChecker.java | 7 ++++++-
.../org/apache/impala/authorization/ranger/RangerAuditLogTest.java | 4 ++--
2 files changed, 8 insertions(+), 3 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/authorization/BaseAuthorizationChecker.java b/fe/src/main/java/org/apache/impala/authorization/BaseAuthorizationChecker.java
index 15cbb9a..9e171d9 100644
--- a/fe/src/main/java/org/apache/impala/authorization/BaseAuthorizationChecker.java
+++ b/fe/src/main/java/org/apache/impala/authorization/BaseAuthorizationChecker.java
@@ -231,7 +231,12 @@ public abstract class BaseAuthorizationChecker implements AuthorizationChecker {
} else {
Preconditions.checkState(
request.getAuthorizable().getType() == Authorizable.Type.COLUMN);
- if (hasTableSelectPriv) continue;
+ // In order to support deny policies on columns
+ if (hasTableSelectPriv &&
+ request.getPrivilege() != Privilege.SELECT &&
+ request.getPrivilege() != Privilege.INSERT) {
+ continue;
+ }
if (hasAccess(authzCtx, analyzer.getUser(), request)) {
hasColumnSelectPriv = true;
continue;
diff --git a/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java b/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java
index 1f4a6a4..48cbd70 100644
--- a/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java
+++ b/fe/src/test/java/org/apache/impala/authorization/ranger/RangerAuditLogTest.java
@@ -102,8 +102,8 @@ public class RangerAuditLogTest extends AuthorizationTestBase {
onUri("hdfs://localhost:20500/test-warehouse/new_table", TPrivilegeLevel.ALL));
authzOk(events -> {
- // Only the table event.
- assertEquals(1, events.size());
+ // Table event and 2 column events
+ assertEquals(3, events.size());
assertEventEquals("@table", "select", "functional/alltypes", 1, events.get(0));
assertEquals("select id, string_col from functional.alltypes",
events.get(0).getRequestData());
[impala] 02/03: IMPALA-8936: Improve queuing reason for unhealthy
executor groups
Posted by cs...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
csringhofer pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 576f205bffadd0a18cbc36167e6f191279fbc911
Author: Lars Volker <lv...@cloudera.com>
AuthorDate: Tue Sep 10 20:13:12 2019 -0700
IMPALA-8936: Improve queuing reason for unhealthy executor groups
In some situations, users might actually expect not having a healthy
executor group around, e.g. when they're starting one and it takes a
while to come online. This change makes the queuing reason more generic
and drops the "unhealthy" concept from it to reduce confusion.
Change-Id: Idceab7fb56335bab9d787b0f351a41e6efd7dd59
Reviewed-on: http://gerrit.cloudera.org:8080/14210
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/scheduling/admission-controller.cc | 5 +++--
tests/custom_cluster/test_coordinators.py | 4 ++--
tests/custom_cluster/test_executor_groups.py | 11 +++++------
3 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/be/src/scheduling/admission-controller.cc b/be/src/scheduling/admission-controller.cc
index ff0ea4c..164fed2 100644
--- a/be/src/scheduling/admission-controller.cc
+++ b/be/src/scheduling/admission-controller.cc
@@ -214,7 +214,8 @@ const string REASON_THREAD_RESERVATION_AGG_LIMIT_EXCEEDED =
// $0 is the error message returned by the scheduler.
const string REASON_SCHEDULER_ERROR = "Error during scheduling: $0";
const string REASON_LOCAL_BACKEND_NOT_STARTED = "Local backend has not started up yet.";
-const string REASON_NO_EXECUTOR_GROUPS = "No healthy executor groups found for pool $0.";
+const string REASON_NO_EXECUTOR_GROUPS = "Waiting for executors to start. Only DDL "
+ "queries can currently run.";
// Queue decision details
// $0 = num running queries, $1 = num queries limit, $2 = num queries limit explanation,
@@ -1193,7 +1194,7 @@ Status AdmissionController::ComputeGroupSchedules(
membership_snapshot->executor_groups, pool_name, &executor_groups);
if (executor_groups.empty()) {
- queue_node->not_admitted_reason = Substitute(REASON_NO_EXECUTOR_GROUPS, pool_name);
+ queue_node->not_admitted_reason = REASON_NO_EXECUTOR_GROUPS;
LOG(WARNING) << queue_node->not_admitted_reason;
return Status::OK();
}
diff --git a/tests/custom_cluster/test_coordinators.py b/tests/custom_cluster/test_coordinators.py
index c6f92f9..997e7ed 100644
--- a/tests/custom_cluster/test_coordinators.py
+++ b/tests/custom_cluster/test_coordinators.py
@@ -295,6 +295,6 @@ class TestCoordinators(CustomClusterTestSuite):
present."""
result = self.execute_query_expect_failure(self.client, "select 2")
expected_error = "Query aborted:Admission for query exceeded timeout 60000ms in " \
- "pool default-pool. Queued reason: No healthy executor groups " \
- "found for pool default-pool."
+ "pool default-pool. Queued reason: Waiting for executors to " \
+ "start. Only DDL queries can currently run."
assert expected_error in str(result)
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index 32e0087..1e8ba39 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -109,7 +109,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
client = self.client
handle = client.execute_async(QUERY)
profile = client.get_runtime_profile(handle)
- assert "No healthy executor groups found for pool" in profile
+ assert "Waiting for executors to start" in profile
assert self.coordinator.service.get_metric_value(
"cluster-membership.executor-groups.total-healthy") == 0
self._add_executor_group("group1", 2)
@@ -138,7 +138,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
# Run query and observe timeout
handle = client.execute_async(QUERY)
profile = client.get_runtime_profile(handle)
- assert "No healthy executor groups found for pool" in profile, profile
+ assert "Waiting for executors to start" in profile, profile
# Restart executor
executor.start()
# Query should now finish
@@ -286,8 +286,7 @@ class TestExecutorGroups(CustomClusterTestSuite):
client = self.client
handle = client.execute_async(QUERY)
profile = client.get_runtime_profile(handle)
- assert "Initial admission queue reason: No healthy executor groups found for pool" \
- in profile
+ assert "Initial admission queue reason: Waiting for executors to start" in profile
initial_state = client.get_state(handle)
# Start another executor and observe that the query stays queued
self._add_executor_group("group1", 3, num_executors=1)
@@ -324,8 +323,8 @@ class TestExecutorGroups(CustomClusterTestSuite):
# Run query to make sure it times out
result = self.execute_query_expect_failure(self.client, QUERY)
expected_error = "Query aborted:Admission for query exceeded timeout 2000ms in " \
- "pool default-pool. Queued reason: No healthy executor groups " \
- "found for pool default-pool."
+ "pool default-pool. Queued reason: Waiting for executors to " \
+ "start. Only DDL queries can currently run."
assert expected_error in str(result)
assert self.coordinator.service.get_metric_value(
"cluster-membership.executor-groups.total-healthy") == 0