You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by dh...@apache.org on 2018/06/21 15:34:31 UTC
[1/3] impala git commit: IMPALA-7187: Fix test_group_impersonation
running inside Docker
Repository: impala
Updated Branches:
refs/heads/master da8e1c2f6 -> 94ffcc64a
IMPALA-7187: Fix test_group_impersonation running inside Docker
test_group_impersonation fails when running with test-with-docker.py
because os.getgroups() returns empty in Docker for the authorized
proxy groups which results in denying all access. The patch is to use
a group name from a current GID.
Testing:
- Ran all test_authorization with test-with-docker.py.
Change-Id: Icb6089273fd9bc3763cb45f0591610b04a6b232e
Reviewed-on: http://gerrit.cloudera.org:8080/10772
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/143e9c47
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/143e9c47
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/143e9c47
Branch: refs/heads/master
Commit: 143e9c47a562102cdb450293a1ea68a0a90bece6
Parents: da8e1c2
Author: Fredy Wijaya <fw...@cloudera.com>
Authored: Wed Jun 20 00:00:13 2018 +0000
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Jun 21 01:30:41 2018 +0000
----------------------------------------------------------------------
tests/authorization/test_authorization.py | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/143e9c47/tests/authorization/test_authorization.py
----------------------------------------------------------------------
diff --git a/tests/authorization/test_authorization.py b/tests/authorization/test_authorization.py
index 5b4bccd..3e6c56e 100644
--- a/tests/authorization/test_authorization.py
+++ b/tests/authorization/test_authorization.py
@@ -36,9 +36,6 @@ from tests.util.filesystem_utils import WAREHOUSE
AUTH_POLICY_FILE = "%s/authz-policy.ini" % WAREHOUSE
-def get_groups():
- return [grp.getgrgid(group).gr_name for group in os.getgroups()]
-
class TestAuthorization(CustomClusterTestSuite):
AUDIT_LOG_DIR = tempfile.mkdtemp(dir=os.getenv('LOG_DIR'))
@@ -192,7 +189,7 @@ class TestAuthorization(CustomClusterTestSuite):
--authorized_proxy_group_config=foo=bar;hue=%s\
--abort_on_failed_audit_event=false\
--audit_event_log_dir=%s" % (AUTH_POLICY_FILE,
- ','.join(get_groups()),
+ grp.getgrgid(os.getgid()).gr_name,
AUDIT_LOG_DIR))
def test_group_impersonation(self):
"""End-to-end group impersonation + authorization test"""
[3/3] impala git commit: IMPALA-5202: Disallow PREPARE:WAIT debug
action
Posted by dh...@apache.org.
IMPALA-5202: Disallow PREPARE:WAIT debug action
In order to simplify FIS startup, we don't allow cancellation until all
FIS have finished Prepare(), so we shouldn't allow PREPARE:WAIT since
there will be no way to cancel out of the loop. Make this explicit.
Change-Id: I1caa4f8e6ce7f32a8a3722648e08e24f34dba35d
Reviewed-on: http://gerrit.cloudera.org:8080/10776
Reviewed-by: Dan Hecht <dh...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/94ffcc64
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/94ffcc64
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/94ffcc64
Branch: refs/heads/master
Commit: 94ffcc64a997022ed582c2b428aac1f06dd2da77
Parents: a9b0196
Author: Dan Hecht <dh...@cloudera.com>
Authored: Wed Jun 20 14:49:57 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Jun 21 07:18:12 2018 +0000
----------------------------------------------------------------------
be/src/exec/exec-node.cc | 2 ++
be/src/runtime/debug-options.cc | 11 ++++++++---
tests/failure/test_failpoints.py | 8 +++++---
3 files changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/94ffcc64/be/src/exec/exec-node.cc
----------------------------------------------------------------------
diff --git a/be/src/exec/exec-node.cc b/be/src/exec/exec-node.cc
index 8b4cec1..eeefaed 100644
--- a/be/src/exec/exec-node.cc
+++ b/be/src/exec/exec-node.cc
@@ -405,6 +405,8 @@ Status ExecNode::ExecDebugActionImpl(TExecNodePhase::type phase, RuntimeState* s
if (debug_options_.action == TDebugAction::FAIL) {
return Status(TErrorCode::INTERNAL_ERROR, "Debug Action: FAIL");
} else if (debug_options_.action == TDebugAction::WAIT) {
+ // See DebugOptions::DebugOptions().
+ DCHECK(phase != TExecNodePhase::PREPARE && phase != TExecNodePhase::CLOSE);
while (!state->is_cancelled()) {
sleep(1);
}
http://git-wip-us.apache.org/repos/asf/impala/blob/94ffcc64/be/src/runtime/debug-options.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/debug-options.cc b/be/src/runtime/debug-options.cc
index 6e5062d..b8b6b35 100644
--- a/be/src/runtime/debug-options.cc
+++ b/be/src/runtime/debug-options.cc
@@ -65,9 +65,14 @@ DebugOptions::DebugOptions(const TQueryOptions& query_options)
phase_ = TExecNodePhase::INVALID;
return;
}
- if (phase_ == TExecNodePhase::CLOSE && action_ == TDebugAction::WAIT) {
- LOG(WARNING) << "Do not use CLOSE:WAIT debug actions because nodes cannot be "
- "cancelled in Close()";
+ if ((phase_ == TExecNodePhase::PREPARE || phase_ == TExecNodePhase::CLOSE) &&
+ action_ == TDebugAction::WAIT) {
+ // Cancellation is not allowed before PREPARE phase completes (to keep FIS
+ // startup simpler - it will be checked immediately after when execution begins).
+ // Close() must always run to completion to free resources, so cancellation does
+ // not make sense for the CLOSE phase and is not checked. So we disallow these.
+ LOG(WARNING) << Substitute("Do not use $0:$1 debug actions because nodes cannot "
+ "be cancelled in that phase.", *phase_str, *action_str);
phase_ = TExecNodePhase::INVALID;
return;
}
http://git-wip-us.apache.org/repos/asf/impala/blob/94ffcc64/tests/failure/test_failpoints.py
----------------------------------------------------------------------
diff --git a/tests/failure/test_failpoints.py b/tests/failure/test_failpoints.py
index 9f54257..7366900 100644
--- a/tests/failure/test_failpoints.py
+++ b/tests/failure/test_failpoints.py
@@ -75,6 +75,11 @@ class TestFailpoints(ImpalaTestSuite):
cls.ImpalaTestMatrix.add_dimension(
create_exec_option_dimension([0], [False], [0]))
+ # Don't create PREPARE:WAIT debug actions because cancellation is not checked until
+ # after the prepare phase once execution is started.
+ cls.ImpalaTestMatrix.add_constraint(
+ lambda v: not (v.get_value('action') == 'CANCEL'
+ and v.get_value('location') == 'PREPARE'))
# Don't create CLOSE:WAIT debug actions to avoid leaking plan fragments (there's no
# way to cancel a plan fragment once Close() has been called)
cls.ImpalaTestMatrix.add_constraint(
@@ -90,9 +95,6 @@ class TestFailpoints(ImpalaTestSuite):
location = vector.get_value('location')
vector.get_value('exec_option')['mt_dop'] = vector.get_value('mt_dop')
- if action == "CANCEL" and location == "PREPARE":
- pytest.xfail(reason="IMPALA-5202 leads to a hang.")
-
try:
plan_node_ids = self.__parse_plan_nodes_from_explain(query, vector)
except ImpalaBeeswaxException as e:
[2/3] impala git commit: [DOCS] Added another scenario when REFRESH
is required
Posted by dh...@apache.org.
[DOCS] Added another scenario when REFRESH is required
Change-Id: Ifff8f6b6f834402a158312d57076a84ad1eeab45
Reviewed-on: http://gerrit.cloudera.org:8080/10773
Reviewed-by: Vuk Ercegovac <ve...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Project: http://git-wip-us.apache.org/repos/asf/impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/a9b01963
Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/a9b01963
Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/a9b01963
Branch: refs/heads/master
Commit: a9b01963b682aa0122d416d642141973ef4a5c8e
Parents: 143e9c4
Author: Alex Rodoni <ar...@cloudera.com>
Authored: Wed Jun 20 10:54:32 2018 -0700
Committer: Impala Public Jenkins <im...@cloudera.com>
Committed: Thu Jun 21 01:36:18 2018 +0000
----------------------------------------------------------------------
docs/topics/impala_refresh.xml | 11 ++++-------
1 file changed, 4 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/impala/blob/a9b01963/docs/topics/impala_refresh.xml
----------------------------------------------------------------------
diff --git a/docs/topics/impala_refresh.xml b/docs/topics/impala_refresh.xml
index cdda8f0..2ed6971 100644
--- a/docs/topics/impala_refresh.xml
+++ b/docs/topics/impala_refresh.xml
@@ -86,13 +86,10 @@ under the License.
</p>
<ul>
- <li>
- Deleting or adding files.
- <p>
- For example, after loading new data files into the HDFS data directory for the table,
- including inserting data from Hive via <codeph>INSERT</codeph> or <codeph>LOAD
- DATA</codeph>.
- </p>
+ <li> Deleting, adding, or modifying files. <p> For example, after loading
+ new data files into the HDFS data directory for the table, appending
+ to an existing HDFS file, inserting data from Hive via
+ <codeph>INSERT</codeph> or <codeph>LOAD DATA</codeph>. </p>
</li>
<li>