You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2017/08/09 17:48:49 UTC
[1/2] incubator-impala git commit: IMPALA-5772: failure in
test_scratch_disk.TestScratchDir didn't occur
Repository: incubator-impala
Updated Branches:
refs/heads/master 4456ead84 -> d30f5f4e3
IMPALA-5772: failure in test_scratch_disk.TestScratchDir didn't occur
The buffer pool changes reduced the memory requirement for the sort,
which seems to have been enough to get the query to execute without
spilling on S3. Reduce the limit in the test to force it to spill.
Testing:
Ran in a loop locally for an hour. Ran custom cluster tests on S3.
Change-Id: If65fee3e6a4b759d0d18e30a1c30bd48db0f2a54
Reviewed-on: http://gerrit.cloudera.org:8080/7615
Reviewed-by: Tim Armstrong <ta...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/7007cd85
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/7007cd85
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/7007cd85
Branch: refs/heads/master
Commit: 7007cd85feaf8cc6631551d889c0d99c6895ca16
Parents: 4456ead
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Mon Aug 7 10:48:23 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Aug 9 00:04:25 2017 +0000
----------------------------------------------------------------------
tests/custom_cluster/test_scratch_disk.py | 7 +++----
1 file changed, 3 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/7007cd85/tests/custom_cluster/test_scratch_disk.py
----------------------------------------------------------------------
diff --git a/tests/custom_cluster/test_scratch_disk.py b/tests/custom_cluster/test_scratch_disk.py
index 579ca1e..bd3c7e4 100644
--- a/tests/custom_cluster/test_scratch_disk.py
+++ b/tests/custom_cluster/test_scratch_disk.py
@@ -37,10 +37,9 @@ class TestScratchDir(CustomClusterTestSuite):
in_mem_query = """
select o_orderdate, o_custkey, o_comment from tpch.orders
"""
- # Block manager memory limit that is low enough to force Impala to spill to disk when
- # executing spill_query and high enough that we can execute in_mem_query without
- # spilling.
- buffer_pool_limit = "64m"
+ # Buffer pool limit that is low enough to force Impala to spill to disk when executing
+ # spill_query.
+ buffer_pool_limit = "32m"
def count_nonempty_dirs(self, dirs):
count = 0
[2/2] incubator-impala git commit: IMPALA-5652: deprecate unlimited
process mem_limit
Posted by ta...@apache.org.
IMPALA-5652: deprecate unlimited process mem_limit
Testing:
Started cluster locally with:
start-impala-cluster.py --impalad_args=--mem_limit=-1
Checked that the warning showed up in the logs:
W0808 17:01:36.528360 6251 exec-env.cc:237] Configured with unlimited process memory limit (--mem_limit='-1'). Starting in the next Impala release, a process memory limit must always be specified. See IMPALA-5653.
Change-Id: I42509a6a9ed49e3752931c82310dfda841dd97df
Reviewed-on: http://gerrit.cloudera.org:8080/7622
Reviewed-by: Matthew Jacobs <mj...@cloudera.com>
Tested-by: Impala Public Jenkins
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/d30f5f4e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/d30f5f4e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/d30f5f4e
Branch: refs/heads/master
Commit: d30f5f4e302845132e346263aaf3dfb8c0bb5e1e
Parents: 7007cd8
Author: Tim Armstrong <ta...@cloudera.com>
Authored: Tue Aug 8 16:58:03 2017 -0700
Committer: Impala Public Jenkins <im...@gerrit.cloudera.org>
Committed: Wed Aug 9 05:48:34 2017 +0000
----------------------------------------------------------------------
be/src/runtime/exec-env.cc | 9 +++++++--
1 file changed, 7 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/d30f5f4e/be/src/runtime/exec-env.cc
----------------------------------------------------------------------
diff --git a/be/src/runtime/exec-env.cc b/be/src/runtime/exec-env.cc
index 8239a68..3427c30 100644
--- a/be/src/runtime/exec-env.cc
+++ b/be/src/runtime/exec-env.cc
@@ -206,7 +206,7 @@ Status ExecEnv::StartServices() {
// memory limit either based on the available physical memory, or if overcommitting
// is turned off, we use the memory commit limit from /proc/meminfo (see
// IMPALA-1690).
- // --mem_limit="" means no memory limit. TODO: IMPALA-5652: deprecate this mode
+ // --mem_limit="" means no memory limit. TODO: IMPALA-5653: remove this mode
int64_t bytes_limit = 0;
bool is_percent;
int64_t system_mem;
@@ -231,8 +231,13 @@ Status ExecEnv::StartServices() {
system_mem = MemInfo::physical_mem();
bytes_limit = ParseUtil::ParseMemSpec(FLAGS_mem_limit, &is_percent, system_mem);
}
- // ParseMemSpec returns 0 to mean unlimited. TODO: IMPALA-5652: deprecate this mode.
+ // ParseMemSpec returns 0 to mean unlimited.
bool no_process_mem_limit = bytes_limit == 0;
+ if (no_process_mem_limit) {
+ LOG(WARNING) << "Configured with unlimited process memory limit (--mem_limit='"
+ << FLAGS_mem_limit << "'). Starting in the next Impala release, "
+ << "a process memory limit must always be specified. See IMPALA-5653.";
+ }
if (bytes_limit < 0) {
return Status("Failed to parse mem limit from '" + FLAGS_mem_limit + "'.");
}