You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by ta...@apache.org on 2020/05/14 22:21:06 UTC
[impala] branch master updated (506a303 -> ea4c7e4)
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git.
from 506a303 IMPALA-6984: coordinator cancels backends on EOS
new 88c2f9a Bump test-with-docker test concurrency for large instances
new ea4c7e4 IMPALA-9712: fix mem consumption of operators above selective scan
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
be/src/runtime/row-batch.h | 2 +-
docker/test-with-docker.py | 7 +++++--
tests/query_test/test_mem_usage_scaling.py | 27 +++++++++++++++++++++++++++
3 files changed, 33 insertions(+), 3 deletions(-)
[impala] 01/02: Bump test-with-docker test concurrency for large
instances
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 88c2f9a5268a71aced3a2788dc9c139182f74f56
Author: Laszlo Gaal <la...@cloudera.com>
AuthorDate: Wed Apr 22 13:02:55 2020 +0200
Bump test-with-docker test concurrency for large instances
The concurrency limits (i.e. how many concurrent Docker containers are
running test shards at the same time) were conservative at the high end:
the largest memory configuration they considered was under 100 GBs.
Bump these limits for the usual m5.12xlarge test worker that has 192 GBs
of RAM, of which about 186 GBs are available.
Also, swap the order of FE and BE tests, as FE tests have now grown
pretty long with the long delay in AuthorizationStmtTest.
Test: ran test-with-docker.py with all default parameters. Verified that
default concurrency was 6 on an m5.12xlarge and core-mode tests passed
in an Ubuntu 16.04 container.
Change-Id: I5c03a78ee65d09212d9bfa007e87fd069cdaabb6
Reviewed-on: http://gerrit.cloudera.org:8080/15834
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Laszlo Gaal <la...@cloudera.com>
---
docker/test-with-docker.py | 7 +++++--
1 file changed, 5 insertions(+), 2 deletions(-)
diff --git a/docker/test-with-docker.py b/docker/test-with-docker.py
index 8bb35b0..9dc867c 100755
--- a/docker/test-with-docker.py
+++ b/docker/test-with-docker.py
@@ -257,7 +257,10 @@ def _compute_defaults():
parallel_test_concurrency = min(cpus, 8)
memlimit_gb = 8
- if total_memory_gb >= 95:
+ if total_memory_gb >= 140:
+ suite_concurrency = 6
+ memlimit_gb = 11
+ elif total_memory_gb >= 95:
suite_concurrency = 4
memlimit_gb = 11
parallel_test_concurrency = min(cpus, 12)
@@ -356,8 +359,8 @@ DEFAULT_SUITES = [
ee_test_serial,
ee_test_parallel,
cluster_test,
- Suite("BE_TEST"),
Suite("FE_TEST"),
+ Suite("BE_TEST"),
Suite("JDBC_TEST")
]
[impala] 02/02: IMPALA-9712: fix mem consumption of operators above
selective scan
Posted by ta...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tarmstrong pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit ea4c7e4add8d97822323b4854b5d15f4a6a46878
Author: Tim Armstrong <ta...@cloudera.com>
AuthorDate: Mon May 4 12:19:52 2020 -0700
IMPALA-9712: fix mem consumption of operators above selective scan
This change is motivated by excessive memory consumption of
TPC-H Q19 which has a hash join and non-grouping aggregate
above a selective scan.
This change fixes RowBatch::AtCapacity() to account for the actual
memory consumed by the RowBatch. It used total_allocated_bytes(),
which does *not* account for unused space in the MemPool chunks.
Instead it now uses total_reserved_bytes(), which includes the
whole chunks. This reduced memory consumption of the agg from
60+MB to ~16MB.
Testing:
Ran TPC-H Q19 on parquet with mt_dop=8. Aggregation mem usage was
reduced from 60+MB to ~16MB.
Added a targeted regression test that ran out of memory before this
fix.
Ran exhaustive tests.
Performance:
No significant change on TPC-H single node run with scale factor 30.
I also ran TPC-H nested scale factor 1 and there was no measureable
change, but generation of the report failed for some reason.
+----------+-----------------------+---------+------------+------------+----------------+
| Workload | File Format | Avg (s) | Delta(Avg) | GeoMean(s) | Delta(GeoMean) |
+----------+-----------------------+---------+------------+------------+----------------+
| TPCH(30) | parquet / none / none | 6.18 | -1.31% | 4.54 | -1.03% |
+----------+-----------------------+---------+------------+------------+----------------+
+----------+----------+-----------------------+--------+-------------+------------+------------+----------------+-------+----------------+---------+-------+
| Workload | Query | File Format | Avg(s) | Base Avg(s) | Delta(Avg) | StdDev(%) | Base StdDev(%) | Iters | Median Diff(%) | MW Zval | Tval |
+----------+----------+-----------------------+--------+-------------+------------+------------+----------------+-------+----------------+---------+-------+
| TPCH(30) | TPCH-Q10 | parquet / none / none | 8.08 | 7.98 | +1.19% | * 14.00% * | * 10.55% * | 20 | +0.63% | 0.60 | 0.30 |
| TPCH(30) | TPCH-Q13 | parquet / none / none | 10.03 | 9.98 | +0.44% | 1.22% | 0.92% | 20 | +0.48% | 1.07 | 1.28 |
| TPCH(30) | TPCH-Q12 | parquet / none / none | 3.20 | 3.19 | +0.34% | 2.02% | 2.68% | 20 | +0.08% | 0.48 | 0.45 |
| TPCH(30) | TPCH-Q21 | parquet / none / none | 22.60 | 22.54 | +0.24% | 2.85% | 2.80% | 20 | +0.17% | 0.22 | 0.27 |
| TPCH(30) | TPCH-Q9 | parquet / none / none | 13.80 | 13.77 | +0.17% | 1.99% | 1.70% | 20 | +0.06% | 0.10 | 0.30 |
| TPCH(30) | TPCH-Q7 | parquet / none / none | 4.52 | 4.52 | -0.01% | 1.68% | 1.71% | 20 | +0.03% | 0.07 | -0.01 |
| TPCH(30) | TPCH-Q8 | parquet / none / none | 5.40 | 5.43 | -0.52% | 1.60% | 1.92% | 20 | -0.23% | -0.98 | -0.94 |
| TPCH(30) | TPCH-Q16 | parquet / none / none | 1.82 | 1.83 | -0.63% | 2.98% | 2.62% | 20 | -0.17% | -1.07 | -0.71 |
| TPCH(30) | TPCH-Q15 | parquet / none / none | 3.80 | 3.82 | -0.46% | 1.31% | 1.21% | 20 | -0.41% | -1.30 | -1.17 |
| TPCH(30) | TPCH-Q4 | parquet / none / none | 2.72 | 2.74 | -0.68% | 3.06% | 2.36% | 20 | -0.23% | -0.92 | -0.79 |
| TPCH(30) | TPCH-Q6 | parquet / none / none | 1.63 | 1.64 | -0.98% | 3.28% | 2.66% | 20 | -0.23% | -1.47 | -1.04 |
| TPCH(30) | TPCH-Q22 | parquet / none / none | 1.99 | 2.01 | -0.98% | 2.59% | 2.99% | 20 | -0.68% | -1.12 | -1.12 |
| TPCH(30) | TPCH-Q1 | parquet / none / none | 5.22 | 5.27 | -0.96% | 1.93% | 2.25% | 20 | -0.93% | -1.39 | -1.45 |
| TPCH(30) | TPCH-Q14 | parquet / none / none | 3.61 | 3.64 | -0.73% | 2.51% | 2.40% | 20 | -1.26% | -1.18 | -0.95 |
| TPCH(30) | TPCH-Q5 | parquet / none / none | 4.52 | 4.58 | -1.23% | 1.41% | 1.39% | 20 | -1.15% | -2.99 | -2.79 |
| TPCH(30) | TPCH-Q19 | parquet / none / none | 4.05 | 4.10 | -1.19% | 2.58% | 2.48% | 20 | -1.27% | -1.77 | -1.49 |
| TPCH(30) | TPCH-Q3 | parquet / none / none | 3.96 | 4.01 | -1.32% | 1.82% | 1.82% | 20 | -1.31% | -2.09 | -2.30 |
| TPCH(30) | TPCH-Q20 | parquet / none / none | 2.93 | 2.98 | -1.47% | 2.16% | 2.60% | 20 | -1.63% | -1.88 | -1.95 |
| TPCH(30) | TPCH-Q2 | parquet / none / none | 2.78 | 2.83 | -1.70% | 2.29% | 2.19% | 20 | -1.77% | -1.97 | -2.41 |
| TPCH(30) | TPCH-Q11 | parquet / none / none | 1.09 | 1.12 | -2.64% | 2.99% | 3.81% | 20 | -3.71% | -1.77 | -2.46 |
| TPCH(30) | TPCH-Q17 | parquet / none / none | 15.07 | 15.73 | -4.20% | 3.24% | 2.26% | 20 | -5.00% | -3.84 | -4.89 |
| TPCH(30) | TPCH-Q18 | parquet / none / none | 13.23 | 14.13 | -6.38% | * 11.87% * | * 26.04% * | 20 | -3.19% | -2.41 | -1.01 |
+----------+----------+-----------------------+--------+-------------+------------+------------+----------------+-------+----------------+---------+-------+
Change-Id: I6debae562826621411bbcbb757978e227b395441
Reviewed-on: http://gerrit.cloudera.org:8080/15863
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
be/src/runtime/row-batch.h | 2 +-
tests/query_test/test_mem_usage_scaling.py | 27 +++++++++++++++++++++++++++
2 files changed, 28 insertions(+), 1 deletion(-)
diff --git a/be/src/runtime/row-batch.h b/be/src/runtime/row-batch.h
index 93cf2cd..007a5e3 100644
--- a/be/src/runtime/row-batch.h
+++ b/be/src/runtime/row-batch.h
@@ -200,7 +200,7 @@ class RowBatch {
// MarkFlushResources().
DCHECK((!needs_deep_copy_ && flush_mode_ == FlushMode::NO_FLUSH_RESOURCES)
|| num_rows_ == capacity_);
- int64_t mem_usage = attached_buffer_bytes_ + tuple_data_pool_.total_allocated_bytes();
+ int64_t mem_usage = attached_buffer_bytes_ + tuple_data_pool_.total_reserved_bytes();
return num_rows_ == capacity_ || mem_usage >= AT_CAPACITY_MEM_USAGE;
}
diff --git a/tests/query_test/test_mem_usage_scaling.py b/tests/query_test/test_mem_usage_scaling.py
index 7e36149..7fcd263 100644
--- a/tests/query_test/test_mem_usage_scaling.py
+++ b/tests/query_test/test_mem_usage_scaling.py
@@ -228,6 +228,7 @@ class TestTpchMemLimitError(TestLowMemoryLimits):
verifier = MetricVerifier(impalad.service)
verifier.wait_for_metric("impala-server.num-fragments-in-flight", 0)
+
@SkipIfNotHdfsMinicluster.tuned_for_minicluster
class TestTpchPrimitivesMemLimitError(TestLowMemoryLimits):
"""
@@ -368,6 +369,32 @@ class TestScanMemLimit(ImpalaTestSuite):
@SkipIfNotHdfsMinicluster.tuned_for_minicluster
+class TestHashJoinMemLimit(ImpalaTestSuite):
+ """Targeted test for scan memory limits."""
+
+ @classmethod
+ def get_workload(self):
+ return 'tpch'
+
+ @classmethod
+ def add_test_dimensions(cls):
+ super(TestHashJoinMemLimit, cls).add_test_dimensions()
+ cls.ImpalaTestMatrix.add_dimension(create_single_exec_option_dimension())
+ cls.ImpalaTestMatrix.add_dimension(create_parquet_dimension(cls.get_workload()))
+
+ def test_low_mem_limit_selective_scan_hash_join(self, vector):
+ """Selective scan with hash join and aggregate above it. Regression test for
+ IMPALA-9712 - before the fix this ran out of memory."""
+ OPTS = {'mem_limit': "60MB", 'mt_dop': 1}
+ self.change_database(self.client, vector.get_value('table_format'))
+ result = self.execute_query_expect_success(self.client,
+ """select sum(l_extendedprice * (1 - l_discount)) as revenue
+ from lineitem join part on p_partkey = l_partkey
+ where l_comment like 'ab%'""", query_options=OPTS)
+ assert result.data[0] == '440443181.0505'
+
+
+@SkipIfNotHdfsMinicluster.tuned_for_minicluster
class TestExchangeMemUsage(ImpalaTestSuite):
"""Targeted test for exchange memory limits."""