You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@impala.apache.org by db...@apache.org on 2023/07/14 09:20:51 UTC
[impala] branch master updated (c0feea2c9 -> c2ef633c6)
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
from c0feea2c9 IMPALA-12260: Crash if '--asm_module_dir' is set
new c9aed342f IMPALA-12281: Disallow unsetting REQUEST_POOL if it is set by client
new 57370fb06 IMPALA-12188: Avoid unnecessary output from sourcing bin/impala-config.sh
new c2ef633c6 IMPALA-12282: Refine correlation factor in AggregationNode
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
bin/bootstrap_development.sh | 2 +-
bin/create_testdata.sh | 2 +-
bin/gen-cscope.sh | 2 +-
bin/init-impala-python.sh | 2 +-
bin/jenkins/all-tests.sh | 2 +-
bin/jenkins/build-all-flag-combinations.sh | 2 +-
bin/jenkins/dockerized-impala-run-tests.sh | 2 +-
bin/run-cpplint.sh | 2 +-
common/thrift/ImpalaService.thrift | 4 +-
.../org/apache/impala/planner/AggregationNode.java | 11 ++++--
.../java/org/apache/impala/service/Frontend.java | 22 +++++++----
testdata/bin/run-ranger-server.sh | 2 +-
.../queries/PlannerTest/tpcds/tpcds-q22.test | 32 +++++++--------
.../queries/PlannerTest/tpcds/tpcds-q67.test | 18 ++++-----
tests/custom_cluster/test_executor_groups.py | 45 +++++++++++++++++-----
15 files changed, 94 insertions(+), 56 deletions(-)
[impala] 02/03: IMPALA-12188: Avoid unnecessary output from sourcing bin/impala-config.sh
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit 57370fb06c15122e201aee44b455220caf6d96f2
Author: Joe McDonnell <jo...@cloudera.com>
AuthorDate: Mon Jun 5 19:13:05 2023 -0700
IMPALA-12188: Avoid unnecessary output from sourcing bin/impala-config.sh
Many scripts source bin/impala-config.sh to get necessary
environment variables. The print statements in bin/impala-config.sh
for those scripts are not interesting and make the build logs
noisier.
This changes a variety of build scripts / utility scripts to
silence the output of sourcing bin/impala-config.sh. This continues
to print the output for invocations of buildall.sh.
Testing:
- Ran a build and looked at the output
Change-Id: Ib4e39f50c7efb8c42a6d3597be0e18c4c79457c5
Reviewed-on: http://gerrit.cloudera.org:8080/20098
Tested-by: Impala Public Jenkins <im...@cloudera.com>
Reviewed-by: Yifan Zhang <ch...@163.com>
Reviewed-by: Joe McDonnell <jo...@cloudera.com>
---
bin/bootstrap_development.sh | 2 +-
bin/create_testdata.sh | 2 +-
bin/gen-cscope.sh | 2 +-
bin/init-impala-python.sh | 2 +-
bin/jenkins/all-tests.sh | 2 +-
bin/jenkins/build-all-flag-combinations.sh | 2 +-
bin/jenkins/dockerized-impala-run-tests.sh | 2 +-
bin/run-cpplint.sh | 2 +-
testdata/bin/run-ranger-server.sh | 2 +-
9 files changed, 9 insertions(+), 9 deletions(-)
diff --git a/bin/bootstrap_development.sh b/bin/bootstrap_development.sh
index 8ff6f4884..f3c552260 100755
--- a/bin/bootstrap_development.sh
+++ b/bin/bootstrap_development.sh
@@ -44,7 +44,7 @@ BINDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
source "${BINDIR}/bootstrap_system.sh"
export MAX_PYTEST_FAILURES=0
-source bin/impala-config.sh
+source bin/impala-config.sh > /dev/null 2>&1
export NUM_CONCURRENT_TESTS=$(nproc)
time -p ./buildall.sh -format -testdata -skiptests
diff --git a/bin/create_testdata.sh b/bin/create_testdata.sh
index 9c703621a..41a17d2ab 100755
--- a/bin/create_testdata.sh
+++ b/bin/create_testdata.sh
@@ -23,7 +23,7 @@ setup_report_build_error
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/impala-config.sh
+. "$bin"/impala-config.sh > /dev/null 2>&1
# location of the generated data
DATALOC=$IMPALA_HOME/testdata/target
diff --git a/bin/gen-cscope.sh b/bin/gen-cscope.sh
index d3b442358..920b8d51b 100755
--- a/bin/gen-cscope.sh
+++ b/bin/gen-cscope.sh
@@ -20,7 +20,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/impala-config.sh
+. "$bin"/impala-config.sh > /dev/null 2>&1
# Generate list of files for Cscope to index.
# -ignore_readdir_race: this scripts runs in parallel with other build steps, so races
diff --git a/bin/init-impala-python.sh b/bin/init-impala-python.sh
index 360d2df89..35c90b90c 100755
--- a/bin/init-impala-python.sh
+++ b/bin/init-impala-python.sh
@@ -25,7 +25,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/impala-config.sh
+. "$bin"/impala-config.sh > /dev/null 2>&1
function print_usage {
echo "init-impala-python.sh - Script called from CMake to init python venvs"
diff --git a/bin/jenkins/all-tests.sh b/bin/jenkins/all-tests.sh
index 7ebe294eb..41eef97ac 100644
--- a/bin/jenkins/all-tests.sh
+++ b/bin/jenkins/all-tests.sh
@@ -67,7 +67,7 @@ if ! bin/bootstrap_development.sh; then
RET_CODE=1
fi
-source bin/impala-config.sh
+source bin/impala-config.sh > /dev/null 2>&1
# Sanity check: bootstrap_development.sh should not have modified any of
# the Impala files. This is important for the continued functioning of
diff --git a/bin/jenkins/build-all-flag-combinations.sh b/bin/jenkins/build-all-flag-combinations.sh
index 0d503d4e5..d10f92f06 100755
--- a/bin/jenkins/build-all-flag-combinations.sh
+++ b/bin/jenkins/build-all-flag-combinations.sh
@@ -30,7 +30,7 @@ setup_report_build_error
export IMPALA_MAVEN_OPTIONS="-U"
-. bin/impala-config.sh
+. bin/impala-config.sh > /dev/null 2>&1
: ${GENERATE_M2_ARCHIVE:=false}
diff --git a/bin/jenkins/dockerized-impala-run-tests.sh b/bin/jenkins/dockerized-impala-run-tests.sh
index a5e3b165a..c2ed71a35 100755
--- a/bin/jenkins/dockerized-impala-run-tests.sh
+++ b/bin/jenkins/dockerized-impala-run-tests.sh
@@ -28,7 +28,7 @@ DOCKER_NETWORK="test-impala-cluster"
# Helper to source impala-config.sh, which may have unbound variables
source_impala_config() {
set +u
- . ./bin/impala-config.sh
+ . ./bin/impala-config.sh > /dev/null 2>&1
set -u
}
diff --git a/bin/run-cpplint.sh b/bin/run-cpplint.sh
index c97074aff..b389e7469 100755
--- a/bin/run-cpplint.sh
+++ b/bin/run-cpplint.sh
@@ -20,7 +20,7 @@
bin=`dirname "$0"`
bin=`cd "$bin"; pwd`
-. "$bin"/impala-config.sh
+. "$bin"/impala-config.sh > /dev/null 2>&1
files=`find $IMPALA_BE_DIR/src -regex '.*\(cc\|h\)' -printf '%p '`
cpplint.py $@ ${files}
diff --git a/testdata/bin/run-ranger-server.sh b/testdata/bin/run-ranger-server.sh
index 9a6e29676..6b03ad3dd 100755
--- a/testdata/bin/run-ranger-server.sh
+++ b/testdata/bin/run-ranger-server.sh
@@ -29,7 +29,7 @@ fi
# IMPALA-8815: don't allow additional potentially incompatible jars to get onto
# the ranger classpath. We should only need the test cluster configs on the classpath.
unset CLASSPATH
-. $IMPALA_HOME/bin/impala-config.sh
+. $IMPALA_HOME/bin/impala-config.sh > /dev/null 2>&1
# Required to start Ranger with Java 11
if [[ ! -d "${RANGER_HOME}"/ews/logs ]]; then
[impala] 03/03: IMPALA-12282: Refine correlation factor in AggregationNode
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c2ef633c60aff6ce3bf0fcffb6fc0763a7fdd3ca
Author: Riza Suminto <ri...@cloudera.com>
AuthorDate: Thu Jul 13 12:46:03 2023 -0700
IMPALA-12282: Refine correlation factor in AggregationNode
IMPALA-11842 implement crude correlation factor calculation that simply
include all grouping expression. This can be made more precise by
excluding literal expression such as string literal or NULL literal that
often comes up in ROLLUP query.
Testing:
- Pass TpcdsPlannerTest
Change-Id: I4ffa9e82b83e7c0042bd918ac132668a47505688
Reviewed-on: http://gerrit.cloudera.org:8080/20194
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Impala Public Jenkins <im...@cloudera.com>
---
common/thrift/ImpalaService.thrift | 4 +--
.../org/apache/impala/planner/AggregationNode.java | 11 ++++++--
.../queries/PlannerTest/tpcds/tpcds-q22.test | 32 +++++++++++-----------
.../queries/PlannerTest/tpcds/tpcds-q67.test | 18 ++++++------
4 files changed, 35 insertions(+), 30 deletions(-)
diff --git a/common/thrift/ImpalaService.thrift b/common/thrift/ImpalaService.thrift
index 726264079..0ab6be4cc 100644
--- a/common/thrift/ImpalaService.thrift
+++ b/common/thrift/ImpalaService.thrift
@@ -827,9 +827,9 @@ enum TImpalaQueryOptions {
// Correlation factor that will be used to calculate a lower memory estimation of
// aggregation node when the default memory estimation exceed
- // LARGE_AGG_MEM_THRESHOLD. Given N as number of grouping expressions,
+ // LARGE_AGG_MEM_THRESHOLD. Given N as number of non-literal grouping expressions,
// the final correlation factor is calculated as:
- // corrFactor = AGG_MEM_CORRELATION_FACTOR ^ N
+ // corrFactor = AGG_MEM_CORRELATION_FACTOR ^ max(0, N - 1)
// Valid values are in [0.0, 1.0]. Setting value 1.0 will result in an equal memory
// estimate as the default estimation (no change). Default to 0.5.
AGG_MEM_CORRELATION_FACTOR = 163
diff --git a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
index dae282169..91eba6686 100644
--- a/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
+++ b/fe/src/main/java/org/apache/impala/planner/AggregationNode.java
@@ -29,6 +29,7 @@ import org.apache.impala.analysis.CaseExpr;
import org.apache.impala.analysis.CaseWhenClause;
import org.apache.impala.analysis.Expr;
import org.apache.impala.analysis.FunctionCallExpr;
+import org.apache.impala.analysis.LiteralExpr;
import org.apache.impala.analysis.MultiAggregateInfo;
import org.apache.impala.analysis.MultiAggregateInfo.AggPhase;
import org.apache.impala.analysis.NumericLiteral;
@@ -676,10 +677,14 @@ public class AggregationNode extends PlanNode {
estimatePerInstanceDataBytes(lowPerInstanceCardinality, inputCardinality));
Preconditions.checkState(lowPerInstanceDataBytes <= perInstanceDataBytes);
- // Given N as number of grouping expressions,
- // corrFactor = AGG_MEM_CORRELATION_FACTOR ^ N
+ // Given N as number of non-literal grouping expressions,
+ // corrFactor = AGG_MEM_CORRELATION_FACTOR ^ max(0, N - 1)
+ long nonLiteralExprCount = aggInfo.getGroupingExprs()
+ .stream()
+ .filter(e -> !(e instanceof LiteralExpr))
+ .count();
double corrFactor = Math.pow(queryOptions.getAgg_mem_correlation_factor(),
- aggInfo.getGroupingExprs().size());
+ Math.max(0, nonLiteralExprCount - 1));
long resolvedPerInstanceDataBytes = lowPerInstanceDataBytes
+ Math.round(corrFactor * (perInstanceDataBytes - lowPerInstanceDataBytes));
if (LOG.isTraceEnabled() && perInstanceDataBytes > resolvedPerInstanceDataBytes) {
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q22.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q22.test
index 264f6aeb9..948bd7f31 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q22.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q22.test
@@ -20,9 +20,9 @@ order by qoh, i_product_name, i_brand, i_class, i_category
limit 100;
---- PLAN
Max Per-Host Resource Reservation: Memory=149.44MB Threads=5
-Per-Host Resource Estimates: Memory=2.88GB
+Per-Host Resource Estimates: Memory=5.86GB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=2.88GB mem-reservation=149.44MB thread-reservation=5 runtime-filters-memory=3.00MB
+| Per-Host Resources: mem-estimate=5.86GB mem-reservation=149.44MB thread-reservation=5 runtime-filters-memory=3.00MB
PLAN-ROOT SINK
| output exprs: CASE valid_tid(5,7,9,11,13) WHEN 5 THEN i_product_name WHEN 7 THEN i_product_name WHEN 9 THEN i_product_name WHEN 11 THEN i_product_name WHEN 13 THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN 5 THEN i_brand WHEN 7 THEN i_brand WHEN 9 THEN i_brand WHEN 11 THEN NULL WHEN 13 THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN 5 THEN i_class WHEN 7 THEN i_class WHEN 9 THEN NULL WHEN 11 THEN NULL WHEN 13 THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN 5 THEN i_category WHEN 7 TH [...]
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
@@ -36,7 +36,7 @@ PLAN-ROOT SINK
08:AGGREGATE [FINALIZE]
| output: aggif(valid_tid(5,7,9,11,13) IN (CAST(5 AS INT), CAST(7 AS INT), CAST(9 AS INT), CAST(11 AS INT), CAST(13 AS INT)), CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(7 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(9 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(11 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(13 AS INT) THEN avg(inv_quantity_on_hand) END)
| group by: CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_product_name WHEN CAST(7 AS INT) THEN i_product_name WHEN CAST(9 AS INT) THEN i_product_name WHEN CAST(11 AS INT) THEN i_product_name WHEN CAST(13 AS INT) THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_brand WHEN CAST(7 AS INT) THEN i_brand WHEN CAST(9 AS INT) THEN i_brand WHEN CAST(11 AS INT) THEN NULL WHEN CAST(13 AS INT) THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_class [...]
-| mem-estimate=571.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| mem-estimate=631.29MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| tuple-ids=14 row-size=60B cardinality=35.25M
| in pipelines: 08(GETNEXT), 07(OPEN)
|
@@ -56,7 +56,7 @@ PLAN-ROOT SINK
| Class 4
| output: avg(CAST(inv_quantity_on_hand AS BIGINT))
| group by: NULL, NULL, NULL, NULL
-| mem-estimate=2.32GB mem-reservation=112.44MB thread-reservation=0
+| mem-estimate=5.24GB mem-reservation=112.44MB thread-reservation=0
| tuple-ids=4N,6N,8N,10N,12N row-size=422B cardinality=35.25M
| in pipelines: 07(GETNEXT), 00(OPEN)
|
@@ -129,7 +129,7 @@ PLAN-ROOT SINK
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
Max Per-Host Resource Reservation: Memory=288.82MB Threads=10
-Per-Host Resource Estimates: Memory=4.42GB
+Per-Host Resource Estimates: Memory=5.67GB
F05:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=4.02MB mem-reservation=4.00MB thread-reservation=1
PLAN-ROOT SINK
@@ -144,7 +144,7 @@ PLAN-ROOT SINK
| in pipelines: 09(GETNEXT)
|
F04:PLAN FRAGMENT [HASH(CASE valid_tid(4,6,8,10,12) WHEN 4 THEN murmur_hash(i_product_name) WHEN 6 THEN murmur_hash(i_product_name) WHEN 8 THEN murmur_hash(i_product_name) WHEN 10 THEN murmur_hash(i_product_name) WHEN 12 THEN murmur_hash(NULL) END,CASE valid_tid(4,6,8,10,12) WHEN 4 THEN murmur_hash(i_brand) WHEN 6 THEN murmur_hash(i_brand) WHEN 8 THEN murmur_hash(i_brand) WHEN 10 THEN murmur_hash(NULL) WHEN 12 THEN murmur_hash(NULL) END,CASE valid_tid(4,6,8,10,12) WHEN 4 THEN murmur_hash [...]
-Per-Host Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-reservation=1
+Per-Host Resources: mem-estimate=2.77GB mem-reservation=142.69MB thread-reservation=1
09:TOP-N [LIMIT=100]
| order by: aggif(valid_tid(5,7,9,11,13) IN (5, 7, 9, 11, 13), CASE valid_tid(5,7,9,11,13) WHEN 5 THEN avg(inv_quantity_on_hand) WHEN 7 THEN avg(inv_quantity_on_hand) WHEN 9 THEN avg(inv_quantity_on_hand) WHEN 11 THEN avg(inv_quantity_on_hand) WHEN 13 THEN avg(inv_quantity_on_hand) END) ASC, CASE valid_tid(5,7,9,11,13) WHEN 5 THEN i_product_name WHEN 7 THEN i_product_name WHEN 9 THEN i_product_name WHEN 11 THEN i_product_name WHEN 13 THEN NULL END ASC, CASE valid_tid(5,7,9,11,13) WHEN 5 [...]
| mem-estimate=5.47KB mem-reservation=0B thread-reservation=0
@@ -154,7 +154,7 @@ Per-Host Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-reservat
08:AGGREGATE [FINALIZE]
| output: aggif(valid_tid(5,7,9,11,13) IN (CAST(5 AS INT), CAST(7 AS INT), CAST(9 AS INT), CAST(11 AS INT), CAST(13 AS INT)), CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(7 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(9 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(11 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(13 AS INT) THEN avg(inv_quantity_on_hand) END)
| group by: CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_product_name WHEN CAST(7 AS INT) THEN i_product_name WHEN CAST(9 AS INT) THEN i_product_name WHEN CAST(11 AS INT) THEN i_product_name WHEN CAST(13 AS INT) THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_brand WHEN CAST(7 AS INT) THEN i_brand WHEN CAST(9 AS INT) THEN i_brand WHEN CAST(11 AS INT) THEN NULL WHEN CAST(13 AS INT) THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_class [...]
-| mem-estimate=533.82MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| mem-estimate=555.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| tuple-ids=14 row-size=60B cardinality=35.25M
| in pipelines: 08(GETNEXT), 14(OPEN)
|
@@ -174,7 +174,7 @@ Per-Host Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-reservat
| Class 4
| output: avg:merge(inv_quantity_on_hand)
| group by: NULL, NULL, NULL, NULL
-| mem-estimate=1.87GB mem-reservation=108.69MB thread-reservation=0
+| mem-estimate=2.22GB mem-reservation=108.69MB thread-reservation=0
| tuple-ids=5N,7N,9N,11N,13N row-size=422B cardinality=35.25M
| in pipelines: 14(GETNEXT), 00(OPEN)
|
@@ -184,7 +184,7 @@ Per-Host Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-reservat
| in pipelines: 00(GETNEXT)
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
-Per-Host Resources: mem-estimate=1.90GB mem-reservation=140.62MB thread-reservation=2 runtime-filters-memory=3.00MB
+Per-Host Resources: mem-estimate=2.78GB mem-reservation=140.62MB thread-reservation=2 runtime-filters-memory=3.00MB
07:AGGREGATE [STREAMING]
| Class 0
| output: avg(CAST(inv_quantity_on_hand AS BIGINT))
@@ -201,7 +201,7 @@ Per-Host Resources: mem-estimate=1.90GB mem-reservation=140.62MB thread-reservat
| Class 4
| output: avg(CAST(inv_quantity_on_hand AS BIGINT))
| group by: NULL, NULL, NULL, NULL
-| mem-estimate=1.76GB mem-reservation=113.00MB thread-reservation=0
+| mem-estimate=2.64GB mem-reservation=113.00MB thread-reservation=0
| tuple-ids=4N,6N,8N,10N,12N row-size=422B cardinality=35.25M
| in pipelines: 00(GETNEXT)
|
@@ -295,7 +295,7 @@ Per-Host Resources: mem-estimate=1.90GB mem-reservation=140.62MB thread-reservat
in pipelines: 00(GETNEXT)
---- PARALLELPLANS
Max Per-Host Resource Reservation: Memory=300.45MB Threads=9
-Per-Host Resource Estimates: Memory=4.26GB
+Per-Host Resource Estimates: Memory=5.52GB
F05:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Instance Resources: mem-estimate=4.02MB mem-reservation=4.00MB thread-reservation=1
PLAN-ROOT SINK
@@ -310,7 +310,7 @@ PLAN-ROOT SINK
| in pipelines: 09(GETNEXT)
|
F04:PLAN FRAGMENT [HASH(CASE valid_tid(4,6,8,10,12) WHEN 4 THEN murmur_hash(i_product_name) WHEN 6 THEN murmur_hash(i_product_name) WHEN 8 THEN murmur_hash(i_product_name) WHEN 10 THEN murmur_hash(i_product_name) WHEN 12 THEN murmur_hash(NULL) END,CASE valid_tid(4,6,8,10,12) WHEN 4 THEN murmur_hash(i_brand) WHEN 6 THEN murmur_hash(i_brand) WHEN 8 THEN murmur_hash(i_brand) WHEN 10 THEN murmur_hash(NULL) WHEN 12 THEN murmur_hash(NULL) END,CASE valid_tid(4,6,8,10,12) WHEN 4 THEN murmur_hash [...]
-Per-Instance Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-reservation=1
+Per-Instance Resources: mem-estimate=2.77GB mem-reservation=142.69MB thread-reservation=1
09:TOP-N [LIMIT=100]
| order by: aggif(valid_tid(5,7,9,11,13) IN (5, 7, 9, 11, 13), CASE valid_tid(5,7,9,11,13) WHEN 5 THEN avg(inv_quantity_on_hand) WHEN 7 THEN avg(inv_quantity_on_hand) WHEN 9 THEN avg(inv_quantity_on_hand) WHEN 11 THEN avg(inv_quantity_on_hand) WHEN 13 THEN avg(inv_quantity_on_hand) END) ASC, CASE valid_tid(5,7,9,11,13) WHEN 5 THEN i_product_name WHEN 7 THEN i_product_name WHEN 9 THEN i_product_name WHEN 11 THEN i_product_name WHEN 13 THEN NULL END ASC, CASE valid_tid(5,7,9,11,13) WHEN 5 [...]
| mem-estimate=5.47KB mem-reservation=0B thread-reservation=0
@@ -320,7 +320,7 @@ Per-Instance Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-rese
08:AGGREGATE [FINALIZE]
| output: aggif(valid_tid(5,7,9,11,13) IN (CAST(5 AS INT), CAST(7 AS INT), CAST(9 AS INT), CAST(11 AS INT), CAST(13 AS INT)), CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(7 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(9 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(11 AS INT) THEN avg(inv_quantity_on_hand) WHEN CAST(13 AS INT) THEN avg(inv_quantity_on_hand) END)
| group by: CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_product_name WHEN CAST(7 AS INT) THEN i_product_name WHEN CAST(9 AS INT) THEN i_product_name WHEN CAST(11 AS INT) THEN i_product_name WHEN CAST(13 AS INT) THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_brand WHEN CAST(7 AS INT) THEN i_brand WHEN CAST(9 AS INT) THEN i_brand WHEN CAST(11 AS INT) THEN NULL WHEN CAST(13 AS INT) THEN NULL END, CASE valid_tid(5,7,9,11,13) WHEN CAST(5 AS INT) THEN i_class [...]
-| mem-estimate=533.82MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| mem-estimate=555.64MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| tuple-ids=14 row-size=60B cardinality=35.25M
| in pipelines: 08(GETNEXT), 14(OPEN)
|
@@ -340,7 +340,7 @@ Per-Instance Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-rese
| Class 4
| output: avg:merge(inv_quantity_on_hand)
| group by: NULL, NULL, NULL, NULL
-| mem-estimate=1.87GB mem-reservation=108.69MB thread-reservation=0
+| mem-estimate=2.22GB mem-reservation=108.69MB thread-reservation=0
| tuple-ids=5N,7N,9N,11N,13N row-size=422B cardinality=35.25M
| in pipelines: 14(GETNEXT), 00(OPEN)
|
@@ -351,7 +351,7 @@ Per-Instance Resources: mem-estimate=2.39GB mem-reservation=142.69MB thread-rese
|
F00:PLAN FRAGMENT [RANDOM] hosts=2 instances=2
Per-Host Shared Resources: mem-estimate=3.00MB mem-reservation=3.00MB thread-reservation=0 runtime-filters-memory=3.00MB
-Per-Instance Resources: mem-estimate=1.79GB mem-reservation=129.00MB thread-reservation=1
+Per-Instance Resources: mem-estimate=2.68GB mem-reservation=129.00MB thread-reservation=1
07:AGGREGATE [STREAMING]
| Class 0
| output: avg(CAST(inv_quantity_on_hand AS BIGINT))
@@ -368,7 +368,7 @@ Per-Instance Resources: mem-estimate=1.79GB mem-reservation=129.00MB thread-rese
| Class 4
| output: avg(CAST(inv_quantity_on_hand AS BIGINT))
| group by: NULL, NULL, NULL, NULL
-| mem-estimate=1.76GB mem-reservation=113.00MB thread-reservation=0
+| mem-estimate=2.64GB mem-reservation=113.00MB thread-reservation=0
| tuple-ids=4N,6N,8N,10N,12N row-size=422B cardinality=35.25M
| in pipelines: 00(GETNEXT)
|
diff --git a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q67.test b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q67.test
index 1c81c16fe..4f90870f7 100644
--- a/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q67.test
+++ b/testdata/workloads/functional-planner/queries/PlannerTest/tpcds/tpcds-q67.test
@@ -42,9 +42,9 @@ order by i_category
limit 100;
---- PLAN
Max Per-Host Resource Reservation: Memory=251.75MB Threads=5
-Per-Host Resource Estimates: Memory=3.59GB
+Per-Host Resource Estimates: Memory=4.19GB
F00:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
-| Per-Host Resources: mem-estimate=3.59GB mem-reservation=251.75MB thread-reservation=5 runtime-filters-memory=3.00MB
+| Per-Host Resources: mem-estimate=4.19GB mem-reservation=251.75MB thread-reservation=5 runtime-filters-memory=3.00MB
PLAN-ROOT SINK
| output exprs: i_category, i_class, i_brand, i_product_name, d_year, d_qoy, d_moy, s_store_id, sumsales, rk
| mem-estimate=4.00MB mem-reservation=4.00MB spill-buffer=2.00MB thread-reservation=0
@@ -81,7 +81,7 @@ PLAN-ROOT SINK
08:AGGREGATE [FINALIZE]
| output: aggif(valid_tid(4,5,6,7,8,9,10,11,12) IN (CAST(4 AS INT), CAST(5 AS INT), CAST(6 AS INT), CAST(7 AS INT), CAST(8 AS INT), CAST(9 AS INT), CAST(10 AS INT), CAST(11 AS INT), CAST(12 AS INT)), CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN CAST(4 AS INT) THEN sum(coalesce(ss_sales_price * ss_quantity, 0)) WHEN CAST(5 AS INT) THEN sum(coalesce(ss_sales_price * ss_quantity, 0)) WHEN CAST(6 AS INT) THEN sum(coalesce(ss_sales_price * ss_quantity, 0)) WHEN CAST(7 AS INT) THEN sum(coalesce( [...]
| group by: CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN CAST(4 AS INT) THEN i_category WHEN CAST(5 AS INT) THEN i_category WHEN CAST(6 AS INT) THEN i_category WHEN CAST(7 AS INT) THEN i_category WHEN CAST(8 AS INT) THEN i_category WHEN CAST(9 AS INT) THEN i_category WHEN CAST(10 AS INT) THEN i_category WHEN CAST(11 AS INT) THEN i_category WHEN CAST(12 AS INT) THEN NULL END, CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN CAST(4 AS INT) THEN i_class WHEN CAST(5 AS INT) THEN i_class WHEN CAST(6 A [...]
-| mem-estimate=513.92MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
+| mem-estimate=515.85MB mem-reservation=34.00MB spill-buffer=2.00MB thread-reservation=0
| tuple-ids=13 row-size=92B cardinality=15.09M
| in pipelines: 08(GETNEXT), 07(OPEN)
|
@@ -113,7 +113,7 @@ PLAN-ROOT SINK
| Class 8
| output: sum(coalesce(ss_sales_price * CAST(ss_quantity AS DECIMAL(10,0)), CAST(0 AS DECIMAL(18,2))))
| group by: NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
-| mem-estimate=3.08GB mem-reservation=210.75MB thread-reservation=0
+| mem-estimate=3.68GB mem-reservation=210.75MB thread-reservation=0
| tuple-ids=4N,5N,6N,7N,8N,9N,10N,11N,12N row-size=1.07KB cardinality=15.09M
| in pipelines: 07(GETNEXT), 00(OPEN)
|
@@ -187,7 +187,7 @@ PLAN-ROOT SINK
in pipelines: 00(GETNEXT)
---- DISTRIBUTEDPLAN
Max Per-Host Resource Reservation: Memory=477.45MB Threads=11
-Per-Host Resource Estimates: Memory=6.58GB
+Per-Host Resource Estimates: Memory=6.69GB
F06:PLAN FRAGMENT [UNPARTITIONED] hosts=1 instances=1
| Per-Host Resources: mem-estimate=4.03MB mem-reservation=4.00MB thread-reservation=1
PLAN-ROOT SINK
@@ -237,7 +237,7 @@ Per-Host Resources: mem-estimate=4.08MB mem-reservation=4.00MB thread-reservatio
| in pipelines: 09(GETNEXT)
|
F04:PLAN FRAGMENT [HASH(CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN 4 THEN murmur_hash(i_category) WHEN 5 THEN murmur_hash(i_category) WHEN 6 THEN murmur_hash(i_category) WHEN 7 THEN murmur_hash(i_category) WHEN 8 THEN murmur_hash(i_category) WHEN 9 THEN murmur_hash(i_category) WHEN 10 THEN murmur_hash(i_category) WHEN 11 THEN murmur_hash(i_category) WHEN 12 THEN murmur_hash(NULL) END,CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN 4 THEN murmur_hash(i_class) WHEN 5 THEN murmur_hash(i_class) WHE [...]
-Per-Host Resources: mem-estimate=3.26GB mem-reservation=243.81MB thread-reservation=1
+Per-Host Resources: mem-estimate=3.27GB mem-reservation=243.81MB thread-reservation=1
09:TOP-N
| order by: CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN 4 THEN i_category WHEN 5 THEN i_category WHEN 6 THEN i_category WHEN 7 THEN i_category WHEN 8 THEN i_category WHEN 9 THEN i_category WHEN 10 THEN i_category WHEN 11 THEN i_category WHEN 12 THEN NULL END ASC NULLS LAST, aggif(valid_tid(4,5,6,7,8,9,10,11,12) IN (4, 5, 6, 7, 8, 9, 10, 11, 12), CASE valid_tid(4,5,6,7,8,9,10,11,12) WHEN 4 THEN sum(coalesce(ss_sales_price * ss_quantity, 0)) WHEN 5 THEN sum(coalesce(ss_sales_price * ss_quan [...]
| limit with ties: 200
@@ -281,7 +281,7 @@ Per-Host Resources: mem-estimate=3.26GB mem-reservation=243.81MB thread-reservat
| Class 8
| output: sum:merge(coalesce(ss_sales_price * ss_quantity, 0))
| group by: NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
-| mem-estimate=2.77GB mem-reservation=209.81MB thread-reservation=0
+| mem-estimate=2.78GB mem-reservation=209.81MB thread-reservation=0
| tuple-ids=4N,5N,6N,7N,8N,9N,10N,11N,12N row-size=1.07KB cardinality=15.09M
| in pipelines: 17(GETNEXT), 00(OPEN)
|
@@ -291,7 +291,7 @@ Per-Host Resources: mem-estimate=3.26GB mem-reservation=243.81MB thread-reservat
| in pipelines: 00(GETNEXT)
|
F00:PLAN FRAGMENT [RANDOM] hosts=3 instances=3
-Per-Host Resources: mem-estimate=3.12GB mem-reservation=223.62MB thread-reservation=2 runtime-filters-memory=3.00MB
+Per-Host Resources: mem-estimate=3.23GB mem-reservation=223.62MB thread-reservation=2 runtime-filters-memory=3.00MB
07:AGGREGATE [STREAMING]
| Class 0
| output: sum(coalesce(ss_sales_price * CAST(ss_quantity AS DECIMAL(10,0)), CAST(0 AS DECIMAL(18,2))))
@@ -320,7 +320,7 @@ Per-Host Resources: mem-estimate=3.12GB mem-reservation=223.62MB thread-reservat
| Class 8
| output: sum(coalesce(ss_sales_price * CAST(ss_quantity AS DECIMAL(10,0)), CAST(0 AS DECIMAL(18,2))))
| group by: NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL
-| mem-estimate=3.03GB mem-reservation=211.00MB thread-reservation=0
+| mem-estimate=3.14GB mem-reservation=211.00MB thread-reservation=0
| tuple-ids=4N,5N,6N,7N,8N,9N,10N,11N,12N row-size=1.07KB cardinality=15.09M
| in pipelines: 00(GETNEXT)
|
[impala] 01/03: IMPALA-12281: Disallow unsetting REQUEST_POOL if it is set by client
Posted by db...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
dbecker pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
commit c9aed342f54494d2f0b5e5aada472d6af6697adc
Author: Riza Suminto <ri...@cloudera.com>
AuthorDate: Wed Jul 12 15:20:56 2023 -0700
IMPALA-12281: Disallow unsetting REQUEST_POOL if it is set by client
IMPALA-12056 enable child query to unset REQUEST_POOL if it is set by
Frontend.java as part of executor group selection. However, the
implementation miss to setRequest_pool_set_by_frontend(false) if
REQUEST_POOL is explicitly set by client request through impala-shell
configuration. This cause child query to always unset REQUEST_POOL if
parent query was executed via impala-shell. This patch fix the issue by
checking query options that comes from client.
This patch also tidy up null and empty REQUEST_POOL checking by using
StringUtils.isNotEmpty().
Testing:
- Add testcase in test_query_cpu_count_divisor_default
Change-Id: Ib5036859d51bc64f568da405f730c8f3ffebb742
Reviewed-on: http://gerrit.cloudera.org:8080/20189
Reviewed-by: Impala Public Jenkins <im...@cloudera.com>
Tested-by: Riza Suminto <ri...@cloudera.com>
---
.../java/org/apache/impala/service/Frontend.java | 22 +++++++----
tests/custom_cluster/test_executor_groups.py | 45 +++++++++++++++++-----
2 files changed, 50 insertions(+), 17 deletions(-)
diff --git a/fe/src/main/java/org/apache/impala/service/Frontend.java b/fe/src/main/java/org/apache/impala/service/Frontend.java
index 18365487d..b2124a1a4 100644
--- a/fe/src/main/java/org/apache/impala/service/Frontend.java
+++ b/fe/src/main/java/org/apache/impala/service/Frontend.java
@@ -52,6 +52,7 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.metastore.IMetaStoreClient;
@@ -1943,7 +1944,8 @@ public class Frontend {
// If defined, request_pool can be a suffix of the group name prefix. For example
// group_set_prefix = root.queue1
// request_pool = queue1
- if (request_pool != null && !e.getExec_group_name_prefix().endsWith(request_pool)) {
+ if (StringUtils.isNotEmpty(request_pool)
+ && !e.getExec_group_name_prefix().endsWith(request_pool)) {
continue;
}
TExecutorGroupSet new_entry = new TExecutorGroupSet(e);
@@ -1965,7 +1967,8 @@ public class Frontend {
}
result.add(new_entry);
}
- if (executorGroupSets.size() > 0 && result.size() == 0 && request_pool != null) {
+ if (executorGroupSets.size() > 0 && result.size() == 0
+ && StringUtils.isNotEmpty(request_pool)) {
throw new AnalysisException("Request pool: " + request_pool
+ " does not map to any known executor group set.");
}
@@ -2018,6 +2021,9 @@ public class Frontend {
TQueryOptions queryOptions = queryCtx.client_request.getQuery_options();
boolean enable_replan = queryOptions.isEnable_replan();
+ final boolean clientSetRequestPool = queryOptions.isSetRequest_pool();
+ Preconditions.checkState(
+ !clientSetRequestPool || !queryOptions.getRequest_pool().isEmpty());
List<TExecutorGroupSet> originalExecutorGroupSets =
ExecutorMembershipSnapshot.getAllExecutorGroupSets();
@@ -2131,7 +2137,7 @@ public class Frontend {
}
if (notScalable) {
- setGroupNamePrefix(default_executor_group, req, group_set);
+ setGroupNamePrefix(default_executor_group, clientSetRequestPool, req, group_set);
addInfoString(
groupSetProfile, VERDICT, "Assign to first group because " + reason);
FrontendProfile.getCurrent().addChildrenProfile(groupSetProfile);
@@ -2187,7 +2193,7 @@ public class Frontend {
}
boolean matchFound = false;
- if (queryOptions.isSetRequest_pool()) {
+ if (clientSetRequestPool) {
if (!default_executor_group) {
Preconditions.checkState(group_set.getExec_group_name_prefix().endsWith(
queryOptions.getRequest_pool()));
@@ -2216,7 +2222,7 @@ public class Frontend {
FrontendProfile.getCurrent().addChildrenProfile(groupSetProfile);
if (matchFound) {
- setGroupNamePrefix(default_executor_group, req, group_set);
+ setGroupNamePrefix(default_executor_group, clientSetRequestPool, req, group_set);
break;
}
@@ -2267,14 +2273,14 @@ public class Frontend {
return req;
}
- private static void setGroupNamePrefix(
- boolean default_executor_group, TExecRequest req, TExecutorGroupSet group_set) {
+ private static void setGroupNamePrefix(boolean default_executor_group,
+ boolean clientSetRequestPool, TExecRequest req, TExecutorGroupSet group_set) {
// Set the group name prefix in both the returned query options and
// the query context for non default group setup.
if (!default_executor_group) {
String namePrefix = group_set.getExec_group_name_prefix();
req.query_options.setRequest_pool(namePrefix);
- req.setRequest_pool_set_by_frontend(true);
+ req.setRequest_pool_set_by_frontend(!clientSetRequestPool);
if (req.query_exec_request != null) {
req.query_exec_request.query_ctx.setRequest_pool(namePrefix);
}
diff --git a/tests/custom_cluster/test_executor_groups.py b/tests/custom_cluster/test_executor_groups.py
index c42c06c15..dd8693d47 100644
--- a/tests/custom_cluster/test_executor_groups.py
+++ b/tests/custom_cluster/test_executor_groups.py
@@ -870,7 +870,9 @@ class TestExecutorGroups(CustomClusterTestSuite):
exec_group_set_prefix="root.large") == 1
def _set_query_options(self, query_options):
- """Set query options"""
+ """Set query options by running it as an SQL statement.
+ To mimic impala-shell behavior, use self.client.set_configuration() instead.
+ """
for k, v in query_options.items():
self.execute_query_expect_success(self.client, "SET {}='{}'".format(k, v))
@@ -904,9 +906,11 @@ class TestExecutorGroups(CustomClusterTestSuite):
@UniqueDatabase.parametrize(sync_ddl=True)
@pytest.mark.execute_serially
def test_query_cpu_count_divisor_default(self, unique_database):
- # Expect to run the query on the small group by default.
coordinator_test_args = ""
self._setup_three_exec_group_cluster(coordinator_test_args)
+ self.client.clear_configuration()
+
+ # Expect to run the query on the small group by default.
self._set_query_options({'COMPUTE_PROCESSING_COST': 'true'})
self._run_query_and_verify_profile(CPU_TEST_QUERY,
["Executor Group: root.small-group", "EffectiveParallelism: 11",
@@ -957,22 +961,36 @@ class TestExecutorGroups(CustomClusterTestSuite):
self._run_query_and_verify_profile(compute_stats_query,
["ExecutorGroupsConsidered: 1",
"Verdict: Assign to first group because query is not auto-scalable"],
- ["Executor Group:"])
+ ["Query Options (set by configuration): REQUEST_POOL=",
+ "Executor Group:"])
self._verify_total_admitted_queries("root.small", 4)
self._verify_total_admitted_queries("root.large", 2)
- # Test that child queries follow REQUEST_POOL that was set by client.
+ # Test that child queries follow REQUEST_POOL that is set through client
+ # configuration. Two child queries should all run in root.small.
+ self.client.set_configuration({'REQUEST_POOL': 'root.small'})
+ self._run_query_and_verify_profile(compute_stats_query,
+ ["Query Options (set by configuration): REQUEST_POOL=root.small",
+ "ExecutorGroupsConsidered: 1",
+ "Verdict: Assign to first group because query is not auto-scalable"],
+ ["Executor Group:"])
+ self._verify_total_admitted_queries("root.small", 6)
+ self.client.clear_configuration()
+
+ # Test that child queries follow REQUEST_POOL that is set through SQL statement.
# Two child queries should all run in root.large.
self._set_query_options({'REQUEST_POOL': 'root.large'})
self._run_query_and_verify_profile(compute_stats_query,
- ["ExecutorGroupsConsidered: 1",
+ ["Query Options (set by configuration): REQUEST_POOL=root.large",
+ "ExecutorGroupsConsidered: 1",
"Verdict: Assign to first group because query is not auto-scalable"],
["Executor Group:"])
self._verify_total_admitted_queries("root.large", 4)
# Test that REQUEST_POOL will override executor group selection
self._run_query_and_verify_profile(CPU_TEST_QUERY,
- ["Executor Group: root.large-group",
+ ["Query Options (set by configuration): REQUEST_POOL=root.large",
+ "Executor Group: root.large-group",
("Verdict: query option REQUEST_POOL=root.large is set. "
"Memory and cpu limit checking is skipped."),
"EffectiveParallelism: 13", "ExecutorGroupsConsidered: 1"])
@@ -982,7 +1000,8 @@ class TestExecutorGroups(CustomClusterTestSuite):
'COMPUTE_PROCESSING_COST': 'false',
'REQUEST_POOL': 'root.large'})
self._run_query_and_verify_profile(CPU_TEST_QUERY,
- ["Executor Group: root.large-group",
+ ["Query Options (set by configuration): REQUEST_POOL=root.large",
+ "Executor Group: root.large-group",
("Verdict: query option REQUEST_POOL=root.large is set. "
"Memory and cpu limit checking is skipped."),
"ExecutorGroupsConsidered: 1"],
@@ -993,6 +1012,14 @@ class TestExecutorGroups(CustomClusterTestSuite):
'REQUEST_POOL': '',
'COMPUTE_PROCESSING_COST': 'true'})
+ # Test that empty REQUEST_POOL should have no impact.
+ self.client.set_configuration({'REQUEST_POOL': ''})
+ self._run_query_and_verify_profile(CPU_TEST_QUERY,
+ ["Executor Group: root.small-group", "ExecutorGroupsConsidered: 2",
+ "Verdict: Match"],
+ ["Query Options (set by configuration): REQUEST_POOL="])
+ self.client.clear_configuration()
+
# Test that GROUPING_TEST_QUERY will get assigned to the large group.
self._run_query_and_verify_profile(GROUPING_TEST_QUERY,
["Executor Group: root.large-group", "ExecutorGroupsConsidered: 3",
@@ -1160,10 +1187,10 @@ class TestExecutorGroups(CustomClusterTestSuite):
# END testing insert + MAX_FS_WRITER
# Check resource pools on the Web queries site and admission site
- self._verify_query_num_for_resource_pool("root.small", 4)
+ self._verify_query_num_for_resource_pool("root.small", 7)
self._verify_query_num_for_resource_pool("root.tiny", 4)
self._verify_query_num_for_resource_pool("root.large", 12)
- self._verify_total_admitted_queries("root.small", 5)
+ self._verify_total_admitted_queries("root.small", 8)
self._verify_total_admitted_queries("root.tiny", 6)
self._verify_total_admitted_queries("root.large", 16)