You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@flink.apache.org by zh...@apache.org on 2022/02/18 15:02:55 UTC
[flink] branch master updated: [FLINK-25225][e2e] Add e2e TPCDS tests to run against the AdaptiveBatchScheduler
This is an automated email from the ASF dual-hosted git repository.
zhuzh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/flink.git
The following commit(s) were added to refs/heads/master by this push:
new 5c7a432 [FLINK-25225][e2e] Add e2e TPCDS tests to run against the AdaptiveBatchScheduler
5c7a432 is described below
commit 5c7a432778f2f8494e104fa89ea08a71e83978b1
Author: Lijie Wang <wa...@gmail.com>
AuthorDate: Thu Feb 10 16:38:32 2022 +0800
[FLINK-25225][e2e] Add e2e TPCDS tests to run against the AdaptiveBatchScheduler
This closes #18720.
---
.../apache/flink/table/tpcds/TpcdsTestProgram.java | 3 ---
flink-end-to-end-tests/run-nightly-tests.sh | 1 +
flink-end-to-end-tests/test-scripts/test_tpcds.sh | 19 +++++++++++++++++--
3 files changed, 18 insertions(+), 5 deletions(-)
diff --git a/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java b/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
index 9ae3d80..c6251aa 100644
--- a/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
+++ b/flink-end-to-end-tests/flink-tpcds-test/src/main/java/org/apache/flink/table/tpcds/TpcdsTestProgram.java
@@ -138,9 +138,6 @@ public class TpcdsTestProgram {
TableEnvironment tEnv = TableEnvironment.create(environmentSettings);
// config Optimizer parameters
- tEnv.getConfig()
- .getConfiguration()
- .setInteger(ExecutionConfigOptions.TABLE_EXEC_RESOURCE_DEFAULT_PARALLELISM, 4);
// TODO use the default shuffle mode of batch runtime mode once FLINK-23470 is implemented
tEnv.getConfig()
.getConfiguration()
diff --git a/flink-end-to-end-tests/run-nightly-tests.sh b/flink-end-to-end-tests/run-nightly-tests.sh
index 566284f..20c4aa1 100755
--- a/flink-end-to-end-tests/run-nightly-tests.sh
+++ b/flink-end-to-end-tests/run-nightly-tests.sh
@@ -217,6 +217,7 @@ function run_group_2 {
run_test "TPC-H end-to-end test" "$END_TO_END_DIR/test-scripts/test_tpch.sh"
run_test "TPC-DS end-to-end test" "$END_TO_END_DIR/test-scripts/test_tpcds.sh"
+ run_test "TPC-DS end-to-end test with adaptive batch scheduler" "$END_TO_END_DIR/test-scripts/test_tpcds.sh AdaptiveBatch"
run_test "Heavy deployment end-to-end test" "$END_TO_END_DIR/test-scripts/test_heavy_deployment.sh" "skip_check_exceptions"
diff --git a/flink-end-to-end-tests/test-scripts/test_tpcds.sh b/flink-end-to-end-tests/test-scripts/test_tpcds.sh
index 4f88c1f..ce87326 100755
--- a/flink-end-to-end-tests/test-scripts/test_tpcds.sh
+++ b/flink-end-to-end-tests/test-scripts/test_tpcds.sh
@@ -54,10 +54,25 @@ cd "$END_TO_END_DIR"
echo "[INFO]Preparing Flink cluster..."
+SCHEDULER="${1:-Ng}"
+
+set_config_key "jobmanager.scheduler" "${SCHEDULER}"
set_config_key "taskmanager.memory.process.size" "4096m"
-set_config_key "taskmanager.numberOfTaskSlots" "4"
-set_config_key "parallelism.default" "4"
set_config_key "taskmanager.memory.network.fraction" "0.2"
+
+if [ "${SCHEDULER}" == "Ng" ]; then
+ set_config_key "taskmanager.numberOfTaskSlots" "4"
+ set_config_key "parallelism.default" "4"
+elif [ "${SCHEDULER}" == "AdaptiveBatch" ]; then
+ set_config_key "taskmanager.numberOfTaskSlots" "8"
+ set_config_key "parallelism.default" "-1"
+ set_config_key "jobmanager.adaptive-batch-scheduler.max-parallelism" "8"
+ set_config_key "jobmanager.adaptive-batch-scheduler.data-volume-per-task" "6m"
+else
+ echo "ERROR: Scheduler ${SCHEDULER} is unsupported for tpcds test. Aborting..."
+ exit 1
+fi
+
start_cluster