You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2023/05/20 02:47:46 UTC
[hudi] branch master updated: [HUDI-6239] Fix clustering pool scheduler conf not take effect bug (#8763)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 84b3dd33edc [HUDI-6239] Fix clustering pool scheduler conf not take effect bug (#8763)
84b3dd33edc is described below
commit 84b3dd33edc774cc166ed0f3d26968645e6b5150
Author: kongwei <ko...@pku.edu.cn>
AuthorDate: Sat May 20 10:47:40 2023 +0800
[HUDI-6239] Fix clustering pool scheduler conf not take effect bug (#8763)
Co-authored-by: wei.kong <we...@shopee.com>
---
.../deltastreamer/SchedulerConfGenerator.java | 26 +++++++++++++++++-----
.../deltastreamer/TestSchedulerConfGenerator.java | 26 ++++++++++++++++++++++
2 files changed, 46 insertions(+), 6 deletions(-)
diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
index e23cb04ae70..038a64b4e2f 100644
--- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
+++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/deltastreamer/SchedulerConfGenerator.java
@@ -51,10 +51,24 @@ public class SchedulerConfGenerator {
public static final String SPARK_SCHEDULER_FAIR_MODE = "FAIR";
private static final String SPARK_SCHEDULING_PATTERN =
- "<?xml version=\"1.0\"?>\n<allocations>\n <pool name=\"%s\">\n"
- + " <schedulingMode>%s</schedulingMode>\n <weight>%s</weight>\n <minShare>%s</minShare>\n"
- + " </pool>\n <pool name=\"%s\">\n <schedulingMode>%s</schedulingMode>\n"
- + " <weight>%s</weight>\n <minShare>%s</minShare>\n </pool>\n</allocations>";
+ "<?xml version=\"1.0\"?>\n"
+ + "<allocations>\n"
+ + " <pool name=\"%s\">\n"
+ + " <schedulingMode>%s</schedulingMode>\n"
+ + " <weight>%s</weight>\n"
+ + " <minShare>%s</minShare>\n"
+ + " </pool>\n"
+ + " <pool name=\"%s\">\n"
+ + " <schedulingMode>%s</schedulingMode>\n"
+ + " <weight>%s</weight>\n"
+ + " <minShare>%s</minShare>\n"
+ + " </pool>\n"
+ + " <pool name=\"%s\">\n"
+ + " <schedulingMode>%s</schedulingMode>\n"
+ + " <weight>%s</weight>\n"
+ + " <minShare>%s</minShare>\n"
+ + " </pool>\n"
+ + "</allocations>";
/**
* Helper to generate spark scheduling configs in XML format with input params.
@@ -67,8 +81,8 @@ public class SchedulerConfGenerator {
* @param clusteringWeight Minshare for clustering
* @return Spark scheduling configs
*/
- private static String generateConfig(Integer deltaSyncWeight, Integer compactionWeight, Integer deltaSyncMinShare,
- Integer compactionMinShare, Integer clusteringWeight, Integer clusteringMinShare) {
+ static String generateConfig(Integer deltaSyncWeight, Integer compactionWeight, Integer deltaSyncMinShare,
+ Integer compactionMinShare, Integer clusteringWeight, Integer clusteringMinShare) {
return String.format(SPARK_SCHEDULING_PATTERN, DELTASYNC_POOL_NAME, SPARK_SCHEDULER_FAIR_MODE,
deltaSyncWeight.toString(), deltaSyncMinShare.toString(), COMPACT_POOL_NAME, SPARK_SCHEDULER_FAIR_MODE,
compactionWeight.toString(), compactionMinShare.toString(), CLUSTERING_POOL_NAME, SPARK_SCHEDULER_FAIR_MODE,
diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
index 26b8bc1c885..5085261b9d1 100644
--- a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
+++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestSchedulerConfGenerator.java
@@ -25,6 +25,7 @@ import org.junit.jupiter.api.Test;
import java.util.Map;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -51,4 +52,29 @@ public class TestSchedulerConfGenerator {
configs = SchedulerConfGenerator.getSparkSchedulingConfigs(cfg);
assertNotNull(configs.get(SparkConfigs.SPARK_SCHEDULER_ALLOCATION_FILE_KEY()), "all satisfies");
}
+
+ @Test
+ public void testGenerateConfig() {
+ String targetConfig =
+ "<?xml version=\"1.0\"?>\n"
+ + "<allocations>\n"
+ + " <pool name=\"hoodiedeltasync\">\n"
+ + " <schedulingMode>FAIR</schedulingMode>\n"
+ + " <weight>1</weight>\n"
+ + " <minShare>2</minShare>\n"
+ + " </pool>\n"
+ + " <pool name=\"hoodiecompact\">\n"
+ + " <schedulingMode>FAIR</schedulingMode>\n"
+ + " <weight>3</weight>\n"
+ + " <minShare>4</minShare>\n"
+ + " </pool>\n"
+ + " <pool name=\"hoodiecluster\">\n"
+ + " <schedulingMode>FAIR</schedulingMode>\n"
+ + " <weight>5</weight>\n"
+ + " <minShare>6</minShare>\n"
+ + " </pool>\n"
+ + "</allocations>";
+ String generatedConfig = SchedulerConfGenerator.generateConfig(1, 3, 2, 4, 5, 6);
+ assertEquals(targetConfig, generatedConfig);
+ }
}