You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sl...@apache.org on 2023/09/25 05:23:09 UTC
[hadoop] branch trunk updated: YARN-9586. Need more doc for yarn.federation.policy-manager-params when LoadBasedRouterPolicy is used. (#6085) Contributed by Shilun Fan.
This is an automated email from the ASF dual-hosted git repository.
slfan1989 pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new bf9975a1b315 YARN-9586. Need more doc for yarn.federation.policy-manager-params when LoadBasedRouterPolicy is used. (#6085) Contributed by Shilun Fan.
bf9975a1b315 is described below
commit bf9975a1b315942ad34928a980b0fc5544361e36
Author: slfan1989 <55...@users.noreply.github.com>
AuthorDate: Mon Sep 25 13:23:02 2023 +0800
YARN-9586. Need more doc for yarn.federation.policy-manager-params when LoadBasedRouterPolicy is used. (#6085) Contributed by Shilun Fan.
Reviewed-by: Inigo Goiri <in...@apache.org>
Signed-off-by: Shilun Fan <sl...@apache.org>
---
.../yarn/server/router/TestRouterServerUtil.java | 39 ++++++++++++++++++++++
.../src/test/resources/yarn-site.xml | 4 +++
.../src/site/markdown/Federation.md | 20 +++++++++++
3 files changed, 63 insertions(+)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java
index e82f67d12d56..dcf3bd5bc0ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterServerUtil.java
@@ -24,6 +24,9 @@ import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.federation.policies.dao.WeightedPolicyInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterIdInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDefinitionInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationRequestsInfo;
@@ -33,7 +36,10 @@ import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.yarn.server.router.webapp.TestFederationInterceptorREST.getReservationSubmissionRequestInfo;
import static org.junit.Assert.assertEquals;
@@ -122,4 +128,37 @@ public class TestRouterServerUtil {
"definitionInfo Or ReservationRequests is Null.",
() -> RouterServerUtil.convertReservationDefinition(definitionInfo3));
}
+
+ @Test
+ public void testLoadFederationPolicyManager() throws Exception {
+
+ // In this unit test, we have configured the yarn-site.xml file with
+ // the yarn.federation.policy-manager-params parameter,
+ // and subsequently, we parse this parameter.
+ // We have configured two subclusters, SC-1 and SC-2,
+ // with routerPolicyWeights set to SC-1:0.7 and SC-2:0.3,
+ // and amrmPolicyWeights set to SC-1:0.6 and SC-2:0.4.
+ // Additionally, headroomAlpha is set to 1.0.
+
+ YarnConfiguration conf = new YarnConfiguration();
+ String defaultPolicyParamString = conf.get(YarnConfiguration.FEDERATION_POLICY_MANAGER_PARAMS,
+ YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+ assertNotNull(defaultPolicyParamString);
+ ByteBuffer defaultPolicyParam = ByteBuffer.wrap(
+ defaultPolicyParamString.getBytes(StandardCharsets.UTF_8));
+ WeightedPolicyInfo policyInfo = WeightedPolicyInfo.fromByteBuffer(defaultPolicyParam);
+ float headroomAlpha = policyInfo.getHeadroomAlpha();
+ Map<SubClusterIdInfo, Float> routerPolicyWeights = policyInfo.getRouterPolicyWeights();
+ Map<SubClusterIdInfo, Float> amrmPolicyWeights = policyInfo.getAMRMPolicyWeights();
+
+ SubClusterIdInfo sc1 = new SubClusterIdInfo("SC-1");
+ SubClusterIdInfo sc2 = new SubClusterIdInfo("SC-2");
+
+ assertEquals(1.0, headroomAlpha, 0.001);
+ assertEquals(0.7, routerPolicyWeights.get(sc1), 0.001);
+ assertEquals(0.3, routerPolicyWeights.get(sc2), 0.001);
+
+ assertEquals(0.6, amrmPolicyWeights.get(sc1), 0.001);
+ assertEquals(0.4, amrmPolicyWeights.get(sc2), 0.001);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
index 84d4171c79cd..55461680b5b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/resources/yarn-site.xml
@@ -39,4 +39,8 @@
<name>yarn.resourcemanager.resource-profiles.source-file</name>
<value>profiles/sample-profiles-1.json</value>
</property>
+ <property>
+ <name>yarn.federation.policy-manager-params</name>
+ <value>{"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"}</value>
+ </property>
</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index abe2384a3928..a61325238447 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -235,6 +235,26 @@ SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
|`yarn.federation.subcluster-resolver.class` | `org.apache.hadoop.yarn.server.federation.resolver.DefaultSubClusterResolverImpl` | The class used to resolve which subcluster a node belongs to, and which subcluster(s) a rack belongs to. |
|`yarn.federation.machine-list` | `<path of machine-list file>` | Path of machine-list file used by `SubClusterResolver`. Each line of the file is a node with sub-cluster and rack information. Below is the example: <br/> <br/> node1, subcluster1, rack1 <br/> node2, subcluster2, rack1 <br/> node3, subcluster3, rack2 <br/> node4, subcluster3, rack2 |
+- yarn.federation.policy-manager-params
+
+ To configure the `yarn.federation.policy-manager-params` parameter, which represents the weight policy for the default queue,
+ and where the relevant information will be parsed as `WeightedPolicyInfo`.
+
+ We can use the following JSON format for configuration:
+
+ ```xml
+ <property>
+ <name>yarn.federation.policy-manager-params</name>
+ <value>{"routerPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.3"},{"key":{"id":"SC-1"},"value":"0.7"}]},"amrmPolicyWeights":{"entry":[{"key":{"id":"SC-2"},"value":"0.4"},{"key":{"id":"SC-1"},"value":"0.6"}]},"headroomAlpha":"1.0"}</value>
+ </property>
+ ```
+
+ This JSON configuration allows you to define the weight policy for default queue, where:
+
+ - The `routerPolicyWeights` section specifies the weightings for router policies. For instance, with a weight of `0.3` assigned to `SC-2` and `0.7` assigned to `SC-1`, this configuration will allocate `30%` of submitted application requests to `SC-2` and `70%` to `SC-1`.
+ - The `amrmPolicyWeights` represents the allocation ratios for Application Master when request containers from different subclusters' RM. For instance, when an AM requests containers, it will request `40%` of the containers from `SC-2` and `60%` of the containers from `SC-1`.
+ - The `headroomAlpha` used by policies that balance weight-based and load-based considerations in their decisions. For policies that use this parameter, values close to 1 indicate that most of the decision should be based on currently observed headroom from various sub-clusters, values close to zero, indicate that the decision should be mostly based on weights and practically ignore current load.
+
How to configure the policy-manager
--------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org