You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/12/05 16:38:14 UTC
[hadoop] branch trunk updated: YARN-5106. Provide a builder
interface for FairScheduler allocations for use in tests. Contributed by
Adam Antal
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 520fe2c YARN-5106. Provide a builder interface for FairScheduler allocations for use in tests. Contributed by Adam Antal
520fe2c is described below
commit 520fe2c99b328a32c62eb5ff2b3d3562fa91f0fe
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Thu Dec 5 17:37:40 2019 +0100
YARN-5106. Provide a builder interface for FairScheduler allocations for use in tests. Contributed by Adam Antal
---
.../api/impl/TestYarnClientWithReservation.java | 36 +-
.../ParameterizedSchedulerTestBase.java | 71 +-
.../resourcemanager/ReservationACLsTestBase.java | 72 +-
.../TestAppManagerWithFairScheduler.java | 121 ++-
.../resourcemanager/TestApplicationACLs.java | 43 +-
.../reservation/ReservationSystemTestUtil.java | 99 +--
.../fair/TestAllocationFileLoaderService.java | 370 ++++----
.../scheduler/fair/TestAppRunnability.java | 21 +-
.../fair/TestApplicationMasterServiceWithFS.java | 35 +-
.../scheduler/fair/TestFSAppStarvation.java | 102 +--
.../scheduler/fair/TestFSLeafQueue.java | 21 +-
.../scheduler/fair/TestFairScheduler.java | 971 +++++++++------------
.../scheduler/fair/TestFairSchedulerFairShare.java | 78 +-
.../fair/TestFairSchedulerPreemption.java | 125 ++-
.../scheduler/fair/TestFairSchedulerQueueACLs.java | 46 +-
.../fair/TestQueueManagerRealScheduler.java | 50 +-
.../scheduler/fair/TestSchedulingPolicy.java | 208 ++---
.../fair/allocationfile/AllocationFileQueue.java | 274 +++++-
.../allocationfile/AllocationFileQueueBuilder.java | 121 ---
...ava => AllocationFileQueuePlacementPolicy.java} | 57 +-
.../AllocationFileQueuePlacementRule.java | 107 +++
.../AllocationFileQueueProperties.java | 214 -----
.../AllocationFileSimpleQueueBuilder.java | 64 --
.../AllocationFileSubQueueBuilder.java | 54 --
.../fair/allocationfile/AllocationFileWriter.java | 118 ++-
.../fair/allocationfile/UserSettings.java | 16 +-
.../converter/FSConfigConverterTestCommons.java | 48 +-
.../webapp/TestRMWebServicesAppsModification.java | 33 +-
.../webapp/TestRMWebServicesReservation.java | 39 +-
29 files changed, 1573 insertions(+), 2041 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
index 7176477..06475b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClientWithReservation.java
@@ -45,6 +45,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.UTCClock;
import org.junit.Assert;
@@ -54,9 +59,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -136,25 +139,16 @@ public class TestYarnClientWithReservation {
private Configuration configureReservationForFairScheduler() {
Configuration conf = new Configuration();
- try {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <queue name=\"default\"></queue>");
- out.println(" <queue name=\"dedicated\">");
- out.println(" <reservation></reservation>");
- // set weight to 10 to make sure this queue get enough steady fair share
- out.println(" <weight>10</weight>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
- } catch (IOException e) {
- Assert.fail(e.getMessage());
- }
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("default").build())
+ .subQueue(new AllocationFileQueue.Builder("dedicated")
+ .reservation()
+ .weight(10)
+ .build())
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
return conf;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
index 4de16dc..2621685 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ParameterizedSchedulerTestBase.java
@@ -26,14 +26,18 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.After;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collection;
import java.util.stream.Collectors;
@@ -73,43 +77,40 @@ public abstract class ParameterizedSchedulerTestBase {
schedulerType = type;
switch (schedulerType) {
- case FAIR:
- configureFairScheduler(conf);
- scheduler = new FairScheduler();
- conf.set(YarnConfiguration.RM_SCHEDULER,
- FairScheduler.class.getName());
- break;
- case CAPACITY:
- scheduler = new CapacityScheduler();
- ((CapacityScheduler)scheduler).setConf(conf);
- conf.set(YarnConfiguration.RM_SCHEDULER,
- CapacityScheduler.class.getName());
- break;
- default:
- throw new IllegalArgumentException("Invalid type: " + type);
+ case FAIR:
+ configureFairScheduler(conf);
+ scheduler = new FairScheduler();
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ FairScheduler.class.getName());
+ break;
+ case CAPACITY:
+ scheduler = new CapacityScheduler();
+ ((CapacityScheduler)scheduler).setConf(conf);
+ conf.set(YarnConfiguration.RM_SCHEDULER,
+ CapacityScheduler.class.getName());
+ break;
+ default:
+ throw new IllegalArgumentException("Invalid type: " + type);
}
}
- protected void configureFairScheduler(YarnConfiguration conf)
- throws IOException {
+ protected void configureFairScheduler(YarnConfiguration configuration) {
// Disable queueMaxAMShare limitation for fair scheduler
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
- out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
- out.println("<queue name=\"root\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" <weight>1.0</weight>");
- out.println(" <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
- out.println(" <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
- out.println(" <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
-
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
- conf.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10);
+ AllocationFileWriter.create()
+ .fairDefaultQueueSchedulingPolicy()
+ .disableQueueMaxAMShareDefault()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .weight(1.0f)
+ .fairSharePreemptionTimeout(100)
+ .minSharePreemptionTimeout(120)
+ .fairSharePreemptionThreshold(.5)
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
+
+ configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE,
+ FS_ALLOC_FILE);
+ configuration.setLong(FairSchedulerConfiguration.UPDATE_INTERVAL_MS, 10);
}
@After
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
index c8ee00e..45060e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/ReservationACLsTestBase.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
@@ -53,6 +51,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.After;
import org.junit.Assert;
@@ -552,49 +556,35 @@ public class ReservationACLsTestBase extends ACLsTestBase {
return csConf;
}
- private static Configuration createFairSchedulerConfiguration() throws
- IOException {
+ private static Configuration createFairSchedulerConfiguration() {
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
- final String TEST_DIR = new File(System.getProperty("test.build.data",
+ final String testDir = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
- final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml")
+ final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println(" <queue name=\"queueA\">");
- out.println(" <aclSubmitReservations>" +
- "queueA_user,common_user " +
- "</aclSubmitReservations>");
- out.println(" <aclAdministerReservations>" +
- "queueA_admin " +
- "</aclAdministerReservations>");
- out.println(" <aclListReservations>common_user </aclListReservations>");
- out.println(" <aclSubmitApps>queueA_user,common_user </aclSubmitApps>");
- out.println(" <aclAdministerApps>queueA_admin </aclAdministerApps>");
- out.println(" <reservation> </reservation>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\">");
- out.println(" <aclSubmitApps>queueB_user,common_user </aclSubmitApps>");
- out.println(" <aclAdministerApps>queueB_admin </aclAdministerApps>");
- out.println(" <aclSubmitReservations>" +
- "queueB_user,common_user " +
- "</aclSubmitReservations>");
- out.println(" <aclAdministerReservations>" +
- "queueB_admin " +
- "</aclAdministerReservations>");
- out.println(" <aclListReservations>common_user </aclListReservations>");
- out.println(" <reservation> </reservation>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueC\">");
- out.println(" <reservation> </reservation>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
- fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .aclSubmitReservations("queueA_user,common_user ")
+ .aclAdministerReservations("queueA_admin ")
+ .aclListReservations("common_user ")
+ .aclSubmitApps("queueA_user,common_user ")
+ .aclAdministerApps("queueA_admin ")
+ .reservation().build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .aclSubmitReservations("queueB_user,common_user ")
+ .aclAdministerReservations("queueB_admin ")
+ .aclListReservations("common_user ")
+ .aclSubmitApps("queueB_user,common_user ")
+ .aclAdministerApps("queueB_admin ")
+ .reservation().build())
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .reservation().build())
+ .writeToFile(allocFile);
+
+ fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
fsConf.setBoolean(YarnConfiguration.RM_RESERVATION_SYSTEM_ENABLE, true);
fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java
index d0dcae0..37ff02d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManagerWithFairScheduler.java
@@ -25,9 +25,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Collections;
import org.apache.hadoop.security.AccessControlException;
@@ -47,6 +45,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.util.Records;
@@ -73,13 +77,9 @@ public class TestAppManagerWithFairScheduler extends AppManagerTestBase {
@Before
public void setup() throws IOException {
// Basic config with one queue (override in test if needed)
- PrintWriter out = new PrintWriter(new FileWriter(allocFileName));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"test\">");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("test").build())
+ .writeToFile(allocFileName);
conf.setClass(YarnConfiguration.RM_SCHEDULER, FairScheduler.class,
ResourceScheduler.class);
@@ -111,19 +111,15 @@ public class TestAppManagerWithFairScheduler extends AppManagerTestBase {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB;
// scheduler config with a limited queue
- PrintWriter out = new PrintWriter(new FileWriter(allocFileName));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"root\">");
- out.println(" <queue name=\"limited\">");
- out.println(" <maxContainerAllocation>" + maxAlloc + " mb 1 vcores");
- out.println(" </maxContainerAllocation>");
- out.println(" </queue>");
- out.println(" <queue name=\"unlimited\">");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("limited")
+ .maxContainerAllocation(maxAlloc + " mb 1 vcores")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("unlimited")
+ .build())
+ .build())
+ .writeToFile(allocFileName);
rmContext.getScheduler().reinitialize(conf, rmContext);
ApplicationId appId = MockApps.newAppID(1);
@@ -153,25 +149,22 @@ public class TestAppManagerWithFairScheduler extends AppManagerTestBase {
conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
- PrintWriter out = new PrintWriter(new FileWriter(allocFileName));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"root\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps> </aclAdministerApps>");
- out.println(" <queue name=\"noaccess\">");
- out.println(" </queue>");
- out.println(" <queue name=\"submitonly\">");
- out.println(" <aclSubmitApps>test </aclSubmitApps>");
- out.println(" <aclAdministerApps> </aclAdministerApps>");
- out.println(" </queue>");
- out.println(" <queue name=\"adminonly\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps>test </aclAdministerApps>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .subQueue(new AllocationFileQueue.Builder("noaccess")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("submitonly")
+ .aclSubmitApps("test ")
+ .aclAdministerApps(" ")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("adminonly")
+ .aclSubmitApps(" ")
+ .aclAdministerApps("test ")
+ .build())
+ .build())
+ .writeToFile(allocFileName);
rmContext.getScheduler().reinitialize(conf, rmContext);
ApplicationId appId = MockApps.newAppID(1);
@@ -206,17 +199,14 @@ public class TestAppManagerWithFairScheduler extends AppManagerTestBase {
conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
- PrintWriter out = new PrintWriter(new FileWriter(allocFileName));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"root\">");
- out.println(" <queue name=\"noaccess\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps> </aclAdministerApps>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("noaccess")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .build())
+ .build())
+ .writeToFile(allocFileName);
rmContext.getScheduler().reinitialize(conf, rmContext);
ApplicationId appId = MockApps.newAppID(1);
@@ -235,20 +225,19 @@ public class TestAppManagerWithFairScheduler extends AppManagerTestBase {
conf.set(YarnConfiguration.YARN_ACL_ENABLE, "true");
- PrintWriter out = new PrintWriter(new FileWriter(allocFileName));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"root\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps> </aclAdministerApps>");
- out.println(" <queue name=\"noaccess\" type=\"parent\">");
- out.println(" </queue>");
- out.println(" <queue name=\"submitonly\" type=\"parent\">");
- out.println(" <aclSubmitApps>test </aclSubmitApps>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .subQueue(new AllocationFileQueue.Builder("noaccess")
+ .parent(true)
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("submitonly")
+ .parent(true)
+ .aclSubmitApps("test ")
+ .build())
+ .build())
+ .writeToFile(allocFileName);
rmContext.getScheduler().reinitialize(conf, rmContext);
ApplicationId appId = MockApps.newAppID(1);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
index e952c50..846c77d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
@@ -23,9 +23,7 @@ import static org.mockito.Mockito.when;
import static org.mockito.ArgumentMatchers.any;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.util.HashMap;
@@ -35,6 +33,16 @@ import java.util.Map;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueuePlacementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueuePlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.After;
import org.junit.Assert;
import org.slf4j.Logger;
@@ -181,26 +189,25 @@ public class TestApplicationACLs extends ParameterizedSchedulerTestBase {
}
@Override
- protected void configureFairScheduler(YarnConfiguration conf)
- throws IOException {
+ protected void configureFairScheduler(YarnConfiguration configuration) {
final String testDir = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
- PrintWriter out = new PrintWriter(new FileWriter(allocFile));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\" >");
- out.println(" <queue name=\"default\">");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("<queuePlacementPolicy>");
- out.println(" <rule name=\"specified\" create=\"false\" />");
- out.println(" <rule name=\"reject\" />");
- out.println("</queuePlacementPolicy>");
- out.println("</allocations>");
- out.close();
- conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("default").build())
+ .build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.SPECIFIED)
+ .create(false))
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.REJECT)))
+ .writeToFile(allocFile);
+
+ configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
index 9c138d1..5dbb3de 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/ReservationSystemTestUtil.java
@@ -24,9 +24,7 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Collections;
import java.util.Map;
import java.util.Random;
@@ -56,6 +54,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.MultiNodeSortingManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.ClientToAMTokenSecretManagerInRM;
import org.apache.hadoop.yarn.server.resourcemanager.security.NMTokenSecretManagerInRM;
@@ -104,62 +107,46 @@ public class ReservationSystemTestUtil {
.assertTrue(plan.getSharingPolicy() instanceof CapacityOverTimePolicy);
}
- public static void setupFSAllocationFile(String allocationFile)
- throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(allocationFile));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<weight>1</weight>");
- out.println("</queue>");
- out.println("<queue name=\"a\">");
- out.println("<weight>1</weight>");
- out.println("<queue name=\"a1\">");
- out.println("<weight>3</weight>");
- out.println("</queue>");
- out.println("<queue name=\"a2\">");
- out.println("<weight>7</weight>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"dedicated\">");
- out.println("<reservation></reservation>");
- out.println("<weight>8</weight>");
- out.println("</queue>");
- out.println(
- "<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ public static void setupFSAllocationFile(String allocationFile) {
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("default")
+ .weight(1).build())
+ .addQueue(new AllocationFileQueue.Builder("a")
+ .weight(1)
+ .subQueue(new AllocationFileQueue.Builder("a1")
+ .weight(3).build())
+ .subQueue(new AllocationFileQueue.Builder("a2")
+ .weight(7).build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("dedicated")
+ .weight(8)
+ .reservation()
+ .build())
+ .writeToFile(allocationFile);
}
- public static void updateFSAllocationFile(String allocationFile)
- throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(allocationFile));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("<weight>5</weight>");
- out.println("</queue>");
- out.println("<queue name=\"a\">");
- out.println("<weight>5</weight>");
- out.println("<queue name=\"a1\">");
- out.println("<weight>3</weight>");
- out.println("</queue>");
- out.println("<queue name=\"a2\">");
- out.println("<weight>7</weight>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"dedicated\">");
- out.println("<reservation></reservation>");
- out.println("<weight>10</weight>");
- out.println("</queue>");
- out.println("<queue name=\"reservation\">");
- out.println("<reservation></reservation>");
- out.println("<weight>80</weight>");
- out.println("</queue>");
- out.println(
- "<defaultQueueSchedulingPolicy>drf</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ public static void updateFSAllocationFile(String allocationFile) {
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("default")
+ .weight(5).build())
+ .addQueue(new AllocationFileQueue.Builder("a")
+ .weight(5)
+ .subQueue(new AllocationFileQueue.Builder("a1")
+ .weight(3).build())
+ .subQueue(new AllocationFileQueue.Builder("a2")
+ .weight(7).build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("dedicated")
+ .weight(10)
+ .reservation()
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("reservation")
+ .weight(80)
+ .reservation()
+ .build())
+ .writeToFile(allocationFile);
}
public static FairScheduler setupFairScheduler(RMContext rmContext,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
index a02ef55..0650027 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAllocationFileLoaderService.java
@@ -34,24 +34,26 @@ import org.apache.hadoop.yarn.server.resourcemanager.placement.PrimaryGroupPlace
import org.apache.hadoop.yarn.server.resourcemanager.placement.SpecifiedPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.placement.UserPlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueuePlacementRule;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.UserSettings;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.apache.hadoop.yarn.util.resource.CustomResourceTypesConfigurationProvider;
import org.apache.hadoop.yarn.util.resource.Resources;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
import java.net.URISyntaxException;
import java.net.URL;
-import java.nio.charset.StandardCharsets;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -97,6 +99,11 @@ public class TestAllocationFileLoaderService {
when(scheduler.getRMContext()).thenReturn(rmContext);
}
+ @After
+ public void teardown() {
+ new File(ALLOC_FILE).delete();
+ }
+
@Test
public void testGetAllocationFileFromFileSystem()
throws IOException, URISyntaxException {
@@ -152,18 +159,14 @@ public class TestAllocationFileLoaderService {
@Test (timeout = 10000)
public void testReload() throws Exception {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\">");
- out.println(" <maxRunningApps>1</maxRunningApps>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\" />");
- out.println(" <queuePlacementPolicy>");
- out.println(" <rule name='default' />");
- out.println(" </queuePlacementPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxRunningApps(1).build())
+ .addQueue(new AllocationFileQueue.Builder("queueB").build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.DEFAULT)))
+ .writeToFile(ALLOC_FILE);
ControlledClock clock = new ControlledClock();
clock.setTime(0);
@@ -195,20 +198,17 @@ public class TestAllocationFileLoaderService {
confHolder.allocConf = null;
// Modify file and advance the clock
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueB\">");
- out.println(" <maxRunningApps>3</maxRunningApps>");
- out.println(" </queue>");
- out.println(" <queuePlacementPolicy>");
- out.println(" <rule name='specified' />");
- out.println(" <rule name='nestedUserQueue' >");
- out.println(" <rule name='primaryGroup' />");
- out.println(" </rule>");
- out.println(" </queuePlacementPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxRunningApps(3).build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.SPECIFIED))
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.NESTED)
+ .addNestedRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.PRIMARY_GROUP))))
+ .writeToFile(ALLOC_FILE);
clock.tickMsec(System.currentTimeMillis()
+ AllocationFileLoaderService.ALLOC_RELOAD_WAIT_MS + 10000);
@@ -242,57 +242,56 @@ public class TestAllocationFileLoaderService {
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
- AllocationFileWriter
- .create()
+ AllocationFileWriter.create()
// Give queue A a minimum of 1024 M
- .queue("queueA")
- .minResources("1024mb,0vcores")
- .maxResources("2048mb,10vcores")
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("1024mb,0vcores")
+ .maxResources("2048mb,10vcores")
+ .build())
// Give queue B a minimum of 2048 M
- .queue("queueB")
+ .addQueue(new AllocationFileQueue.Builder("queueB")
.minResources("2048mb,0vcores")
.maxResources("5120mb,110vcores")
.aclAdministerApps("alice,bob admins")
.schedulingPolicy("fair")
- .buildQueue()
+ .build())
// Give queue C no minimum
- .queue("queueC")
- .minResources("5120mb,0vcores")
- .aclSubmitApps("alice,bob admins")
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .minResources("5120mb,0vcores")
+ .aclSubmitApps("alice,bob admins")
+ .build())
// Give queue D a limit of 3 running apps and 0.4f maxAMShare
- .queue("queueD")
- .maxRunningApps(3)
- .maxAMShare(0.4)
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueD")
+ .maxRunningApps(3)
+ .maxAMShare(0.4)
+ .build())
// Give queue E a preemption timeout of one minute
- .queue("queueE")
- .minSharePreemptionTimeout(60)
- .buildQueue()
+ .addQueue(new AllocationFileQueue.Builder("queueE")
+ .minSharePreemptionTimeout(60)
+ .build())
// Make queue F a parent queue without configured leaf queues
// using the 'type' attribute
- .queue("queueF")
- .parent(true)
- .maxChildResources("2048mb,64vcores")
- .buildQueue()
- .queue("queueG")
- .maxChildResources("2048mb,64vcores")
- .fairSharePreemptionTimeout(120)
- .minSharePreemptionTimeout(50)
- .fairSharePreemptionThreshold(0.6)
- .maxContainerAllocation(
+ .addQueue(new AllocationFileQueue.Builder("queueF")
+ .parent(true)
+ .maxChildResources("2048mb,64vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueG")
+ .maxChildResources("2048mb,64vcores")
+ .fairSharePreemptionTimeout(120)
+ .minSharePreemptionTimeout(50)
+ .fairSharePreemptionThreshold(0.6)
+ .maxContainerAllocation(
"vcores=16, memory-mb=512, " + A_CUSTOM_RESOURCE + "=10")
// Create hierarchical queues G,H, with different min/fair
// share preemption timeouts and preemption thresholds.
// Also add a child default to make sure it doesn't impact queue H.
- .subQueue("queueH")
- .fairSharePreemptionTimeout(180)
- .minSharePreemptionTimeout(40)
- .fairSharePreemptionThreshold(0.7)
- .maxContainerAllocation("1024mb,8vcores")
- .buildSubQueue()
- .buildQueue()
+ .subQueue(new AllocationFileQueue.Builder("queueH")
+ .fairSharePreemptionTimeout(180)
+ .minSharePreemptionTimeout(40)
+ .fairSharePreemptionThreshold(0.7)
+ .maxContainerAllocation("1024mb,8vcores")
+ .build())
+ .build())
// Set default limit of apps per queue to 15
.queueMaxAppsDefault(15)
// Set default limit of max resource per queue to 4G and 100 cores
@@ -308,11 +307,11 @@ public class TestAllocationFileLoaderService {
// Set default fair share preemption threshold to 0.4
.defaultFairSharePreemptionThreshold(0.4)
// Set default scheduling policy to DRF
- .defaultQueueSchedulingPolicy("drf")
+ .drfDefaultQueueSchedulingPolicy()
// Give user1 a limit of 10 jobs
- .userSettings("user1")
+ .userSettings(new UserSettings.Builder("user1")
.maxRunningApps(10)
- .build()
+ .build())
.writeToFile(ALLOC_FILE);
allocLoader.init(conf);
@@ -488,48 +487,51 @@ public class TestAllocationFileLoaderService {
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- // Give queue A a minimum of 1024 M
- out.println("<pool name=\"queueA\">");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</pool>");
- // Give queue B a minimum of 2048 M
- out.println("<pool name=\"queueB\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("<aclAdministerApps>alice,bob admins</aclAdministerApps>");
- out.println("</pool>");
- // Give queue C no minimum
- out.println("<pool name=\"queueC\">");
- out.println("<aclSubmitApps>alice,bob admins</aclSubmitApps>");
- out.println("</pool>");
- // Give queue D a limit of 3 running apps
- out.println("<pool name=\"queueD\">");
- out.println("<maxRunningApps>3</maxRunningApps>");
- out.println("</pool>");
- // Give queue E a preemption timeout of one minute and 0.3f threshold
- out.println("<pool name=\"queueE\">");
- out.println("<minSharePreemptionTimeout>60</minSharePreemptionTimeout>");
- out.println("<fairSharePreemptionThreshold>0.3</fairSharePreemptionThreshold>");
- out.println("</pool>");
- // Set default limit of apps per queue to 15
- out.println("<queueMaxAppsDefault>15</queueMaxAppsDefault>");
- // Set default limit of apps per user to 5
- out.println("<userMaxAppsDefault>5</userMaxAppsDefault>");
- // Give user1 a limit of 10 jobs
- out.println("<user name=\"user1\">");
- out.println("<maxRunningApps>10</maxRunningApps>");
- out.println("</user>");
- // Set default min share preemption timeout to 2 minutes
- out.println("<defaultMinSharePreemptionTimeout>120"
- + "</defaultMinSharePreemptionTimeout>");
- // Set fair share preemption timeout to 5 minutes
- out.println("<fairSharePreemptionTimeout>300</fairSharePreemptionTimeout>");
- // Set default fair share preemption threshold to 0.6f
- out.println("<defaultFairSharePreemptionThreshold>0.6</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .useLegacyTagNameForQueues()
+ // Give queue A a minimum of 1024 M
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("1024mb,0vcores")
+ .build())
+ // Give queue B a minimum of 2048 M
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("2048mb,0vcores")
+ .aclAdministerApps("alice,bob admins")
+ .build())
+ // Give queue C no minimum
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .aclAdministerApps("alice,bob admins")
+ .build())
+ // Give queue D a limit of 3 running apps
+ .addQueue(new AllocationFileQueue.Builder("queueD")
+ .maxRunningApps(3)
+ .build())
+ // Give queue E a preemption timeout of one minute and 0.3f threshold
+ .addQueue(new AllocationFileQueue.Builder("queueE")
+ .minSharePreemptionTimeout(60)
+ .fairSharePreemptionThreshold(0.3)
+ .build())
+ // Set default limit of apps per queue to 15
+ .queueMaxAppsDefault(15)
+ // Set default limit of apps per user to 5
+ .userMaxAppsDefault(5)
+ // Set default limit of max resource per queue to 4G and 100 cores
+ .queueMaxResourcesDefault("4096mb,100vcores")
+ // Set default limit of AMResourceShare to 0.5f
+ .queueMaxAMShareDefault(0.5)
+ // Set default min share preemption timeout to 2 minutes
+ .defaultMinSharePreemptionTimeout(120)
+ // Set default fair share preemption timeout to 5 minutes
+ .defaultFairSharePreemptionTimeout(300)
+ // Set default fair share preemption threshold to 0.6
+ .defaultFairSharePreemptionThreshold(0.6)
+ // Set default scheduling policy to DRF
+ .drfDefaultQueueSchedulingPolicy()
+ // Give user1 a limit of 10 jobs
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(10)
+ .build())
+ .writeToFile(ALLOC_FILE);
allocLoader.init(conf);
ReloadListener confHolder = new ReloadListener();
@@ -602,11 +604,7 @@ public class TestAllocationFileLoaderService {
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
conf.setBoolean(FairSchedulerConfiguration.USER_AS_DEFAULT_QUEUE, false);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create().writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -632,15 +630,10 @@ public class TestAllocationFileLoaderService {
public void testQueueAlongsideRoot() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("</queue>");
- out.println("<queue name=\"other\">");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root").build())
+ .addQueue(new AllocationFileQueue.Builder("other").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -658,13 +651,9 @@ public class TestAllocationFileLoaderService {
public void testQueueNameContainingPeriods() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"parent1.child1\">");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent1.child").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -682,13 +671,9 @@ public class TestAllocationFileLoaderService {
public void testQueueNameContainingOnlyWhitespace() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\" \">");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder(" ").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -702,15 +687,12 @@ public class TestAllocationFileLoaderService {
public void testParentTagWithReservation() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"parent\" type=\"parent\">");
- out.println("<reservation>");
- out.println("</reservation>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .parent(true)
+ .reservation()
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -731,17 +713,13 @@ public class TestAllocationFileLoaderService {
public void testParentWithReservation() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"parent\">");
- out.println("<reservation>");
- out.println("</reservation>");
- out.println(" <queue name=\"child\">");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .parent(true)
+ .subQueue(new AllocationFileQueue.Builder("child").build())
+ .reservation()
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -762,15 +740,12 @@ public class TestAllocationFileLoaderService {
public void testParentTagWithChild() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"parent\" type=\"parent\">");
- out.println(" <queue name=\"child\">");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .parent(true)
+ .subQueue(new AllocationFileQueue.Builder("child").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -794,14 +769,9 @@ public class TestAllocationFileLoaderService {
public void testQueueNameContainingNBWhitespace() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new OutputStreamWriter(
- new FileOutputStream(ALLOC_FILE), StandardCharsets.UTF_8));
- out.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"\u00a0\">");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("\u00a0").build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -818,13 +788,9 @@ public class TestAllocationFileLoaderService {
public void testDefaultQueueSchedulingModeIsFIFO() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<defaultQueueSchedulingPolicy>fifo" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .fifoDefaultQueueSchedulingPolicy()
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -838,19 +804,14 @@ public class TestAllocationFileLoaderService {
public void testReservableQueue() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"reservable\">");
- out.println("<reservation>");
- out.println("</reservation>");
- out.println("</queue>");
- out.println("<queue name=\"other\">");
- out.println("</queue>");
- out.println("<reservation-agent>DummyAgentName</reservation-agent>");
- out.println("<reservation-policy>AnyAdmissionPolicy</reservation-policy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("reservable")
+ .reservation()
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("other").build())
+ .reservationAgent("DummyAgentName")
+ .reservationPolicy("AnyAdmissionPolicy")
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
@@ -900,15 +861,12 @@ public class TestAllocationFileLoaderService {
throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"notboth\" type=\"parent\" >");
- out.println("<reservation>");
- out.println("</reservation>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("notboth")
+ .parent(true)
+ .reservation()
+ .build())
+ .writeToFile(ALLOC_FILE);
AllocationFileLoaderService allocLoader =
new AllocationFileLoaderService(scheduler);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index b7ef471..9777e14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -24,9 +24,7 @@ import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -43,6 +41,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptA
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -164,17 +168,14 @@ public class TestAppRunnability extends FairSchedulerTestBase {
}
@Test
- public void testDontAllowUndeclaredPools() throws Exception {
+ public void testDontAllowUndeclaredPools() {
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"jerry\">");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("jerry").build())
+ .writeToFile(ALLOC_FILE);
+
// Restarting resource manager since the file location and content is
// changed.
resourceManager.stop();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java
index 9fbdf89..def3749 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestApplicationMasterServiceWithFS.java
@@ -19,9 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -38,6 +35,12 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRMAppSubmitter;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -58,7 +61,7 @@ public class TestApplicationMasterServiceWithFS {
private static YarnConfiguration configuration;
@BeforeClass
- public static void setup() throws IOException {
+ public static void setup() {
String allocFile =
GenericTestUtils.getTestDir(TEST_FOLDER).getAbsolutePath();
@@ -67,21 +70,15 @@ public class TestApplicationMasterServiceWithFS {
ResourceScheduler.class);
configuration.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
- PrintWriter out = new PrintWriter(new FileWriter(allocFile));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\">");
- out.println(
- " <maxContainerAllocation>2048 mb 1 vcores</maxContainerAllocation>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\">");
- out.println(
- " <maxContainerAllocation>3072 mb 1 vcores</maxContainerAllocation>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueC\">");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxContainerAllocation("2048 mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxContainerAllocation("3072 mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueC").build())
+ .writeToFile(allocFile);
}
@AfterClass
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
index 9665f9a..9aa9d5c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSAppStarvation.java
@@ -22,6 +22,11 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.junit.After;
@@ -33,12 +38,10 @@ import org.junit.Before;
import org.junit.Test;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
/**
- * Test class to verify identification of app starvation
+ * Test class to verify identification of app starvation.
*/
public class TestFSAppStarvation extends FairSchedulerTestBase {
@@ -186,53 +189,41 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
* 3. Add two nodes to the cluster
* 4. Submit an app that uses up all resources on the cluster
*/
- private void setupStarvedCluster() throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
-
- // Default queue
- out.println("<queue name=\"default\">");
- out.println("</queue>");
-
- // Queue with preemption disabled
- out.println("<queue name=\"no-preemption\">");
- out.println("<fairSharePreemptionThreshold>0" +
- "</fairSharePreemptionThreshold>");
- out.println("</queue>");
-
- // Queue with minshare preemption enabled
- out.println("<queue name=\"minshare\">");
- out.println("<fairSharePreemptionThreshold>0" +
- "</fairSharePreemptionThreshold>");
- out.println("<minSharePreemptionTimeout>0" +
- "</minSharePreemptionTimeout>");
- out.println("<minResources>2048mb,2vcores</minResources>");
- out.println("</queue>");
-
- // FAIR queue with fairshare preemption enabled
- out.println("<queue name=\"fairshare\">");
- out.println("<fairSharePreemptionThreshold>1" +
- "</fairSharePreemptionThreshold>");
- out.println("<fairSharePreemptionTimeout>0" +
- "</fairSharePreemptionTimeout>");
- out.println("<schedulingPolicy>fair</schedulingPolicy>");
- addChildQueue(out, "fair");
- out.println("</queue>");
-
- // DRF queue with fairshare preemption enabled
- out.println("<queue name=\"drf\">");
- out.println("<fairSharePreemptionThreshold>1" +
- "</fairSharePreemptionThreshold>");
- out.println("<fairSharePreemptionTimeout>0" +
- "</fairSharePreemptionTimeout>");
- out.println("<schedulingPolicy>drf</schedulingPolicy>");
- addChildQueue(out, "drf");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ private void setupStarvedCluster() {
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ // Default queue
+ .addQueue(new AllocationFileQueue.Builder("default").build())
+ // Queue with preemption disabled
+ .addQueue(new AllocationFileQueue.Builder("no-preemption")
+ .fairSharePreemptionThreshold(0).build())
+ // Queue with minshare preemption enabled
+ .addQueue(new AllocationFileQueue.Builder("minshare")
+ .fairSharePreemptionThreshold(0)
+ .minSharePreemptionTimeout(0)
+ .minResources("2048mb,2vcores")
+ .build())
+ // FAIR queue with fairshare preemption enabled
+ .addQueue(new AllocationFileQueue.Builder("fairshare")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("fair").build())
+ .build())
+ // DRF queue with fairshare preemption enabled
+ .addQueue(new AllocationFileQueue.Builder("drf")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .schedulingPolicy("drf").build())
+ .build())
+ .writeToFile(ALLOC_FILE.getAbsolutePath());
assertTrue("Allocation file does not exist, not running the test",
ALLOC_FILE.exists());
@@ -258,17 +249,6 @@ public class TestFSAppStarvation extends FairSchedulerTestBase {
assertEquals(8, scheduler.getSchedulerApp(app).getLiveContainers().size());
}
- private void addChildQueue(PrintWriter out, String policy) {
- // Child queue under fairshare with same settings
- out.println("<queue name=\"child\">");
- out.println("<fairSharePreemptionThreshold>1" +
- "</fairSharePreemptionThreshold>");
- out.println("<fairSharePreemptionTimeout>0" +
- "</fairSharePreemptionTimeout>");
- out.println("<schedulingPolicy>" + policy + "</schedulingPolicy>");
- out.println("</queue>");
- }
-
private void submitAppsToEachLeafQueue() {
for (String queue : QUEUES) {
createSchedulingRequest(1024, 1, "root." + queue, "user", 1);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index 9c47b6b..6ecf809 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -24,9 +24,7 @@ import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -49,6 +47,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -110,17 +110,14 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
}
@Test (timeout = 5000)
- public void test() throws Exception {
+ public void test() {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queueMaxAMShareDefault>" + MAX_AM_SHARE +
- "</queueMaxAMShareDefault>");
- out.println("<queue name=\"queueA\"></queue>");
- out.println("<queue name=\"queueB\"></queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .queueMaxAMShareDefault(MAX_AM_SHARE)
+ .addQueue(new AllocationFileQueue.Builder("queueA").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB").build())
+ .writeToFile(ALLOC_FILE);
resourceManager = new MockRM(conf);
resourceManager.start();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index 4f51120..e6fe83a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -90,6 +90,16 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerEx
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueuePlacementPolicy;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueuePlacementRule;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.UserSettings;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -101,9 +111,7 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
-import org.xml.sax.SAXException;
-import javax.xml.parsers.ParserConfigurationException;
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
@@ -340,27 +348,18 @@ public class TestFairScheduler extends FairSchedulerTestBase {
int tooHighQueueAllocation = RM_SCHEDULER_MAXIMUM_ALLOCATION_MB_VALUE +1;
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\">");
- out.println(
- " <maxContainerAllocation>512 mb 1 vcores</maxContainerAllocation>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\">");
- out.println(" </queue>");
- out.println(" <queue name=\"queueC\">");
- out.println(
- " <maxContainerAllocation>2048 mb 3 vcores</maxContainerAllocation>");
- out.println(" <queue name=\"queueD\">");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueE\">");
- out.println(" <maxContainerAllocation>" + tooHighQueueAllocation
- + " mb 1 vcores</maxContainerAllocation>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxContainerAllocation("512 mb 1 vcores").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB").build())
+ .addQueue(new AllocationFileQueue.Builder("queueC")
+ .maxContainerAllocation("2048 mb 3 vcores")
+ .subQueue(new AllocationFileQueue.Builder("queueD").build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueE")
+ .maxContainerAllocation(tooHighQueueAllocation + " mb 1 vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -396,17 +395,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
int queueMaxAllocation = 4096;
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\">");
- out.println(" <maxContainerAllocation>" + queueMaxAllocation
- + " mb 1 vcores" + "</maxContainerAllocation>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\">");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxContainerAllocation(queueMaxAllocation + " mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB").build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -454,19 +448,17 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// set queueA and queueB maxResources,
// the sum of queueA and queueB maxResources is more than
// Integer.MAX_VALUE.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<maxResources>1073741824 mb 1000 vcores</maxResources>");
- out.println("<weight>.25</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<maxResources>1073741824 mb 1000 vcores</maxResources>");
- out.println("<weight>.75</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxResources("1073741824 mb 1000 vcores")
+ .weight(.25f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxResources("1073741824 mb 1000 vcores")
+ .weight(.75f)
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -508,18 +500,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testFairShareWithLowMaxResources() throws IOException {
PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\">");
- out.println(" <maxResources>1024 mb 1 vcores</maxResources>");
- out.println(" <weight>0.75</weight>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\">");
- out.println(" <maxResources>3072 mb 3 vcores</maxResources>");
- out.println(" <weight>0.25</weight>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .maxResources("1024 mb 1 vcores")
+ .weight(.75f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .maxResources("3072 mb 3 vcores")
+ .weight(.25f)
+ .build())
+ .writeToFile(ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.init(conf);
@@ -578,15 +568,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
*/
@Test
public void testChildMaxResources() throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\" type=\"parent\">");
- out.println(" <maxChildResources>2048mb,2vcores</maxChildResources>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .parent(true)
+ .maxChildResources("2048mb,2vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
scheduler.init(conf);
@@ -635,14 +622,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("App 2 is not running with the correct number of containers",
2, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\" type=\"parent\">");
- out.println(" <maxChildResources>3072mb,3vcores</maxChildResources>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .parent(true)
+ .maxChildResources("3072mb,3vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
@@ -661,14 +646,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("App 2 is not running with the correct number of containers",
3, scheduler.getSchedulerApp(attId2).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println(" <queue name=\"queueA\" type=\"parent\">");
- out.println(" <maxChildResources>1024mb,1vcores</maxChildResources>");
- out.println(" </queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .parent(true)
+ .maxChildResources("1024mb,1vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
//ensure that a 7th node heartbeat does not allocate more containers
scheduler.handle(nodeEvent);
@@ -698,17 +681,13 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testFairShareWithZeroWeight() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight zero.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>0.0</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>0.0</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0.0f).build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .weight(0.0f).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -746,26 +725,23 @@ public class TestFairScheduler extends FairSchedulerTestBase {
@Test
public void testComputeMaxAMResource() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueFSZeroWithMax\">");
- out.println("<weight>0</weight>");
- out.println("<maxAMShare>0.5</maxAMShare>");
- out.println("<maxResources>4096 mb 4 vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueFSZeroWithAVL\">");
- out.println("<weight>0.0</weight>");
- out.println("<maxAMShare>0.5</maxAMShare>");
- out.println("</queue>");
- out.println("<queue name=\"queueFSNonZero\">");
- out.println("<weight>1</weight>");
- out.println("<maxAMShare>0.5</maxAMShare>");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithMax")
+ .weight(0)
+ .maxAMShare(0.5)
+ .maxResources("4096 mb 4 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueFSZeroWithAVL")
+ .weight(0.0f)
+ .maxAMShare(0.5)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueFSNonZero")
+ .weight(1)
+ .maxAMShare(0.5)
+ .build())
+ .drfDefaultQueueSchedulingPolicy()
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -880,19 +856,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight zero.
// set queueA and queueB minResources 1.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<minResources>1 mb 1 vcores</minResources>");
- out.println("<weight>0.0</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<minResources>1 mb 1 vcores</minResources>");
- out.println("<weight>0.0</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0)
+ .minResources("1 mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("1 mb 1 vcores")
+ .weight(0.0f)
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -930,19 +903,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set queueA and queueB weight 0.5.
// set queueA and queueB minResources 1024.
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<minResources>1024 mb 1 vcores</minResources>");
- out.println("<weight>0.5</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<minResources>1024 mb 1 vcores</minResources>");
- out.println("<weight>0.5</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0.5f)
+ .minResources("1024 mb 1 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .weight(0.5f)
+ .minResources("1024 mb 1 vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -978,17 +948,14 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testQueueInfo() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<weight>.25</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<weight>.75</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .weight(0.25f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .weight(0.75f)
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1504,19 +1471,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testContainerReservationAttemptExceedingQueueMax()
throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<queue name=\"queue1\">");
- out.println("<maxResources>2048mb,5vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queue2\">");
- out.println("<maxResources>2048mb,10vcores</maxResources>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .maxResources("2048mb,5vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queue2")
+ .maxResources("2048mb,10vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1566,22 +1529,20 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testRequestAMResourceInZeroFairShareQueue() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<weight>0.0</weight>");
- out.println("<maxResources>4096mb,10vcores</maxResources>");
- out.println("<maxAMShare>0.5</maxAMShare>");
- out.println("</queue>");
- out.println("<queue name=\"queue2\">");
- out.println("<weight>2.0</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queue3\">");
- out.println("<weight>0.000001</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .weight(0)
+ .maxAMShare(0.5)
+ .maxResources("4096mb,10vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue2")
+ .weight(2.0f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue3")
+ .weight(0.000001f)
+ .build())
+ .drfDefaultQueueSchedulingPolicy()
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1639,19 +1600,15 @@ public class TestFairScheduler extends FairSchedulerTestBase {
@Test (timeout = 500000)
public void testContainerReservationNotExceedingQueueMax() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<queue name=\"queue1\">");
- out.println("<maxResources>3072mb,10vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queue2\">");
- out.println("<maxResources>2048mb,10vcores</maxResources>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .maxResources("3072mb,10vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queue2")
+ .maxResources("2048mb,10vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1700,19 +1657,14 @@ public class TestFairScheduler extends FairSchedulerTestBase {
getCurrentReservation().getMemorySize());
// Now reduce max Resources of queue1 down to 2048
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<queue name=\"queue1\">");
- out.println("<maxResources>2048mb,10vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queue2\">");
- out.println("<maxResources>2048mb,10vcores</maxResources>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .maxResources("2048mb,10vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queue2")
+ .maxResources("2048mb,10vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -1733,13 +1685,9 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testReservationThresholdGatesReservations() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .writeToFile(ALLOC_FILE);
// Set threshold to 2 * 1024 ==> 2048 MB & 2 * 1 ==> 2 vcores (test will
// use vcores)
@@ -1885,17 +1833,13 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testFairShareWithMinAlloc() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("1024mb,0vcores").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("2048mb,0vcores")
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -1930,21 +1874,21 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testFairShareAndWeightsInNestedUserQueueRule() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
-
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"parentq\" type=\"parent\">");
- out.println("<minResources>1024mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queuePlacementPolicy>");
- out.println("<rule name=\"nestedUserQueue\">");
- out.println(" <rule name=\"specified\" create=\"false\" />");
- out.println("</rule>");
- out.println("<rule name=\"default\" />");
- out.println("</queuePlacementPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parentq")
+ .parent(true)
+ .minResources("1024mb,0vcores")
+ .build())
+ .queuePlacementPolicy(new AllocationFileQueuePlacementPolicy()
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.NESTED)
+ .addNestedRule(
+ new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.SPECIFIED)
+ .create(false)))
+ .addRule(new AllocationFileQueuePlacementRule(
+ AllocationFileQueuePlacementRule.RuleName.DEFAULT)))
+ .writeToFile(ALLOC_FILE);
RMApp rmApp1 = new MockRMApp(0, 0, RMAppState.NEW);
RMApp rmApp2 = new MockRMApp(1, 1, RMAppState.NEW);
@@ -1991,21 +1935,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testSteadyFairShareWithReloadAndNodeAddRemove() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
- out.println("<queue name=\"root\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <weight>1</weight>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <weight>1</weight>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .fairDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .weight(1).build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .weight(1).build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2033,24 +1972,19 @@ public class TestFairScheduler extends FairSchedulerTestBase {
.getSteadyFairShare().getMemorySize());
// Reload the allocation configuration file
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
- out.println("<queue name=\"root\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <weight>1</weight>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <weight>2</weight>");
- out.println(" </queue>");
- out.println(" <queue name=\"child3\">");
- out.println(" <weight>2</weight>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .fairDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .weight(1).build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .weight(2).build())
+ .subQueue(new AllocationFileQueue.Builder("child3")
+ .weight(2).build())
+ .build())
+ .writeToFile(ALLOC_FILE);
+
scheduler.reinitialize(conf, resourceManager.getRMContext());
// The steady fair shares for all queues should be updated
@@ -2170,27 +2104,19 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
@Test
- public void testHierarchicalQueueAllocationFileParsing() throws IOException, SAXException,
- AllocationConfigurationException, ParserConfigurationException {
+ public void testHierarchicalQueueAllocationFileParsing() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("<queue name=\"queueC\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("<queue name=\"queueD\">");
- out.println("<minResources>2048mb,0vcores</minResources>");
- out.println("</queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources("2048mb,0vcores").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("2048mb,0vcores")
+ .subQueue(new AllocationFileQueue.Builder("queueC")
+ .minResources("2048mb,0vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("queueD")
+ .minResources("2048mb,0vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2211,27 +2137,22 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testConfigureRootQueue() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
- out.println("<queue name=\"root\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <minResources>1024mb,1vcores</minResources>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <minResources>1024mb,4vcores</minResources>");
- out.println(" </queue>");
- out.println(" <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
- out.println(" <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
- out.println(" <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
- out.println("</queue>");
- out.println("<defaultFairSharePreemptionTimeout>300</defaultFairSharePreemptionTimeout>");
- out.println("<defaultMinSharePreemptionTimeout>200</defaultMinSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionThreshold>.6</defaultFairSharePreemptionThreshold>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .fairDefaultQueueSchedulingPolicy()
+ .defaultFairSharePreemptionTimeout(300)
+ .defaultMinSharePreemptionTimeout(200)
+ .defaultFairSharePreemptionThreshold(.6)
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .fairSharePreemptionTimeout(100)
+ .fairSharePreemptionThreshold(.5)
+ .minSharePreemptionTimeout(120)
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .minResources("1024mb,1vcores").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .minResources("1024mb,4vcores").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2286,14 +2207,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Set max running apps
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<user name=\"user1\">");
- out.println("<maxRunningApps>1</maxRunningApps>");
- out.println("</user>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2338,65 +2255,57 @@ public class TestFairScheduler extends FairSchedulerTestBase {
@Test (timeout = 5000)
public void testIncreaseQueueMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = "<?xml version=\"1.0\"?>" +
- "<allocations>" +
- "<queue name=\"root\">" +
- "<queue name=\"queue1\">" +
- "<maxRunningApps>1</maxRunningApps>" +
- "</queue>" +
- "</queue>" +
- "</allocations>";
-
- String allocAfter = "<?xml version=\"1.0\"?>" +
- "<allocations>" +
- "<queue name=\"root\">" +
- "<queue name=\"queue1\">" +
- "<maxRunningApps>3</maxRunningApps>" +
- "</queue>" +
- "</queue>" +
- "</allocations>";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(1)
+ .build())
+ .build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(3)
+ .build())
+ .build());
testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
@Test (timeout = 5000)
public void testIncreaseUserMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = "<?xml version=\"1.0\"?>"+
- "<allocations>"+
- "<queue name=\"root\">"+
- "<queue name=\"queue1\">"+
- "<maxRunningApps>10</maxRunningApps>"+
- "</queue>"+
- "</queue>"+
- "<user name=\"user1\">"+
- "<maxRunningApps>1</maxRunningApps>"+
- "</user>"+
- "</allocations>";
-
- String allocAfter = "<?xml version=\"1.0\"?>"+
- "<allocations>"+
- "<queue name=\"root\">"+
- "<queue name=\"queue1\">"+
- "<maxRunningApps>10</maxRunningApps>"+
- "</queue>"+
- "</queue>"+
- "<user name=\"user1\">"+
- "<maxRunningApps>3</maxRunningApps>"+
- "</user>"+
- "</allocations>";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(3).build());
testIncreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
- private void testIncreaseQueueSettingOnTheFlyInternal(String allocBefore,
- String allocAfter) throws Exception {
+ private void testIncreaseQueueSettingOnTheFlyInternal(
+ AllocationFileWriter allocBefore,
+ AllocationFileWriter allocAfter) throws Exception {
// Set max running apps
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocBefore);
- out.close();
-
+ allocBefore.writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
scheduler.reinitialize(conf, resourceManager.getRMContext());
@@ -2444,9 +2353,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// App 4 should not be running
assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocAfter);
- out.close();
+ allocAfter.writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
@@ -2481,64 +2388,56 @@ public class TestFairScheduler extends FairSchedulerTestBase {
@Test (timeout = 5000)
public void testDecreaseQueueMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = "<?xml version=\"1.0\"?>" +
- "<allocations>" +
- "<queue name=\"root\">" +
- "<queue name=\"queue1\">" +
- "<maxRunningApps>3</maxRunningApps>" +
- "</queue>" +
- "</queue>" +
- "</allocations>";
-
- String allocAfter = "<?xml version=\"1.0\"?>" +
- "<allocations>" +
- "<queue name=\"root\">" +
- "<queue name=\"queue1\">" +
- "<maxRunningApps>1</maxRunningApps>" +
- "</queue>" +
- "</queue>" +
- "</allocations>";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(3)
+ .build())
+ .build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(1)
+ .build())
+ .build());
testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
@Test (timeout = 5000)
public void testDecreaseUserMaxRunningAppsOnTheFly() throws Exception {
- String allocBefore = "<?xml version=\"1.0\"?>"+
- "<allocations>"+
- "<queue name=\"root\">"+
- "<queue name=\"queue1\">"+
- "<maxRunningApps>10</maxRunningApps>"+
- "</queue>"+
- "</queue>"+
- "<user name=\"user1\">"+
- "<maxRunningApps>3</maxRunningApps>"+
- "</user>"+
- "</allocations>";
-
- String allocAfter = "<?xml version=\"1.0\"?>"+
- "<allocations>"+
- "<queue name=\"root\">"+
- "<queue name=\"queue1\">"+
- "<maxRunningApps>10</maxRunningApps>"+
- "</queue>"+
- "</queue>"+
- "<user name=\"user1\">"+
- "<maxRunningApps>1</maxRunningApps>"+
- "</user>"+
- "</allocations>";
+ AllocationFileWriter allocBefore = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(3).build());
+
+ AllocationFileWriter allocAfter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(
+ new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(10)
+ .build())
+ .build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build());
testDecreaseQueueSettingOnTheFlyInternal(allocBefore, allocAfter);
}
- private void testDecreaseQueueSettingOnTheFlyInternal(String allocBefore,
- String allocAfter) throws Exception {
+ private void testDecreaseQueueSettingOnTheFlyInternal(
+ AllocationFileWriter allocBefore,
+ AllocationFileWriter allocAfter) throws Exception {
// Set max running apps
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
-
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocBefore);
- out.close();
+ allocBefore.writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -2587,9 +2486,7 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// App 4 should not be running
assertEquals(0, scheduler.getSchedulerApp(attId4).getLiveContainers().size());
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println(allocAfter);
- out.close();
+ allocAfter.writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, resourceManager.getRMContext());
scheduler.update();
@@ -2702,19 +2599,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Set acl's
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps> </aclAdministerApps>");
- out.println(" <queue name=\"queue1\">");
- out.println(" <aclSubmitApps>norealuserhasthisname</aclSubmitApps>");
- out.println(" <aclAdministerApps>norealuserhasthisname</aclAdministerApps>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .aclSubmitApps("norealuserhasthisname")
+ .aclAdministerApps("norealuserhasthisname")
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3041,19 +2935,17 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testNotAllowSubmitApplication() throws Exception {
// Set acl's
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps> </aclAdministerApps>");
- out.println(" <queue name=\"queue1\">");
- out.println(" <aclSubmitApps>userallow</aclSubmitApps>");
- out.println(" <aclAdministerApps>userallow</aclAdministerApps>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps(" ")
+ .subQueue(new AllocationFileQueue.Builder("queue1")
+ .aclSubmitApps("userallow")
+ .aclAdministerApps("userallow")
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3595,17 +3487,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testUserAndQueueMaxRunningApps() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<maxRunningApps>2</maxRunningApps>");
- out.println("</queue>");
- out.println("<user name=\"user1\">");
- out.println("<maxRunningApps>1</maxRunningApps>");
- out.println("</user>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(2).build())
+ .userSettings(new UserSettings.Builder("user1")
+ .maxRunningApps(1).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3648,14 +3535,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
// Set up a fair scheduler
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<maxAMShare>0.2</maxAMShare>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxAMShare(0.2).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3698,14 +3581,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testQueueMaxAMShare() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<maxAMShare>0.2</maxAMShare>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxAMShare(0.2).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -3928,25 +3807,18 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, 6);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("</queue>");
- out.println("<queue name=\"queue2\">");
- out.println("<maxAMShare>0.4</maxAMShare>");
- out.println("</queue>");
- out.println("<queue name=\"queue3\">");
- out.println("<maxResources>10240 mb 4 vcores</maxResources>");
- out.println("</queue>");
- out.println("<queue name=\"queue4\">");
- out.println("</queue>");
- out.println("<queue name=\"queue5\">");
- out.println("</queue>");
- out.println(
- "<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .fairDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("queue1").build())
+ .addQueue(new AllocationFileQueue.Builder("queue2")
+ .maxAMShare(0.4f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue3")
+ .maxResources("10240 mb 4 vcores")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queue4").build())
+ .addQueue(new AllocationFileQueue.Builder("queue5").build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4089,14 +3961,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testQueueMaxAMShareWithContainerReservation() throws Exception {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 1f);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println("<maxAMShare>0.5</maxAMShare>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxAMShare(0.5).build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4355,19 +4223,16 @@ public class TestFairScheduler extends FairSchedulerTestBase {
ControlledClock clock = new ControlledClock();
scheduler.setClock(clock);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queue1\">");
- out.println(" <maxRunningApps>3</maxRunningApps>");
- out.println(" <queue name=\"sub1\"></queue>");
- out.println(" <queue name=\"sub2\"></queue>");
- out.println(" <queue name=\"sub3\">");
- out.println(" <maxRunningApps>1</maxRunningApps>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queue1")
+ .maxRunningApps(3)
+ .subQueue(new AllocationFileQueue.Builder("sub1").build())
+ .subQueue(new AllocationFileQueue.Builder("sub2").build())
+ .subQueue(new AllocationFileQueue.Builder("sub3")
+ .maxRunningApps(1)
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4481,11 +4346,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
conf.setBoolean(FairSchedulerConfiguration.ALLOW_UNDECLARED_POOLS, false);
// Create an alloc file with no queue placement policy
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
scheduler.start();
@@ -4946,16 +4808,13 @@ public class TestFairScheduler extends FairSchedulerTestBase {
}
@Test
- public void testEmptyQueueNameInConfigFile() throws IOException {
+ public void testEmptyQueueNameInConfigFile() {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
// set empty queue name
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"\">");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("").build())
+ .writeToFile(ALLOC_FILE);
+
try {
scheduler.init(conf);
Assert.fail("scheduler init should fail because" +
@@ -5190,6 +5049,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
new HAServiceProtocol.StateChangeRequestInfo(
HAServiceProtocol.RequestSource.REQUEST_BY_USER);
+ //ensure ALLOC_FILE contains an 'empty' config
+ AllocationFileWriter.create().writeToFile(ALLOC_FILE);
// 1. start a standby RM, file 'ALLOC_FILE' is empty, so there is no queues
MockRM rm1 = new MockRM(conf, null);
rm1.init(conf);
@@ -5197,14 +5058,10 @@ public class TestFairScheduler extends FairSchedulerTestBase {
rm1.getAdminService().transitionToStandby(requestInfo);
// 2. add a new queue "test_queue"
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"test_queue\">");
- out.println(" <maxRunningApps>3</maxRunningApps>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("test_queue")
+ .maxRunningApps(3).build())
+ .writeToFile(ALLOC_FILE);
conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
// 3. start a active RM
@@ -5348,16 +5205,12 @@ public class TestFairScheduler extends FairSchedulerTestBase {
public void testDumpState() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"parent\">");
- out.println(" <queue name=\"child1\">");
- out.println(" <weight>1</weight>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("parent")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .weight(1).build())
+ .build())
+ .writeToFile(ALLOC_FILE);
ControlledClock clock = new ControlledClock();
scheduler.setClock(clock);
@@ -5538,29 +5391,25 @@ public class TestFairScheduler extends FairSchedulerTestBase {
+ "maximum queue resources: .+"));
}
- private void generateAllocationFileWithZeroResource(String resource)
- throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"queueA\">");
-
+ private void generateAllocationFileWithZeroResource(String resource) {
String resources = "";
if (resource.equals(ResourceInformation.MEMORY_URI)) {
resources = "0 mb,2vcores";
} else if (resource.equals(ResourceInformation.VCORES_URI)) {
resources = "10000 mb,0vcores";
}
- out.println("<minResources>" + resources + "</minResources>");
- out.println("<maxResources>" + resources + "</maxResources>");
- out.println("<weight>2.0</weight>");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<minResources>1 mb 1 vcores</minResources>");
- out.println("<weight>0.0</weight>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("queueA")
+ .minResources(resources)
+ .maxResources(resources)
+ .weight(2.0f)
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .minResources("1 mb 1 vcores")
+ .weight(0.0f)
+ .build())
+ .writeToFile(ALLOC_FILE);
}
@Test
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
index a79aacc..5ab920f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerFairShare.java
@@ -21,9 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import static org.junit.Assert.assertEquals;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Collection;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
@@ -33,6 +31,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptS
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
@@ -57,34 +60,33 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
conf = null;
}
- private void createClusterWithQueuesAndOneNode(int mem, String policy)
- throws IOException {
- createClusterWithQueuesAndOneNode(mem, 0, policy);
+ private void createClusterWithQueuesAndOneNode(int mem) {
+ createClusterWithQueuesAndOneNode(mem, 0, "fair");
}
private void createClusterWithQueuesAndOneNode(int mem, int vCores,
- String policy) throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\" >");
- out.println(" <queue name=\"parentA\" >");
- out.println(" <weight>8</weight>");
- out.println(" <queue name=\"childA1\" />");
- out.println(" <queue name=\"childA2\" />");
- out.println(" <queue name=\"childA3\" />");
- out.println(" <queue name=\"childA4\" />");
- out.println(" </queue>");
- out.println(" <queue name=\"parentB\" >");
- out.println(" <weight>1</weight>");
- out.println(" <queue name=\"childB1\" />");
- out.println(" <queue name=\"childB2\" />");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>" + policy
- + "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ String policy) {
+ AllocationFileWriter allocationFileWriter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("parentA")
+ .weight(8)
+ .subQueue(new AllocationFileQueue.Builder("childA1").build())
+ .subQueue(new AllocationFileQueue.Builder("childA2").build())
+ .subQueue(new AllocationFileQueue.Builder("childA3").build())
+ .subQueue(new AllocationFileQueue.Builder("childA4").build())
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("parentB")
+ .weight(1)
+ .subQueue(new AllocationFileQueue.Builder("childB1").build())
+ .subQueue(new AllocationFileQueue.Builder("childB2").build())
+ .build())
+ .build());
+ if (policy.equals("fair")) {
+ allocationFileWriter.fairDefaultQueueSchedulingPolicy();
+ } else if (policy.equals("drf")) {
+ allocationFileWriter.drfDefaultQueueSchedulingPolicy();
+ }
+ allocationFileWriter.writeToFile(ALLOC_FILE);
resourceManager = new MockRM(conf);
resourceManager.start();
@@ -97,9 +99,9 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
}
@Test
- public void testFairShareNoAppsRunning() throws IOException {
+ public void testFairShareNoAppsRunning() {
int nodeCapacity = 16 * 1024;
- createClusterWithQueuesAndOneNode(nodeCapacity, "fair");
+ createClusterWithQueuesAndOneNode(nodeCapacity);
scheduler.update();
// No apps are running in the cluster,verify if fair share is zero
@@ -121,9 +123,9 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
}
@Test
- public void testFairShareOneAppRunning() throws IOException {
+ public void testFairShareOneAppRunning() {
int nodeCapacity = 16 * 1024;
- createClusterWithQueuesAndOneNode(nodeCapacity, "fair");
+ createClusterWithQueuesAndOneNode(nodeCapacity);
// Run a app in a childA1. Verify whether fair share is 100% in childA1,
// since it is the only active queue.
@@ -149,10 +151,9 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
}
@Test
- public void testFairShareMultipleActiveQueuesUnderSameParent()
- throws IOException {
+ public void testFairShareMultipleActiveQueuesUnderSameParent() {
int nodeCapacity = 16 * 1024;
- createClusterWithQueuesAndOneNode(nodeCapacity, "fair");
+ createClusterWithQueuesAndOneNode(nodeCapacity);
// Run apps in childA1,childA2,childA3
createSchedulingRequest(2 * 1024, "root.parentA.childA1", "user1");
@@ -179,7 +180,7 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
public void testFairShareMultipleActiveQueuesUnderDifferentParent()
throws IOException {
int nodeCapacity = 16 * 1024;
- createClusterWithQueuesAndOneNode(nodeCapacity, "fair");
+ createClusterWithQueuesAndOneNode(nodeCapacity);
// Run apps in childA1,childA2 which are under parentA
createSchedulingRequest(2 * 1024, "root.parentA.childA1", "user1");
@@ -218,9 +219,9 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
}
@Test
- public void testFairShareResetsToZeroWhenAppsComplete() throws IOException {
+ public void testFairShareResetsToZeroWhenAppsComplete() {
int nodeCapacity = 16 * 1024;
- createClusterWithQueuesAndOneNode(nodeCapacity, "fair");
+ createClusterWithQueuesAndOneNode(nodeCapacity);
// Run apps in childA1,childA2 which are under parentA
ApplicationAttemptId app1 = createSchedulingRequest(2 * 1024,
@@ -268,8 +269,7 @@ public class TestFairSchedulerFairShare extends FairSchedulerTestBase {
}
@Test
- public void testFairShareWithDRFMultipleActiveQueuesUnderDifferentParent()
- throws IOException {
+ public void testFairShareWithDRFMultipleActiveQueuesUnderDifferentParent() {
int nodeMem = 16 * 1024;
int nodeVCores = 10;
createClusterWithQueuesAndOneNode(nodeMem, nodeVCores, "drf");
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
index f86987f..1312b18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.util.ControlledClock;
import org.apache.hadoop.yarn.util.SystemClock;
import org.junit.After;
@@ -38,9 +40,7 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
@@ -107,7 +107,7 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
}
}
- private void writeAllocFile() throws IOException {
+ private void writeAllocFile() {
/*
* Queue hierarchy:
* root
@@ -115,80 +115,73 @@ public class TestFairSchedulerPreemption extends FairSchedulerTestBase {
* |--- child-1
* |--- child-2
* |--- preemptable-sibling
- * |--- nonpreemptible
+ * |--- nonpreemptable
* |--- child-1
* |--- child-2
*/
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
-
- out.println("<queue name=\"preemptable\">");
- writePreemptionParams(out);
-
- // Child-1
- out.println("<queue name=\"child-1\">");
- writeResourceParams(out);
- out.println("</queue>");
-
- // Child-2
- out.println("<queue name=\"child-2\">");
- writeResourceParams(out);
- out.println("</queue>");
-
- out.println("</queue>"); // end of preemptable queue
-
- out.println("<queue name=\"preemptable-sibling\">");
- writePreemptionParams(out);
- out.println("</queue>");
-
- // Queue with preemption disallowed
- out.println("<queue name=\"nonpreemptable\">");
- out.println("<allowPreemptionFrom>false" +
- "</allowPreemptionFrom>");
- writePreemptionParams(out);
-
- // Child-1
- out.println("<queue name=\"child-1\">");
- writeResourceParams(out);
- out.println("</queue>");
-
- // Child-2
- out.println("<queue name=\"child-2\">");
- writeResourceParams(out);
- out.println("</queue>");
-
- out.println("</queue>"); // end of nonpreemptable queue
+ AllocationFileWriter allocationFileWriter;
+ if (fairsharePreemption) {
+ allocationFileWriter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("preemptable")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .subQueue(new AllocationFileQueue.Builder("child-1")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("child-2")
+ .build())
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("preemptable-sibling")
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("nonpreemptable")
+ .allowPreemptionFrom(false)
+ .fairSharePreemptionThreshold(1)
+ .fairSharePreemptionTimeout(0)
+ .subQueue(new AllocationFileQueue.Builder("child-1")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("child-2")
+ .build())
+ .build())
+ .build());
+ } else {
+ allocationFileWriter = AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("preemptable")
+ .minSharePreemptionTimeout(0)
+ .subQueue(new AllocationFileQueue.Builder("child-1")
+ .minResources("4096mb,4vcores")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("child-2")
+ .minResources("4096mb,4vcores")
+ .build())
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("preemptable-sibling")
+ .minSharePreemptionTimeout(0)
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("nonpreemptable")
+ .allowPreemptionFrom(false)
+ .minSharePreemptionTimeout(0)
+ .subQueue(new AllocationFileQueue.Builder("child-1")
+ .minResources("4096mb,4vcores")
+ .build())
+ .subQueue(new AllocationFileQueue.Builder("child-2")
+ .minResources("4096mb,4vcores")
+ .build())
+ .build())
+ .build());
+ }
if (drf) {
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
+ allocationFileWriter.drfDefaultQueueSchedulingPolicy();
}
- out.println("</allocations>");
- out.close();
+ allocationFileWriter.writeToFile(ALLOC_FILE.getAbsolutePath());
assertTrue("Allocation file does not exist, not running the test",
ALLOC_FILE.exists());
}
- private void writePreemptionParams(PrintWriter out) {
- if (fairsharePreemption) {
- out.println("<fairSharePreemptionThreshold>1" +
- "</fairSharePreemptionThreshold>");
- out.println("<fairSharePreemptionTimeout>0" +
- "</fairSharePreemptionTimeout>");
- } else {
- out.println("<minSharePreemptionTimeout>0" +
- "</minSharePreemptionTimeout>");
- }
- }
-
- private void writeResourceParams(PrintWriter out) {
- if (!fairsharePreemption) {
- out.println("<minResources>4096mb,4vcores</minResources>");
- }
- }
-
private void setupCluster() throws IOException {
resourceManager = new MockRM(conf);
scheduler = (FairScheduler) resourceManager.getResourceScheduler();
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
index 32aab55..ad56a20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerQueueACLs.java
@@ -18,41 +18,39 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.io.PrintWriter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.QueueACLsTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
public class TestFairSchedulerQueueACLs extends QueueACLsTestBase {
@Override
- protected Configuration createConfiguration() throws IOException {
+ protected Configuration createConfiguration() {
FairSchedulerConfiguration fsConf = new FairSchedulerConfiguration();
- final String TEST_DIR = new File(System.getProperty("test.build.data",
+ final String testDir = new File(System.getProperty("test.build.data",
"/tmp")).getAbsolutePath();
- final String ALLOC_FILE = new File(TEST_DIR, "test-queues.xml")
+ final String allocFile = new File(testDir, "test-queues.xml")
.getAbsolutePath();
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <aclSubmitApps> </aclSubmitApps>");
- out.println(" <aclAdministerApps>root_admin </aclAdministerApps>");
- out.println(" <queue name=\"queueA\">");
- out.println(" <aclSubmitApps>queueA_user,common_user </aclSubmitApps>");
- out.println(" <aclAdministerApps>queueA_admin </aclAdministerApps>");
- out.println(" </queue>");
- out.println(" <queue name=\"queueB\">");
- out.println(" <aclSubmitApps>queueB_user,common_user </aclSubmitApps>");
- out.println(" <aclAdministerApps>queueB_admin </aclAdministerApps>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
- fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclSubmitApps(" ")
+ .aclAdministerApps("root_admin ")
+ .subQueue(new AllocationFileQueue.Builder("queueA")
+ .aclSubmitApps("queueA_user,common_user ")
+ .aclAdministerApps("queueA_admin ").build())
+ .subQueue(new AllocationFileQueue.Builder("queueB")
+ .aclSubmitApps("queueB_user,common_user ")
+ .aclAdministerApps("queueB_admin ").build())
+ .build())
+ .writeToFile(allocFile);
+
+ fsConf.set(FairSchedulerConfiguration.ALLOCATION_FILE, allocFile);
fsConf.setBoolean(YarnConfiguration.YARN_ACL_ENABLE, true);
fsConf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
index 5736f75..e9a0b39 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManagerRealScheduler.java
@@ -18,14 +18,16 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import static org.junit.Assert.assertEquals;
@@ -38,7 +40,7 @@ public class TestQueueManagerRealScheduler extends FairSchedulerTestBase {
@Before
public void setup() throws IOException {
createConfiguration();
- writeAllocFile(30, 40);
+ writeAllocFile(30);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE,
ALLOC_FILE.getAbsolutePath());
@@ -56,32 +58,20 @@ public class TestQueueManagerRealScheduler extends FairSchedulerTestBase {
}
}
- private void writeAllocFile(int defaultFairShareTimeout,
- int fairShareTimeout) throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"default\">");
- out.println("</queue>");
- out.println("<queue name=\"queueA\">");
- out.println("</queue>");
- out.println("<queue name=\"queueB\">");
- out.println("<queue name=\"queueB1\">");
- out.println("<minSharePreemptionTimeout>5</minSharePreemptionTimeout>");
- out.println("</queue>");
- out.println("<queue name=\"queueB2\">");
- out.println("</queue>");
- out.println("</queue>");
- out.println("<queue name=\"queueC\">");
- out.println("</queue>");
- out.println("<defaultMinSharePreemptionTimeout>15"
- + "</defaultMinSharePreemptionTimeout>");
- out.println("<defaultFairSharePreemptionTimeout>" +
- + defaultFairShareTimeout + "</defaultFairSharePreemptionTimeout>");
- out.println("<fairSharePreemptionTimeout>"
- + fairShareTimeout + "</fairSharePreemptionTimeout>");
- out.println("</allocations>");
- out.close();
+ private void writeAllocFile(int defaultFairShareTimeout) {
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("default")
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueA").build())
+ .addQueue(new AllocationFileQueue.Builder("queueB")
+ .subQueue(new AllocationFileQueue.Builder("queueB1")
+ .minSharePreemptionTimeout(5).build())
+ .subQueue(new AllocationFileQueue.Builder("queueB2").build())
+ .build())
+ .addQueue(new AllocationFileQueue.Builder("queueC").build())
+ .defaultMinSharePreemptionTimeout(15)
+ .defaultFairSharePreemptionTimeout(defaultFairShareTimeout)
+ .writeToFile(ALLOC_FILE.getAbsolutePath());
}
@Test
@@ -120,7 +110,7 @@ public class TestQueueManagerRealScheduler extends FairSchedulerTestBase {
// Lower the fairshare preemption timeouts and verify it is picked
// correctly.
- writeAllocFile(25, 30);
+ writeAllocFile(25);
scheduler.reinitialize(conf, resourceManager.getRMContext());
assertEquals(25000, queueMgr.getQueue("root")
.getFairSharePreemptionTimeout());
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
index e2be0d9..edb43db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestSchedulingPolicy.java
@@ -19,9 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.util.Collection;
import java.util.Comparator;
import java.util.Stack;
@@ -32,6 +30,10 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.placement.PlacementManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.DominantResourceFairnessPolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FairSharePolicy;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.policies.FifoPolicy;
@@ -319,22 +321,16 @@ public class TestSchedulingPolicy {
@Test
public void testSchedulingPolicyViolation() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("drf").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fair").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -349,22 +345,16 @@ public class TestSchedulingPolicy {
+ " policy if its parent policy is 'fair'.", dynamicQueue);
// Set child1 to 'fair' and child2 to 'drf', the reload the allocation file.
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("fair").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("drf").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
child1 = scheduler.getQueueManager().getQueue("child1");
@@ -379,26 +369,21 @@ public class TestSchedulingPolicy {
}
@Test
- public void testSchedulingPolicyViolationInTheMiddleLevel()
- throws IOException {
+ public void testSchedulingPolicyViolationInTheMiddleLevel() {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"level2\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"level3\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" <queue name=\"leaf\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("level2")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("level3")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("leaf")
+ .schedulingPolicy("fair").build())
+ .build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -417,19 +402,16 @@ public class TestSchedulingPolicy {
public void testFIFOPolicyOnlyForLeafQueues()
throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <queue name=\"intermediate\">");
- out.println(" <schedulingPolicy>fifo</schedulingPolicy>");
- out.println(" <queue name=\"leaf\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("intermediate")
+ .schedulingPolicy("fifo")
+ .subQueue(new AllocationFileQueue.Builder("leaf")
+ .schedulingPolicy("fair").build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
@@ -437,19 +419,15 @@ public class TestSchedulingPolicy {
assertNull("Queue 'intermediate' should be null since 'fifo' is only for "
+ "leaf queue.", intermediate);
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <queue name=\"intermediate\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"leaf\">");
- out.println(" <schedulingPolicy>fifo</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .subQueue(new AllocationFileQueue.Builder("intermediate")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("leaf")
+ .schedulingPolicy("fifo").build())
+ .build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
@@ -461,41 +439,30 @@ public class TestSchedulingPolicy {
}
@Test
- public void testPolicyReinitilization() throws IOException {
+ public void testPolicyReinitialization() throws IOException {
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
- PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("fair").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fair").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.init(conf);
// Set child1 to 'drf' which is not allowed, then reload the allocation file
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <schedulingPolicy>fifo</schedulingPolicy>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
-
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("drf").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fifo").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
FSQueue child1 = scheduler.getQueueManager().getQueue("child1");
@@ -508,20 +475,15 @@ public class TestSchedulingPolicy {
child2.getPolicy() instanceof FairSharePolicy);
// Set both child1 and root to 'drf', then reload the allocation file
- out = new PrintWriter(new FileWriter(ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println("<schedulingPolicy>drf</schedulingPolicy>");
- out.println(" <queue name=\"child1\">");
- out.println(" <schedulingPolicy>drf</schedulingPolicy>");
- out.println(" </queue>");
- out.println(" <queue name=\"child2\">");
- out.println(" <schedulingPolicy>fifo</schedulingPolicy>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("drf")
+ .subQueue(new AllocationFileQueue.Builder("child1")
+ .schedulingPolicy("drf").build())
+ .subQueue(new AllocationFileQueue.Builder("child2")
+ .schedulingPolicy("fifo").build())
+ .build())
+ .writeToFile(ALLOC_FILE);
scheduler.reinitialize(conf, null);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
index db81613..ebf0616 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueue.java
@@ -16,69 +16,261 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
+import com.google.common.collect.Lists;
+
import java.io.PrintWriter;
import java.io.StringWriter;
import java.util.List;
-class AllocationFileQueue {
- private final AllocationFileQueueProperties properties;
+import static org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter.addIfPresent;
+
+/**
+ * DAO for Allocation File Queue.
+ */
+final public class AllocationFileQueue {
+ private static final String DEFAULT_TAG_NAME = "queue";
+ private static final String LEGACY_TAG_NAME = "pool";
+
+ private final String queueName;
+ private final String minResources;
+ private final String maxResources;
+ private final String aclAdministerApps;
+ private final String aclSubmitApps;
+ private final String aclSubmitReservations;
+ private final String aclAdministerReservations;
+ private final String aclListReservations;
+ private final String schedulingPolicy;
+ private final Integer maxRunningApps;
+ private final Double maxAMShare;
+ private final Boolean allowPreemptionFrom;
+ private final Integer minSharePreemptionTimeout;
+ private final String maxChildResources;
+ private final Integer fairSharePreemptionTimeout;
+ private final Double fairSharePreemptionThreshold;
+ private final String maxContainerAllocation;
private final List<AllocationFileQueue> subQueues;
+ private final Float weight;
+ private String tagName;
- AllocationFileQueue(AllocationFileQueueProperties properties,
- List<AllocationFileQueue> subQueues) {
- this.properties = properties;
- this.subQueues = subQueues;
+ private final boolean parent;
+ private final boolean reservation;
+
+ private AllocationFileQueue(Builder builder) {
+ this.queueName = builder.name;
+ this.parent = builder.parent;
+ this.minResources = builder.minResources;
+ this.maxResources = builder.maxResources;
+ this.aclAdministerApps = builder.aclAdministerApps;
+ this.aclSubmitApps = builder.aclSubmitApps;
+ this.aclSubmitReservations = builder.aclSubmitReservations;
+ this.aclAdministerReservations = builder.aclAdministerReservations;
+ this.aclListReservations = builder.aclListReservations;
+ this.schedulingPolicy = builder.schedulingPolicy;
+ this.maxRunningApps = builder.maxRunningApps;
+ this.maxAMShare = builder.maxAMShare;
+ this.allowPreemptionFrom = builder.allowPreemptionFrom;
+ this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout;
+ this.maxChildResources = builder.maxChildResources;
+ this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout;
+ this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold;
+ this.maxContainerAllocation = builder.maxContainerAllocation;
+ this.weight = builder.weight;
+ this.reservation = builder.reservation;
+ this.subQueues = builder.subQueues;
+ this.tagName = DEFAULT_TAG_NAME;
}
String render() {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
printStartTag(pw);
- AllocationFileWriter.printQueues(pw, subQueues);
- AllocationFileWriter.addIfPresent(pw, "minResources",
- properties::getMinResources);
- AllocationFileWriter.addIfPresent(pw, "maxResources",
- properties::getMaxResources);
- AllocationFileWriter.addIfPresent(pw, "aclAdministerApps",
- properties::getAclAdministerApps);
- AllocationFileWriter.addIfPresent(pw, "aclSubmitApps",
- properties::getAclSubmitApps);
- AllocationFileWriter.addIfPresent(pw, "schedulingPolicy",
- properties::getSchedulingPolicy);
- AllocationFileWriter.addIfPresent(pw, "maxRunningApps",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getMaxRunningApps()));
- AllocationFileWriter.addIfPresent(pw, "maxAMShare",
- () -> AllocationFileWriter.createNumberSupplier(properties
- .getMaxAMShare()));
- AllocationFileWriter.addIfPresent(pw, "minSharePreemptionTimeout",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getMinSharePreemptionTimeout()));
- AllocationFileWriter.addIfPresent(pw, "maxChildResources",
- properties::getMaxChildResources);
- AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionTimeout",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getFairSharePreemptionTimeout()));
- AllocationFileWriter.addIfPresent(pw, "fairSharePreemptionThreshold",
- () -> AllocationFileWriter.createNumberSupplier(
- properties.getFairSharePreemptionThreshold()));
- AllocationFileWriter.addIfPresent(pw, "maxContainerAllocation",
- () -> AllocationFileWriter
- .createNumberSupplier(properties.getMaxContainerAllocation()));
+ AllocationFileWriter.printQueues(pw, subQueues,
+ tagName.equals(LEGACY_TAG_NAME));
+ addIfPresent(pw, "minResources", minResources);
+ addIfPresent(pw, "maxResources", maxResources);
+ addIfPresent(pw, "aclAdministerApps", aclAdministerApps);
+ addIfPresent(pw, "aclSubmitApps", aclSubmitApps);
+ addIfPresent(pw, "aclSubmitReservations", aclSubmitReservations);
+ addIfPresent(pw, "aclAdministerReservations", aclAdministerReservations);
+ addIfPresent(pw, "aclListReservations", aclListReservations);
+ addIfPresent(pw, "schedulingPolicy", schedulingPolicy);
+ addIfPresent(pw, "maxRunningApps", maxRunningApps);
+ addIfPresent(pw, "maxAMShare", maxAMShare);
+ addIfPresent(pw, "allowPreemptionFrom", allowPreemptionFrom);
+ addIfPresent(pw, "minSharePreemptionTimeout", minSharePreemptionTimeout);
+ addIfPresent(pw, "maxChildResources", maxChildResources);
+ addIfPresent(pw, "fairSharePreemptionTimeout", fairSharePreemptionTimeout);
+ addIfPresent(pw, "fairSharePreemptionThreshold",
+ fairSharePreemptionThreshold);
+ addIfPresent(pw, "maxContainerAllocation", maxContainerAllocation);
+ addIfPresent(pw, "weight", weight);
+ if (reservation) {
+ pw.println("<reservation></reservation>");
+ }
printEndTag(pw);
pw.close();
+
return sw.toString();
}
+ String renderWithLegacyTag() {
+ this.tagName = LEGACY_TAG_NAME;
+ return render();
+ }
+
private void printStartTag(PrintWriter pw) {
- pw.print("<queue name=\"" + properties.getQueueName() + "\" ");
- if (properties.getParent()) {
- pw.print("type=\"parent\"");
+ String queueWithName = String.format("<%s name=\"%s\"", tagName, queueName);
+ pw.print(queueWithName);
+ if (parent) {
+ pw.print(" type=\"parent\"");
}
pw.println(">");
}
private void printEndTag(PrintWriter pw) {
- pw.println("</queue>");
+ pw.println("</" + tagName + ">");
+ }
+
+ /**
+ * Class that can build queues (with subqueues) for testcases.
+ * The intention of having this class to group the common properties of
+ * simple queues and subqueues by methods delegating calls to a
+ * queuePropertiesBuilder instance.
+ */
+ public static class Builder {
+ private String name;
+ private String minResources;
+ private String maxResources;
+ private String aclAdministerApps;
+ private String aclSubmitApps;
+ private String aclSubmitReservations;
+ private String aclAdministerReservations;
+ private String aclListReservations;
+ private String schedulingPolicy;
+ private Integer maxRunningApps;
+ private Double maxAMShare;
+ private Boolean allowPreemptionFrom;
+ private Integer minSharePreemptionTimeout;
+ private boolean parent;
+ private String maxChildResources;
+ private Integer fairSharePreemptionTimeout;
+ private Double fairSharePreemptionThreshold;
+ private String maxContainerAllocation;
+ private boolean reservation;
+ private final List<AllocationFileQueue> subQueues = Lists.newArrayList();
+ private Float weight;
+
+ public Builder(String name) {
+ this.name = name;
+ }
+
+ public Builder parent(boolean value) {
+ this.parent = value;
+ return this;
+ }
+
+ public Builder minResources(String value) {
+ this.minResources = value;
+ return this;
+ }
+
+ public Builder maxResources(String value) {
+ this.maxResources = value;
+ return this;
+ }
+
+ public Builder aclAdministerApps(String value) {
+ this.aclAdministerApps = value;
+ return this;
+ }
+
+ public Builder aclSubmitApps(String value) {
+ this.aclSubmitApps = value;
+ return this;
+ }
+
+ public Builder aclSubmitReservations(String value) {
+ this.aclSubmitReservations = value;
+ return this;
+ }
+
+ public Builder aclAdministerReservations(String value) {
+ this.aclAdministerReservations = value;
+ return this;
+ }
+
+ public Builder aclListReservations(String value) {
+ this.aclListReservations = value;
+ return this;
+ }
+
+ public Builder schedulingPolicy(String value) {
+ this.schedulingPolicy = value;
+ return this;
+ }
+
+ public Builder maxRunningApps(int value) {
+ this.maxRunningApps = value;
+ return this;
+ }
+
+ public Builder maxAMShare(double value) {
+ this.maxAMShare = value;
+ return this;
+ }
+
+ public Builder allowPreemptionFrom(boolean value) {
+ this.allowPreemptionFrom = value;
+ return this;
+ }
+
+ public Builder minSharePreemptionTimeout(int value) {
+ this.minSharePreemptionTimeout = value;
+ return this;
+ }
+
+ public Builder maxChildResources(String value) {
+ this.maxChildResources = value;
+ return this;
+ }
+
+ public Builder fairSharePreemptionTimeout(Integer value) {
+ this.fairSharePreemptionTimeout = value;
+ return this;
+ }
+
+ public Builder fairSharePreemptionThreshold(
+ double value) {
+ this.fairSharePreemptionThreshold = value;
+ return this;
+ }
+
+ public Builder maxContainerAllocation(String value) {
+ this.maxContainerAllocation = value;
+ return this;
+ }
+
+ public Builder weight(float value) {
+ this.weight = value;
+ return this;
+ }
+
+ public Builder reservation() {
+ this.reservation = true;
+ return this;
+ }
+
+ public Builder subQueue(AllocationFileQueue queue) {
+ if (queue == null) {
+ throw new IllegalArgumentException("Subqueue cannot be null!");
+ }
+ subQueues.add(queue);
+ return this;
+ }
+
+ public AllocationFileQueue build() {
+ return new AllocationFileQueue(this);
+ }
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
deleted file mode 100644
index 176024e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueBuilder.java
+++ /dev/null
@@ -1,121 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-/**
- * Abstract base class for building simple queues and subqueues for testcases.
- * Currently there are two concrete types subclassed from this class:
- * {@link AllocationFileSimpleQueueBuilder} and
- * {@link AllocationFileSubQueueBuilder}.
- * The intention of having this class to group the common properties of
- * simple queues and subqueues by methods delegating calls to a
- * queuePropertiesBuilder instance.
- */
-public abstract class AllocationFileQueueBuilder {
- final AllocationFileQueueProperties.Builder queuePropertiesBuilder;
-
- AllocationFileQueueBuilder() {
- this.queuePropertiesBuilder =
- AllocationFileQueueProperties.Builder.create();
- }
-
- public AllocationFileQueueBuilder parent(boolean parent) {
- this.queuePropertiesBuilder.parent(parent);
- return this;
- }
-
- public AllocationFileQueueBuilder minResources(String value) {
- this.queuePropertiesBuilder.minResources(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxResources(String value) {
- this.queuePropertiesBuilder.maxResources(value);
- return this;
- }
-
- public AllocationFileQueueBuilder aclAdministerApps(String value) {
- this.queuePropertiesBuilder.aclAdministerApps(value);
- return this;
- }
-
- public AllocationFileQueueBuilder aclSubmitApps(String value) {
- this.queuePropertiesBuilder.aclSubmitApps(value);
- return this;
- }
-
- public AllocationFileQueueBuilder schedulingPolicy(String value) {
- this.queuePropertiesBuilder.schedulingPolicy(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxRunningApps(int value) {
- this.queuePropertiesBuilder.maxRunningApps(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxAMShare(double value) {
- this.queuePropertiesBuilder.maxAMShare(value);
- return this;
- }
-
- public AllocationFileQueueBuilder minSharePreemptionTimeout(int value) {
- this.queuePropertiesBuilder.minSharePreemptionTimeout(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxChildResources(String value) {
- this.queuePropertiesBuilder.maxChildResources(value);
- return this;
- }
-
- public AllocationFileQueueBuilder fairSharePreemptionTimeout(Integer value) {
- this.queuePropertiesBuilder.fairSharePreemptionTimeout(value);
- return this;
- }
-
- public AllocationFileQueueBuilder fairSharePreemptionThreshold(
- double value) {
- this.queuePropertiesBuilder.fairSharePreemptionThreshold(value);
- return this;
- }
-
- public AllocationFileQueueBuilder maxContainerAllocation(
- String maxContainerAllocation) {
- this.queuePropertiesBuilder.maxContainerAllocation(maxContainerAllocation);
- return this;
- }
-
- public AllocationFileQueueBuilder subQueue(String queueName) {
- if (this instanceof AllocationFileSimpleQueueBuilder) {
- return new AllocationFileSubQueueBuilder(
- (AllocationFileSimpleQueueBuilder) this, queueName);
- } else {
- throw new IllegalStateException(
- "subQueue can only be invoked on instances of "
- + AllocationFileSimpleQueueBuilder.class);
- }
- }
-
- public abstract AllocationFileWriter buildQueue();
-
- public abstract AllocationFileSimpleQueueBuilder buildSubQueue();
-
- AllocationFileQueueProperties.Builder getqueuePropertiesBuilder() {
- return queuePropertiesBuilder;
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java
similarity index 50%
copy from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
copy to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java
index 7a5656e..48aab06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementPolicy.java
@@ -16,28 +16,31 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
+
+import com.google.common.collect.Lists;
+
import java.io.PrintWriter;
import java.io.StringWriter;
+import java.util.List;
/**
- * Value class that stores user settings and can render data in XML format,
- * see {@link #render()}.
+ * Helper class to manage {@link AllocationFileQueuePlacementRule}
+ * instances for {@link AllocationFileWriter}.
*/
-class UserSettings {
- private final String username;
- private final Integer maxRunningApps;
+public class AllocationFileQueuePlacementPolicy {
+ private List<AllocationFileQueuePlacementRule> rules = Lists.newArrayList();
- UserSettings(Builder builder) {
- this.username = builder.username;
- this.maxRunningApps = builder.maxRunningApps;
+ public AllocationFileQueuePlacementPolicy addRule(
+ AllocationFileQueuePlacementRule rule) {
+ this.rules.add(rule);
+ return this;
}
public String render() {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
addStartTag(pw);
- AllocationFileWriter.addIfPresent(pw, "maxRunningApps",
- () -> AllocationFileWriter.createNumberSupplier(maxRunningApps));
+ addRules(pw);
addEndTag(pw);
pw.close();
@@ -45,36 +48,16 @@ class UserSettings {
}
private void addStartTag(PrintWriter pw) {
- pw.println("<user name=\"" + username + "\">");
- }
-
- private void addEndTag(PrintWriter pw) {
- pw.println("</user>");
+ pw.println("<queuePlacementPolicy>");
}
- /**
- * Builder class for {@link UserSettings}
- */
- public static class Builder {
- private final AllocationFileWriter allocationFileWriter;
- private final String username;
- private Integer maxRunningApps;
-
- Builder(AllocationFileWriter allocationFileWriter, String username) {
- this.allocationFileWriter = allocationFileWriter;
- this.username = username;
+ private void addRules(PrintWriter pw) {
+ for (AllocationFileQueuePlacementRule rule : rules) {
+ pw.println(rule.render());
}
+ }
- public Builder maxRunningApps(int value) {
- this.maxRunningApps = value;
- return this;
- }
-
- public AllocationFileWriter build() {
- UserSettings userSettings = new UserSettings(this);
- allocationFileWriter.setUserSettings(userSettings);
-
- return allocationFileWriter;
- }
+ private void addEndTag(PrintWriter pw) {
+ pw.println("</queuePlacementPolicy>");
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java
new file mode 100644
index 0000000..93118d3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueuePlacementRule.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
+
+
+import com.google.common.collect.Lists;
+
+import java.io.PrintWriter;
+import java.io.StringWriter;
+import java.util.List;
+
+/**
+ * Helper class for {@link AllocationFileWriter} to manage
+ * queue placement rules.
+ */
+public class AllocationFileQueuePlacementRule {
+ public enum RuleName {
+ DEFAULT("default"),
+ SPECIFIED("specified"),
+ REJECT("reject"),
+ NESTED("nestedUserQueue"),
+ PRIMARY_GROUP("primaryGroup");
+
+ private String name;
+
+ RuleName(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public String toString() {
+ return name;
+ }
+ }
+
+ private RuleName name;
+ private boolean create = true;
+ private String queue;
+ private List<AllocationFileQueuePlacementRule> nestedRules = Lists
+ .newArrayList();
+
+ public AllocationFileQueuePlacementRule(RuleName name) {
+ this.name = name;
+ }
+
+ public AllocationFileQueuePlacementRule create(boolean shouldCreate) {
+ this.create = shouldCreate;
+ return this;
+ }
+
+ public AllocationFileQueuePlacementRule queue(String selectedQueue) {
+ this.queue = selectedQueue;
+ return this;
+ }
+
+ public AllocationFileQueuePlacementRule addNestedRule(
+ AllocationFileQueuePlacementRule rule) {
+ this.nestedRules.add(rule);
+ return this;
+ }
+
+ public String render() {
+ StringWriter sw = new StringWriter();
+ PrintWriter pw = new PrintWriter(sw);
+ addStartTag(pw);
+ addNestedRules(pw);
+ addEndTag(pw);
+ pw.close();
+
+ return sw.toString();
+ }
+
+ private void addStartTag(PrintWriter pw) {
+ pw.print("<rule name=\"" + name.toString() + "\" create=\"" +
+ String.valueOf(create) + "\"");
+ if (queue != null) {
+ pw.print("queue=\"" + queue + "\"");
+ }
+ pw.println(">");
+ }
+
+ private void addNestedRules(PrintWriter pw) {
+ if (nestedRules != null && !nestedRules.isEmpty()) {
+ for (AllocationFileQueuePlacementRule rule : nestedRules) {
+ pw.println(rule.render());
+ }
+ }
+ }
+
+ private void addEndTag(PrintWriter pw) {
+ pw.println("</rule>");
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
deleted file mode 100644
index 0a0f330..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileQueueProperties.java
+++ /dev/null
@@ -1,214 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-/**
- * The purpose of this class is to store all properties of a queue.
- */
-public class AllocationFileQueueProperties {
- private final String queueName;
- private final String minResources;
- private final String maxResources;
- private final String aclAdministerApps;
- private final String aclSubmitApps;
- private final String schedulingPolicy;
- private final Integer maxRunningApps;
- private final Double maxAMShare;
- private final Integer minSharePreemptionTimeout;
- private final Boolean parent;
- private final String maxChildResources;
- private final Integer fairSharePreemptionTimeout;
- private final Double fairSharePreemptionThreshold;
- private final String maxContainerAllocation;
-
- AllocationFileQueueProperties(Builder builder) {
- this.queueName = builder.queueName;
- this.parent = builder.parent;
- this.minResources = builder.minResources;
- this.maxResources = builder.maxResources;
- this.aclAdministerApps = builder.aclAdministerApps;
- this.aclSubmitApps = builder.aclSubmitApps;
- this.schedulingPolicy = builder.schedulingPolicy;
- this.maxRunningApps = builder.maxRunningApps;
- this.maxAMShare = builder.maxAMShare;
- this.minSharePreemptionTimeout = builder.minSharePreemptionTimeout;
- this.maxChildResources = builder.maxChildResources;
- this.fairSharePreemptionTimeout = builder.fairSharePreemptionTimeout;
- this.fairSharePreemptionThreshold = builder.fairSharePreemptionThreshold;
- this.maxContainerAllocation = builder.maxContainerAllocation;
- }
-
- public String getQueueName() {
- return queueName;
- }
-
- public String getMinResources() {
- return minResources;
- }
-
- public String getMaxResources() {
- return maxResources;
- }
-
- public String getAclAdministerApps() {
- return aclAdministerApps;
- }
-
- public String getAclSubmitApps() {
- return aclSubmitApps;
- }
-
- public String getSchedulingPolicy() {
- return schedulingPolicy;
- }
-
- public Integer getMaxRunningApps() {
- return maxRunningApps;
- }
-
- public Double getMaxAMShare() {
- return maxAMShare;
- }
-
- public Integer getMinSharePreemptionTimeout() {
- return minSharePreemptionTimeout;
- }
-
- public Boolean getParent() {
- return parent;
- }
-
- public String getMaxChildResources() {
- return maxChildResources;
- }
-
- public Integer getFairSharePreemptionTimeout() {
- return fairSharePreemptionTimeout;
- }
-
- public Double getFairSharePreemptionThreshold() {
- return fairSharePreemptionThreshold;
- }
-
- public String getMaxContainerAllocation() {
- return maxContainerAllocation;
- }
-
- /**
- * Builder class for {@link AllocationFileQueueProperties}.
- */
- public static final class Builder {
- private String queueName;
- private Boolean parent = false;
- private String minResources;
- private String maxResources;
- private String aclAdministerApps;
- private String aclSubmitApps;
- private String schedulingPolicy;
- private Integer maxRunningApps;
- private Double maxAMShare;
- private Integer minSharePreemptionTimeout;
- private String maxChildResources;
- private Integer fairSharePreemptionTimeout;
- private Double fairSharePreemptionThreshold;
- private String maxContainerAllocation;
-
- Builder() {
- }
-
- public static Builder create() {
- return new Builder();
- }
-
- public Builder queueName(String queueName) {
- this.queueName = queueName;
- return this;
- }
-
- public Builder minResources(String minResources) {
- this.minResources = minResources;
- return this;
- }
-
- public Builder maxResources(String maxResources) {
- this.maxResources = maxResources;
- return this;
- }
-
- public Builder aclAdministerApps(String aclAdministerApps) {
- this.aclAdministerApps = aclAdministerApps;
- return this;
- }
-
- public Builder aclSubmitApps(String aclSubmitApps) {
- this.aclSubmitApps = aclSubmitApps;
- return this;
- }
-
- public Builder schedulingPolicy(String schedulingPolicy) {
- this.schedulingPolicy = schedulingPolicy;
- return this;
- }
-
- public Builder maxRunningApps(Integer maxRunningApps) {
- this.maxRunningApps = maxRunningApps;
- return this;
- }
-
- public Builder maxAMShare(Double maxAMShare) {
- this.maxAMShare = maxAMShare;
- return this;
- }
-
- public Builder maxContainerAllocation(String maxContainerAllocation) {
- this.maxContainerAllocation = maxContainerAllocation;
- return this;
- }
-
- public Builder minSharePreemptionTimeout(
- Integer minSharePreemptionTimeout) {
- this.minSharePreemptionTimeout = minSharePreemptionTimeout;
- return this;
- }
-
- public Builder parent(Boolean parent) {
- this.parent = parent;
- return this;
- }
-
- public Builder maxChildResources(String maxChildResources) {
- this.maxChildResources = maxChildResources;
- return this;
- }
-
- public Builder fairSharePreemptionTimeout(
- Integer fairSharePreemptionTimeout) {
- this.fairSharePreemptionTimeout = fairSharePreemptionTimeout;
- return this;
- }
-
- public Builder fairSharePreemptionThreshold(
- Double fairSharePreemptionThreshold) {
- this.fairSharePreemptionThreshold = fairSharePreemptionThreshold;
- return this;
- }
-
- public AllocationFileQueueProperties build() {
- return new AllocationFileQueueProperties(this);
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java
deleted file mode 100644
index 93d100e..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSimpleQueueBuilder.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * Queue builder that can build a simple queue with its properties.
- * Subqueues can be added with {@link #addSubQueue(AllocationFileQueue)}.
- */
-public class AllocationFileSimpleQueueBuilder
- extends AllocationFileQueueBuilder {
- private final AllocationFileWriter allocationFileWriter;
- private final List<AllocationFileQueue> subQueues = new ArrayList<>();
-
- AllocationFileSimpleQueueBuilder(AllocationFileWriter allocationFileWriter,
- String queueName) {
- this.allocationFileWriter = allocationFileWriter;
- getqueuePropertiesBuilder().queueName(queueName);
- }
-
- void addSubQueue(AllocationFileQueue queue) {
- subQueues.add(queue);
- }
-
- @Override
- public AllocationFileWriter buildQueue() {
- AllocationFileQueueProperties queueProperties =
- getqueuePropertiesBuilder().build();
- AllocationFileQueue queue =
- new AllocationFileQueue(queueProperties, subQueues);
-
- if (allocationFileWriter != null) {
- allocationFileWriter.addQueue(queue);
- } else {
- throw new IllegalStateException(
- "allocationFileWriter field has to be set on a " + getClass());
- }
-
- return allocationFileWriter;
- }
-
- @Override
- public AllocationFileSimpleQueueBuilder buildSubQueue() {
- throw new IllegalStateException(
- "buildSubQueue is not supported in " + getClass());
- }
-
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java
deleted file mode 100644
index 728aedc..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileSubQueueBuilder.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- * http://www.apache.org/licenses/LICENSE-2.0
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-
-import com.google.common.collect.Lists;
-
-
-/**
- * Queue builder that can build a subqueue with its properties.
- */
-public class AllocationFileSubQueueBuilder extends AllocationFileQueueBuilder {
- private AllocationFileSimpleQueueBuilder parentQueueBuilder;
-
- AllocationFileSubQueueBuilder(
- AllocationFileSimpleQueueBuilder parentQueueBuilder, String queueName) {
- getqueuePropertiesBuilder().queueName(queueName);
- this.parentQueueBuilder = parentQueueBuilder;
- }
-
- @Override
- public AllocationFileWriter buildQueue() {
- throw new IllegalStateException(
- "BuildQueue is not supported in " + getClass());
- }
-
- public AllocationFileSimpleQueueBuilder buildSubQueue() {
- AllocationFileQueueProperties queueProperties =
- getqueuePropertiesBuilder().build();
- AllocationFileQueue queue =
- new AllocationFileQueue(queueProperties, Lists.newArrayList());
-
- if (parentQueueBuilder != null) {
- parentQueueBuilder.addSubQueue(queue);
- return parentQueueBuilder;
- } else {
- throw new IllegalStateException(
- "parentQueueBuilder field has to be set on a " + getClass());
- }
- }
-}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java
index df1cc53..614f2e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/AllocationFileWriter.java
@@ -16,12 +16,10 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile;
-import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.List;
-import java.util.function.Supplier;
/**
* This class is capable of serializing allocation file data to a file
@@ -29,6 +27,10 @@ import java.util.function.Supplier;
* See {@link #writeToFile(String)} method for the implementation.
*/
public final class AllocationFileWriter {
+ private static final String DRF = "drf";
+ private static final String FAIR = "fair";
+ private static final String FIFO = "fifo";
+
private Integer queueMaxAppsDefault;
private String queueMaxResourcesDefault;
private Integer userMaxAppsDefault;
@@ -39,6 +41,10 @@ public final class AllocationFileWriter {
private String defaultQueueSchedulingPolicy;
private List<AllocationFileQueue> queues = new ArrayList<>();
private UserSettings userSettings;
+ private boolean useLegacyTagNameForQueues = false;
+ private String reservationAgent;
+ private String reservationPolicy;
+ private AllocationFileQueuePlacementPolicy queuePlacementPolicy;
private AllocationFileWriter() {
}
@@ -47,8 +53,9 @@ public final class AllocationFileWriter {
return new AllocationFileWriter();
}
- public AllocationFileSimpleQueueBuilder queue(String queueName) {
- return new AllocationFileSimpleQueueBuilder(this, queueName);
+ public AllocationFileWriter addQueue(AllocationFileQueue queue) {
+ queues.add(queue);
+ return this;
}
public AllocationFileWriter queueMaxAppsDefault(int value) {
@@ -71,6 +78,11 @@ public final class AllocationFileWriter {
return this;
}
+ public AllocationFileWriter disableQueueMaxAMShareDefault() {
+ this.queueMaxAMShareDefault = -1.0d;
+ return this;
+ }
+
public AllocationFileWriter defaultMinSharePreemptionTimeout(int value) {
this.defaultMinSharePreemptionTimeout = value;
return this;
@@ -87,26 +99,57 @@ public final class AllocationFileWriter {
return this;
}
- public AllocationFileWriter defaultQueueSchedulingPolicy(String value) {
- this.defaultQueueSchedulingPolicy = value;
+ public AllocationFileWriter drfDefaultQueueSchedulingPolicy() {
+ this.defaultQueueSchedulingPolicy = DRF;
+ return this;
+ }
+
+ public AllocationFileWriter fairDefaultQueueSchedulingPolicy() {
+ this.defaultQueueSchedulingPolicy = FAIR;
+ return this;
+ }
+
+ public AllocationFileWriter fifoDefaultQueueSchedulingPolicy() {
+ this.defaultQueueSchedulingPolicy = FIFO;
+ return this;
+ }
+
+ public AllocationFileWriter useLegacyTagNameForQueues() {
+ this.useLegacyTagNameForQueues = true;
+ return this;
+ }
+
+ public AllocationFileWriter reservationAgent(String value) {
+ this.reservationAgent = value;
return this;
}
- public UserSettings.Builder userSettings(String username) {
- return new UserSettings.Builder(this, username);
+ public AllocationFileWriter reservationPolicy(String value) {
+ this.reservationPolicy = value;
+ return this;
}
- void addQueue(AllocationFileQueue queue) {
- this.queues.add(queue);
+ public AllocationFileWriter userSettings(UserSettings settings) {
+ this.userSettings = settings;
+ return this;
}
- void setUserSettings(UserSettings userSettings) {
- this.userSettings = userSettings;
+ public AllocationFileWriter queuePlacementPolicy(
+ AllocationFileQueuePlacementPolicy policy) {
+ this.queuePlacementPolicy = policy;
+ return this;
}
- static void printQueues(PrintWriter pw, List<AllocationFileQueue> queues) {
+ static void printQueues(PrintWriter pw, List<AllocationFileQueue> queues,
+ boolean useLegacyTagName) {
for (AllocationFileQueue queue : queues) {
- pw.println(queue.render());
+ final String queueStr;
+ if (useLegacyTagName) {
+ queueStr = queue.renderWithLegacyTag();
+ } else {
+ queueStr = queue.render();
+ }
+ pw.println(queueStr);
}
}
@@ -114,22 +157,18 @@ public final class AllocationFileWriter {
pw.println(userSettings.render());
}
- static void addIfPresent(PrintWriter pw, String tag,
- Supplier<String> supplier) {
- if (supplier.get() != null) {
- pw.println("<" + tag + ">" + supplier.get() + "</" + tag + ">");
- }
+ private void printQueuePlacementPolicy(PrintWriter pw) {
+ pw.println(queuePlacementPolicy.render());
}
- static String createNumberSupplier(Object number) {
- if (number != null) {
- return number.toString();
+ static void addIfPresent(PrintWriter pw, String tag, Object obj) {
+ if (obj != null) {
+ pw.println("<" + tag + ">" + obj.toString() + "</" + tag + ">");
}
- return null;
}
private void writeHeader(PrintWriter pw) {
- pw.println("<?xml version=\"1.0\"?>");
+ pw.println("<?xml version=\"1.0\" encoding=\"UTF-8\"?>");
pw.println("<allocations>");
}
@@ -140,34 +179,37 @@ public final class AllocationFileWriter {
public void writeToFile(String filename) {
PrintWriter pw;
try {
- pw = new PrintWriter(new FileWriter(filename));
+ pw = new PrintWriter(filename, "UTF-8");
} catch (IOException e) {
throw new RuntimeException(e);
}
writeHeader(pw);
if (!queues.isEmpty()) {
- printQueues(pw, queues);
+ printQueues(pw, queues, useLegacyTagNameForQueues);
}
if (userSettings != null) {
printUserSettings(pw);
}
- addIfPresent(pw, "queueMaxAppsDefault",
- () -> createNumberSupplier(queueMaxAppsDefault));
- addIfPresent(pw, "queueMaxResourcesDefault",
- () -> queueMaxResourcesDefault);
- addIfPresent(pw, "userMaxAppsDefault",
- () -> createNumberSupplier(userMaxAppsDefault));
- addIfPresent(pw, "queueMaxAMShareDefault",
- () -> createNumberSupplier(queueMaxAMShareDefault));
+ if (queuePlacementPolicy != null) {
+ printQueuePlacementPolicy(pw);
+ }
+
+ addIfPresent(pw, "queueMaxAppsDefault", queueMaxAppsDefault);
+ addIfPresent(pw, "queueMaxResourcesDefault", queueMaxResourcesDefault);
+ addIfPresent(pw, "userMaxAppsDefault", userMaxAppsDefault);
+ addIfPresent(pw, "queueMaxAMShareDefault", queueMaxAMShareDefault);
addIfPresent(pw, "defaultMinSharePreemptionTimeout",
- () -> createNumberSupplier(defaultMinSharePreemptionTimeout));
+ defaultMinSharePreemptionTimeout);
addIfPresent(pw, "defaultFairSharePreemptionTimeout",
- () -> createNumberSupplier(defaultFairSharePreemptionTimeout));
+ defaultFairSharePreemptionTimeout);
addIfPresent(pw, "defaultFairSharePreemptionThreshold",
- () -> createNumberSupplier(defaultFairSharePreemptionThreshold));
+ defaultFairSharePreemptionThreshold);
addIfPresent(pw, "defaultQueueSchedulingPolicy",
- () -> defaultQueueSchedulingPolicy);
+ defaultQueueSchedulingPolicy);
+ addIfPresent(pw, "reservation-agent", reservationAgent);
+ addIfPresent(pw, "reservation-policy", reservationPolicy);
+
writeFooter(pw);
pw.close();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
index 7a5656e..e8457d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/allocationfile/UserSettings.java
@@ -23,7 +23,7 @@ import java.io.StringWriter;
* Value class that stores user settings and can render data in XML format,
* see {@link #render()}.
*/
-class UserSettings {
+public class UserSettings {
private final String username;
private final Integer maxRunningApps;
@@ -36,8 +36,7 @@ class UserSettings {
StringWriter sw = new StringWriter();
PrintWriter pw = new PrintWriter(sw);
addStartTag(pw);
- AllocationFileWriter.addIfPresent(pw, "maxRunningApps",
- () -> AllocationFileWriter.createNumberSupplier(maxRunningApps));
+ AllocationFileWriter.addIfPresent(pw, "maxRunningApps", maxRunningApps);
addEndTag(pw);
pw.close();
@@ -56,12 +55,10 @@ class UserSettings {
* Builder class for {@link UserSettings}
*/
public static class Builder {
- private final AllocationFileWriter allocationFileWriter;
private final String username;
private Integer maxRunningApps;
- Builder(AllocationFileWriter allocationFileWriter, String username) {
- this.allocationFileWriter = allocationFileWriter;
+ public Builder(String username) {
this.username = username;
}
@@ -70,11 +67,8 @@ class UserSettings {
return this;
}
- public AllocationFileWriter build() {
- UserSettings userSettings = new UserSettings(this);
- allocationFileWriter.setUserSettings(userSettings);
-
- return allocationFileWriter;
+ public UserSettings build() {
+ return new UserSettings(this);
}
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
index 0f57241..4f67809 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/converter/FSConfigConverterTestCommons.java
@@ -17,8 +17,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.converter;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.allocationfile.AllocationFileWriter;
import java.io.ByteArrayOutputStream;
import java.io.File;
@@ -113,38 +114,23 @@ public class FSConfigConverterTestCommons {
configureDummyConversionRulesFile();
}
- @SuppressWarnings("checkstyle:linelength")
- public static void configureFairSchedulerXml() throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queueMaxAMShareDefault>-1.0</queueMaxAMShareDefault>");
- out.println("<defaultQueueSchedulingPolicy>fair</defaultQueueSchedulingPolicy>");
- addQueue(out, "");
- out.println("</allocations>");
- out.close();
- }
-
- @SuppressWarnings("checkstyle:linelength")
- private static void addQueue(PrintWriter out, String additionalConfig) {
- out.println("<queue name=\"root\">");
- out.println(" <schedulingPolicy>fair</schedulingPolicy>");
- out.println(" <weight>1.0</weight>");
- out.println(" <fairSharePreemptionTimeout>100</fairSharePreemptionTimeout>");
- out.println(" <minSharePreemptionTimeout>120</minSharePreemptionTimeout>");
- out.println(" <fairSharePreemptionThreshold>.5</fairSharePreemptionThreshold>");
-
- if (StringUtils.isNotEmpty(additionalConfig)) {
- out.println(additionalConfig);
- }
- out.println("</queue>");
+ public static void configureFairSchedulerXml() {
+ AllocationFileWriter.create()
+ .disableQueueMaxAMShareDefault()
+ .fairDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .schedulingPolicy("fair")
+ .weight(1.0f)
+ .fairSharePreemptionTimeout(100)
+ .minSharePreemptionTimeout(120)
+ .fairSharePreemptionThreshold(0.5f)
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
}
- public static void configureEmptyFairSchedulerXml() throws IOException {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations></allocations>");
- out.close();
+ public static void configureEmptyFairSchedulerXml() {
+ AllocationFileWriter.create()
+ .writeToFile(FS_ALLOC_FILE);
}
public static void configureYarnSiteXmlWithFsAllocFileDefined()
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
index a93effb..f5f21aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesAppsModification.java
@@ -26,9 +26,7 @@ import static org.junit.Assume.assumeTrue;
import java.io.ByteArrayInputStream;
import java.io.DataInputStream;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.io.StringReader;
import java.io.StringWriter;
import java.net.URI;
@@ -83,6 +81,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
@@ -204,23 +207,15 @@ public class TestRMWebServicesAppsModification extends JerseyTestBase {
private class FairTestServletModule extends TestServletModule {
@Override
public void configureScheduler() {
- try {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
- out.println(" <queue name=\"default\">");
- out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
- out.println(" </queue>");
- out.println(" <queue name=\"test\">");
- out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("</allocations>");
- out.close();
- } catch(IOException e) {
- }
+ AllocationFileWriter.create()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclAdministerApps("someuser ")
+ .subQueue(new AllocationFileQueue.Builder("default")
+ .aclAdministerApps("someuser ").build())
+ .subQueue(new AllocationFileQueue.Builder("test")
+ .aclAdministerApps("someuser ").build())
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
index adf6d7c..a048128 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesReservation.java
@@ -24,9 +24,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import java.io.File;
-import java.io.FileWriter;
import java.io.IOException;
-import java.io.PrintWriter;
import java.io.StringReader;
import java.net.URL;
import java.util.Arrays;
@@ -53,6 +51,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairSchedulerConfiguration;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair
+ .allocationfile.AllocationFileWriter;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo;
@@ -181,27 +184,17 @@ public class TestRMWebServicesReservation extends JerseyTestBase {
private static class FairTestServletModule extends TestServletModule {
@Override
public void configureScheduler() {
- try {
- PrintWriter out = new PrintWriter(new FileWriter(FS_ALLOC_FILE));
- out.println("<?xml version=\"1.0\"?>");
- out.println("<allocations>");
- out.println("<queue name=\"root\">");
- out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
- out.println(" <queue name=\"default\">");
- out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
- out.println(" </queue>");
- out.println(" <queue name=\"dedicated\">");
- out.println(" <reservation>");
- out.println(" </reservation>");
- out.println(" <aclAdministerApps>someuser </aclAdministerApps>");
- out.println(" </queue>");
- out.println("</queue>");
- out.println("<defaultQueueSchedulingPolicy>drf" +
- "</defaultQueueSchedulingPolicy>");
- out.println("</allocations>");
- out.close();
- } catch (IOException e) {
- }
+ AllocationFileWriter.create()
+ .drfDefaultQueueSchedulingPolicy()
+ .addQueue(new AllocationFileQueue.Builder("root")
+ .aclAdministerApps("someuser ")
+ .subQueue(new AllocationFileQueue.Builder("default")
+ .aclAdministerApps("someuser ").build())
+ .subQueue(new AllocationFileQueue.Builder("dedicated")
+ .reservation()
+ .aclAdministerApps("someuser ").build())
+ .build())
+ .writeToFile(FS_ALLOC_FILE);
conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, FS_ALLOC_FILE);
conf.set(YarnConfiguration.RM_SCHEDULER, FairScheduler.class.getName());
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org