You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by xy...@apache.org on 2018/10/01 18:18:01 UTC
[12/50] [abbrv] hadoop git commit: YARN-8824. App Nodelabel missed
after RM restart for finished apps. Contributed by Bibin A Chundatt.
YARN-8824. App Nodelabel missed after RM restart for finished apps. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e5287a4f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e5287a4f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e5287a4f
Branch: refs/heads/HDDS-4
Commit: e5287a4fe0bb03d929f066fc50eb0e7bd74bb759
Parents: 3f61950
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Wed Sep 26 12:30:26 2018 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Wed Sep 26 12:30:26 2018 +0530
----------------------------------------------------------------------
.../resourcemanager/recovery/RMStateStore.java | 1 +
.../resourcemanager/MockMemoryRMStateStore.java | 1 +
.../yarn/server/resourcemanager/MockRM.java | 31 +++++++++++++
.../server/resourcemanager/TestRMRestart.java | 49 ++++++++++++++++++++
4 files changed, 82 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index a0b10b1..ccd6fc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -307,6 +307,7 @@ public abstract class RMStateStore extends AbstractService {
context.setApplicationTags(srcCtx.getApplicationTags());
context.setApplicationType(srcCtx.getApplicationType());
context.setUnmanagedAM(srcCtx.getUnmanagedAM());
+ context.setNodeLabelExpression(srcCtx.getNodeLabelExpression());
ContainerLaunchContextPBImpl amContainerSpec =
new ContainerLaunchContextPBImpl();
amContainerSpec.setApplicationACLs(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
index 698f1c6..ef9606f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockMemoryRMStateStore.java
@@ -105,6 +105,7 @@ public class MockMemoryRMStateStore extends MemoryRMStateStore {
oldAppSubCtxt.getAMContainerResourceRequests());
context.setLogAggregationContext(oldAppSubCtxt.getLogAggregationContext());
context.setApplicationType(oldAppSubCtxt.getApplicationType());
+ context.setNodeLabelExpression(oldAppSubCtxt.getNodeLabelExpression());
this.appSubCtxtCopy.put(oldAppSubCtxt.getApplicationId(), context);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 6a024f5..df86f28 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -722,6 +722,17 @@ public class MockRM extends ResourceManager {
amResourceRequests.get(0).getNodeLabelExpression(), null, null);
}
+ public RMApp submitApp(List<ResourceRequest> amResourceRequests,
+ String appNodeLabel) throws Exception {
+ return submitApp(amResourceRequests, "app1", "user", null, false, null,
+ super.getConfig().getInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
+ YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS), null, null, true,
+ false, false, null, 0, null, true,
+ amResourceRequests.get(0).getPriority(),
+ amResourceRequests.get(0).getNodeLabelExpression(), null, null, null,
+ appNodeLabel);
+ }
+
public RMApp submitApp(Resource capability, String name, String user,
Map<ApplicationAccessType, String> acls, boolean unmanaged, String queue,
int maxAppAttempts, Credentials ts, String appType,
@@ -771,6 +782,23 @@ public class MockRM extends ResourceManager {
boolean cancelTokensWhenComplete, Priority priority, String amLabel,
Map<ApplicationTimeoutType, Long> applicationTimeouts,
ByteBuffer tokensConf, Set<String> applicationTags) throws Exception {
+ return submitApp(amResourceRequests, name, user, acls, unmanaged, queue,
+ maxAppAttempts, ts, appType, waitForAccepted, keepContainers,
+ isAppIdProvided, applicationId, attemptFailuresValidityInterval,
+ logAggregationContext, cancelTokensWhenComplete, priority, amLabel,
+ applicationTimeouts, tokensConf, applicationTags, null);
+ }
+
+ public RMApp submitApp(List<ResourceRequest> amResourceRequests, String name,
+ String user, Map<ApplicationAccessType, String> acls, boolean unmanaged,
+ String queue, int maxAppAttempts, Credentials ts, String appType,
+ boolean waitForAccepted, boolean keepContainers, boolean isAppIdProvided,
+ ApplicationId applicationId, long attemptFailuresValidityInterval,
+ LogAggregationContext logAggregationContext,
+ boolean cancelTokensWhenComplete, Priority priority, String amLabel,
+ Map<ApplicationTimeoutType, Long> applicationTimeouts,
+ ByteBuffer tokensConf, Set<String> applicationTags, String appNodeLabel)
+ throws Exception {
ApplicationId appId = isAppIdProvided ? applicationId : null;
ApplicationClientProtocol client = getClientRMService();
if (! isAppIdProvided) {
@@ -801,6 +829,9 @@ public class MockRM extends ResourceManager {
if (priority != null) {
sub.setPriority(priority);
}
+ if (appNodeLabel != null) {
+ sub.setNodeLabelExpression(appNodeLabel);
+ }
sub.setApplicationType(appType);
ContainerLaunchContext clc = Records
.newRecord(ContainerLaunchContext.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e5287a4f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index a8b8d89..d5880ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -80,6 +80,7 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ExecutionType;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -88,6 +89,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
@@ -457,6 +459,49 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
}
@Test(timeout = 60000)
+ public void testAppReportNodeLabelRMRestart() throws Exception {
+ if (getSchedulerType() != SchedulerType.CAPACITY) {
+ return;
+ }
+ // Create RM
+ YarnConfiguration newConf = new YarnConfiguration(conf);
+ newConf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ MockRM rm1 = createMockRM(newConf);
+ NodeLabel amLabel = NodeLabel.newInstance("AMLABEL");
+ NodeLabel appLabel = NodeLabel.newInstance("APPLABEL");
+ List<NodeLabel> labels = new ArrayList<>();
+ labels.add(amLabel);
+ labels.add(appLabel);
+ MemoryRMStateStore memStore = (MemoryRMStateStore) rm1.getRMStateStore();
+ rm1.start();
+ // Add label
+ rm1.getAdminService().addToClusterNodeLabels(
+ AddToClusterNodeLabelsRequest.newInstance(labels));
+ // create app and launch the AM
+ ResourceRequest amResourceRequest = ResourceRequest
+ .newInstance(Priority.newInstance(0), ResourceRequest.ANY,
+ Resource.newInstance(200, 1), 1, true, amLabel.getName());
+ ArrayList resReqs = new ArrayList<>();
+ resReqs.add(amResourceRequest);
+ RMApp app0 = rm1.submitApp(resReqs, appLabel.getName());
+ rm1.killApp(app0.getApplicationId());
+ rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED);
+ // start new RM
+ MockRM rm2 = createMockRM(conf, memStore);
+ rm2.start();
+ Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
+ ApplicationReport appReport = rm2.getClientRMService().getApplicationReport(
+ GetApplicationReportRequest.newInstance(app0.getApplicationId()))
+ .getApplicationReport();
+ Assert
+ .assertEquals(amLabel.getName(), appReport.getAmNodeLabelExpression());
+ Assert.assertEquals(appLabel.getName(),
+ appReport.getAppNodeLabelExpression());
+ rm1.stop();
+ rm2.stop();
+ }
+
+ @Test(timeout = 60000)
public void testUnManagedRMRestart() throws Exception {
// Create RM
MockRM rm1 = createMockRM(conf);
@@ -471,6 +516,10 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
MockRM rm2 = createMockRM(conf, memStore);
rm2.start();
Assert.assertEquals(1, rm2.getRMContext().getRMApps().size());
+ ApplicationReport appReport = rm2.getClientRMService().getApplicationReport(
+ GetApplicationReportRequest.newInstance(app0.getApplicationId()))
+ .getApplicationReport();
+ Assert.assertEquals(true, appReport.isUnmanagedApp());
rm1.stop();
rm2.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org