You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/05/22 23:42:01 UTC
[43/50] [abbrv] hadoop git commit: YARN-3609. Load node labels from
storage inside RM serviceStart. Contributed by Wangda Tan
YARN-3609. Load node labels from storage inside RM serviceStart. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64074a35
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64074a35
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64074a35
Branch: refs/heads/YARN-2928
Commit: 64074a358cd2fda775b4c2edac1137250c14ed42
Parents: f416078
Author: Jian He <ji...@apache.org>
Authored: Wed May 20 16:26:02 2015 -0700
Committer: Zhijie Shen <zj...@apache.org>
Committed: Fri May 22 12:42:57 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodelabels/CommonNodeLabelsManager.java | 8 +-
.../TestFileSystemNodeLabelsStore.java | 4 +
.../yarn/server/resourcemanager/MockRM.java | 26 ++++--
.../server/resourcemanager/RMHATestBase.java | 4 +-
.../resourcemanager/TestRMHAForNodeLabels.java | 93 ++++++++++++++++++++
6 files changed, 125 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f1b5a2d..fa796ac 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -610,6 +610,9 @@ Release 2.7.1 - UNRELEASED
YARN-3681. yarn cmd says "could not find main class 'queue'" in windows.
(Craig Welch and Varun Saxena via xgong)
+ YARN-3609. Load node labels from storage inside RM serviceStart. (Wangda
+ Tan via jianhe)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index badf4d6..2d57ad6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -222,10 +222,6 @@ public class CommonNodeLabelsManager extends AbstractService {
isDistributedNodeLabelConfiguration =
YarnConfiguration.isDistributedNodeLabelConfiguration(conf);
-
- if (nodeLabelsEnabled) {
- initNodeLabelStore(conf);
- }
labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL));
}
@@ -245,6 +241,10 @@ public class CommonNodeLabelsManager extends AbstractService {
@Override
protected void serviceStart() throws Exception {
+ if (nodeLabelsEnabled) {
+ initNodeLabelStore(getConfig());
+ }
+
// init dispatcher only when service start, because recover will happen in
// service init, we don't want to trigger any event handling at that time.
initDispatcher(getConfig());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index fb60cd6..4b052c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -108,6 +108,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr = new MockNodeLabelManager();
mgr.init(conf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
@@ -127,6 +128,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr.stop();
mgr = new MockNodeLabelManager();
mgr.init(conf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
@@ -165,6 +167,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
cf.set(YarnConfiguration.NODELABEL_CONFIGURATION_TYPE,
YarnConfiguration.DISTRIBUTED_NODELABEL_CONFIGURATION_TYPE);
mgr.init(cf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabels().size());
@@ -205,6 +208,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr = new MockNodeLabelManager();
mgr.init(conf);
+ mgr.start();
// check variables
Assert.assertEquals(3, mgr.getClusterNodeLabelNames().size());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
index 3469b0c..672ce13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockRM.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabels
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
@@ -98,6 +97,8 @@ public class MockRM extends ResourceManager {
static final Logger LOG = Logger.getLogger(MockRM.class);
static final String ENABLE_WEBAPP = "mockrm.webapp.enabled";
+
+ final private boolean useNullRMNodeLabelsManager;
public MockRM() {
this(new YarnConfiguration());
@@ -108,20 +109,31 @@ public class MockRM extends ResourceManager {
}
public MockRM(Configuration conf, RMStateStore store) {
- super();
+ this(conf, store, true);
+ }
+
+ public MockRM(Configuration conf, RMStateStore store,
+ boolean useNullRMNodeLabelsManager) {
+ super();
+ this.useNullRMNodeLabelsManager = useNullRMNodeLabelsManager;
init(conf instanceof YarnConfiguration ? conf : new YarnConfiguration(conf));
if(store != null) {
setRMStateStore(store);
}
Logger rootLogger = LogManager.getRootLogger();
- rootLogger.setLevel(Level.DEBUG);
+ rootLogger.setLevel(Level.DEBUG);
}
@Override
- protected RMNodeLabelsManager createNodeLabelManager() {
- RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
- mgr.init(getConfig());
- return mgr;
+ protected RMNodeLabelsManager createNodeLabelManager()
+ throws InstantiationException, IllegalAccessException {
+ if (useNullRMNodeLabelsManager) {
+ RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
+ mgr.init(getConfig());
+ return mgr;
+ } else {
+ return super.createNodeLabelManager();
+ }
}
public void waitForState(ApplicationId appId, RMAppState finalState)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
index 9f54de8..6f3666f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/RMHATestBase.java
@@ -106,8 +106,8 @@ public class RMHATestBase extends ClientBaseWithFixes{
}
protected void startRMs() throws IOException {
- rm1 = new MockRM(confForRM1);
- rm2 = new MockRM(confForRM2);
+ rm1 = new MockRM(confForRM1, null, false);
+ rm2 = new MockRM(confForRM2, null, false);
startRMs(rm1, confForRM1, rm2, confForRM2);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64074a35/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java
new file mode 100644
index 0000000..25d9c56
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMHAForNodeLabels.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableSet;
+
+public class TestRMHAForNodeLabels extends RMHATestBase {
+ public static final Log LOG = LogFactory
+ .getLog(TestSubmitApplicationWithRMHA.class);
+
+ @Before
+ @Override
+ public void setup() throws Exception {
+ super.setup();
+
+ // Create directory for node label store
+ File tempDir = File.createTempFile("nlb", ".tmp");
+ tempDir.delete();
+ tempDir.mkdirs();
+ tempDir.deleteOnExit();
+
+ confForRM1.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ confForRM1.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+ tempDir.getAbsolutePath());
+
+ confForRM2.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ confForRM2.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+ tempDir.getAbsolutePath());
+ }
+
+ @Test
+ public void testRMHARecoverNodeLabels() throws Exception {
+ // start two RMs, and transit rm1 to active, rm2 to standby
+ startRMs();
+
+ // Add labels to rm1
+ rm1.getRMContext()
+ .getNodeLabelManager()
+ .addToCluserNodeLabels(
+ Arrays.asList(NodeLabel.newInstance("a"),
+ NodeLabel.newInstance("b"), NodeLabel.newInstance("c")));
+
+ Map<NodeId, Set<String>> nodeToLabels = new HashMap<>();
+ nodeToLabels.put(NodeId.newInstance("host1", 0), ImmutableSet.of("a"));
+ nodeToLabels.put(NodeId.newInstance("host2", 0), ImmutableSet.of("b"));
+
+ rm1.getRMContext().getNodeLabelManager().replaceLabelsOnNode(nodeToLabels);
+
+ // Do the failover
+ explicitFailover();
+
+ // Check labels in rm2
+ Assert
+ .assertTrue(rm2.getRMContext().getNodeLabelManager()
+ .getClusterNodeLabelNames()
+ .containsAll(ImmutableSet.of("a", "b", "c")));
+ Assert.assertTrue(rm2.getRMContext().getNodeLabelManager()
+ .getNodeLabels().get(NodeId.newInstance("host1", 0)).contains("a"));
+ Assert.assertTrue(rm2.getRMContext().getNodeLabelManager()
+ .getNodeLabels().get(NodeId.newInstance("host2", 0)).contains("b"));
+ }
+}