You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/09/12 11:15:03 UTC
[25/29] hadoop git commit: YARN-8740. Clear node attribute path after
each test run. Contributed by Bibin A Chundatt.
YARN-8740. Clear node attribute path after each test run. Contributed by Bibin A Chundatt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e64e62d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e64e62d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e64e62d
Branch: refs/heads/trunk
Commit: 5e64e62deeb467ac2969bc357a24810db855cffa
Parents: c44088a
Author: Sunil G <su...@apache.org>
Authored: Mon Sep 3 12:11:19 2018 +0530
Committer: Sunil G <su...@apache.org>
Committed: Wed Sep 12 16:01:01 2018 +0530
----------------------------------------------------------------------
.../nodelabels/NodeAttributesManagerImpl.java | 13 +++--
.../resourcemanager/NodeAttributeTestUtils.java | 54 ++++++++++++++++++++
.../resourcemanager/TestClientRMService.java | 9 ++--
.../resourcemanager/TestRMAdminService.java | 3 +-
.../TestFileSystemNodeAttributeStore.java | 28 +++++++---
.../nodelabels/TestNodeAttributesManager.java | 9 +---
6 files changed, 93 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e64e62d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index 81f42d9..5533436 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+import com.google.common.annotations.VisibleForTesting;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
@@ -151,7 +152,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
}
}
- private void internalUpdateAttributesOnNodes(
+ @VisibleForTesting
+ protected void internalUpdateAttributesOnNodes(
Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
AttributeMappingOperationType op,
Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded,
@@ -195,7 +197,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
logMsg.append(" NM = ");
logMsg.append(entry.getKey());
logMsg.append(", attributes=[ ");
- logMsg.append(StringUtils.join(entry.getValue().entrySet(), ","));
+ logMsg.append(StringUtils.join(entry.getValue().keySet(), ","));
logMsg.append("] ,");
}
@@ -700,9 +702,10 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
new HashMap<>();
Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
validate(nodeAttributeMapping, newAttributesToBeAdded, false);
-
- internalUpdateAttributesOnNodes(validMapping, mappingType,
- newAttributesToBeAdded, attributePrefix);
+ if (validMapping.size() > 0) {
+ internalUpdateAttributesOnNodes(validMapping, mappingType,
+ newAttributesToBeAdded, attributePrefix);
+ }
}
protected void stopDispatcher() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e64e62d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java
new file mode 100644
index 0000000..1d12cb7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeAttributeTestUtils.java
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager;
+
+import java.io.File;
+import java.io.IOException;
+import org.apache.commons.io.FileUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * Test Utils for NodeAttribute.
+ */
+public final class NodeAttributeTestUtils {
+
+ private NodeAttributeTestUtils() {
+
+ }
+
+ public static YarnConfiguration getRandomDirConf(Configuration conf)
+ throws IOException {
+ YarnConfiguration newConf;
+ if (conf == null) {
+ newConf = new YarnConfiguration();
+ } else {
+ newConf = new YarnConfiguration(conf);
+ }
+ File tempDir = GenericTestUtils.getRandomizedTestDir();
+ FileUtils.deleteDirectory(tempDir);
+ tempDir.mkdirs();
+ tempDir.deleteOnExit();
+ newConf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
+ tempDir.getAbsolutePath());
+ return newConf;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e64e62d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 39892f2..efabbe2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -2012,7 +2012,8 @@ public class TestClientRMService {
@Test(timeout = 120000)
public void testGetClusterNodeAttributes() throws IOException, YarnException {
- MockRM rm = new MockRM() {
+ Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
+ MockRM rm = new MockRM(newConf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
@@ -2061,7 +2062,8 @@ public class TestClientRMService {
@Test(timeout = 120000)
public void testGetAttributesToNodes() throws IOException, YarnException {
- MockRM rm = new MockRM() {
+ Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
+ MockRM rm = new MockRM(newConf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
@@ -2152,7 +2154,8 @@ public class TestClientRMService {
@Test(timeout = 120000)
public void testGetNodesToAttributes() throws IOException, YarnException {
- MockRM rm = new MockRM() {
+ Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
+ MockRM rm = new MockRM(newConf) {
protected ClientRMService createClientRMService() {
return new ClientRMService(this.rmContext, scheduler, this.rmAppManager,
this.applicationACLsManager, this.queueACLsManager,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e64e62d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
index a1b95b8..475dd8c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAdminService.java
@@ -1557,7 +1557,8 @@ public class TestRMAdminService {
// 1.5. Test with unknown node when failOnUnknownNodes is false
// also test : 3. Ensure Appropriate manager Method call is done
- rm = new MockRM();
+ Configuration newConf = NodeAttributeTestUtils.getRandomDirConf(null);
+ rm = new MockRM(newConf);
NodeAttributesManager spiedAttributesManager =
Mockito.spy(rm.getRMContext().getNodeAttributesManager());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e64e62d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
index 07ea20d..4a7a59f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
@@ -17,6 +17,11 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+
import com.google.common.collect.ImmutableSet;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
@@ -25,12 +30,12 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.nodelabels.AttributeValue;
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
+import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
@@ -68,12 +73,7 @@ public class TestFileSystemNodeAttributeStore {
conf = new Configuration();
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
- File tempDir = File.createTempFile("nattr", ".tmp");
- tempDir.delete();
- tempDir.mkdirs();
- tempDir.deleteOnExit();
- conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
- tempDir.getAbsolutePath());
+ conf = NodeAttributeTestUtils.getRandomDirConf(conf);
mgr.init(conf);
mgr.start();
}
@@ -87,6 +87,20 @@ public class TestFileSystemNodeAttributeStore {
}
@Test(timeout = 10000)
+ public void testEmptyRecoverSkipInternalUdpate() throws Exception {
+ // Stop manager
+ mgr.stop();
+
+ // Start new attribute manager with same path
+ mgr = spy(new MockNodeAttrbuteManager());
+ mgr.init(conf);
+ mgr.start();
+
+ verify(mgr, times(0))
+ .internalUpdateAttributesOnNodes(any(), any(), any(), any());
+ }
+
+ @Test(timeout = 10000)
public void testRecoverWithMirror() throws Exception {
//------host0----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e64e62d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
index 9bc9388..b181677 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
@@ -28,12 +28,12 @@ import org.apache.hadoop.yarn.nodelabels.AttributeValue;
import org.apache.hadoop.yarn.nodelabels.NodeAttributeStore;
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabelUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.NodeAttributeTestUtils;
import org.junit.Test;
import org.junit.Before;
import org.junit.After;
import org.junit.Assert;
-import java.io.File;
import java.io.IOException;
import java.util.HashMap;
import java.util.HashSet;
@@ -57,12 +57,7 @@ public class TestNodeAttributesManager {
attributesManager = new NodeAttributesManagerImpl();
conf.setClass(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_IMPL_CLASS,
FileSystemNodeAttributeStore.class, NodeAttributeStore.class);
- File tempDir = File.createTempFile("nattr", ".tmp");
- tempDir.delete();
- tempDir.mkdirs();
- tempDir.deleteOnExit();
- conf.set(YarnConfiguration.FS_NODE_ATTRIBUTE_STORE_ROOT_DIR,
- tempDir.getAbsolutePath());
+ conf = NodeAttributeTestUtils.getRandomDirConf(conf);
attributesManager.init(conf);
attributesManager.start();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org