You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2018/08/25 15:50:04 UTC
[47/50] [abbrv] hadoop git commit: YARN-7892. Revisit NodeAttribute
class structure. Contributed by Naganarasimha G R.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 9f3e925..682d6ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.yarn.api;
import java.io.IOException;
+import java.util.Arrays;
import org.apache.commons.lang3.Range;
import org.apache.hadoop.security.proto.SecurityProtos.CancelDelegationTokenRequestProto;
@@ -112,7 +113,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersRequestP
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.StopContainersResponsePBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationRequestPBImpl;
import org.apache.hadoop.yarn.api.protocolrecords.impl.pb.SubmitApplicationResponsePBImpl;
-import org.apache.hadoop.yarn.api.records.CollectorInfo;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -120,6 +120,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.ApplicationTimeout;
+import org.apache.hadoop.yarn.api.records.CollectorInfo;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
@@ -131,9 +132,12 @@ import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
+import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeReport;
+import org.apache.hadoop.yarn.api.records.NodeToAttributeValue;
import org.apache.hadoop.yarn.api.records.PreemptionContainer;
import org.apache.hadoop.yarn.api.records.PreemptionContract;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
@@ -152,8 +156,8 @@ import org.apache.hadoop.yarn.api.records.ReservationRequest;
import org.apache.hadoop.yarn.api.records.ReservationRequests;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceAllocationRequest;
-import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.ResourceSizing;
@@ -183,10 +187,13 @@ import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ExecutionTypeRequestPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeKeyPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributeInfoPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeAttributePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeLabelPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeReportPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.NodeToAttributeValuePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionContractPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
@@ -222,10 +229,14 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ContainerRetryContextProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ExecutionTypeRequestProto;
import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeKeyProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeInfoProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeAttributeProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeLabelProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributeValueProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionContractProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PreemptionMessageProto;
@@ -243,7 +254,6 @@ import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
import org.apache.hadoop.yarn.proto.YarnProtos.StrictPreemptionContractProto;
import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
import org.apache.hadoop.yarn.proto.YarnProtos.YarnClusterMetricsProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.NodeToAttributesProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.AddToClusterNodeLabelsResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.CheckForDecommissioningNodesRequestProto;
@@ -274,6 +284,7 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.AllocateResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.FinishApplicationMasterResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptReportResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationAttemptsRequestProto;
@@ -304,6 +315,8 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueInfoResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetQueueUserAclsInfoResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.IncreaseContainersResourceResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.KillApplicationRequestProto;
@@ -328,9 +341,6 @@ import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersRequestProto
import org.apache.hadoop.yarn.proto.YarnServiceProtos.StopContainersResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SubmitApplicationResponseProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetAllResourceProfilesResponseProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetResourceProfileResponseProto;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodesToAttributesMappingRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
@@ -366,7 +376,6 @@ import org.junit.Ignore;
import org.junit.Test;
import com.google.common.collect.ImmutableSet;
-import java.util.Arrays;
/**
* Test class for YARN API protocol records.
@@ -450,10 +459,12 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(SchedulingRequest.class);
generateByNewInstance(RejectedSchedulingRequest.class);
//for Node attribute support
+ generateByNewInstance(NodeAttributeKey.class);
generateByNewInstance(NodeAttribute.class);
generateByNewInstance(NodeToAttributes.class);
+ generateByNewInstance(NodeToAttributeValue.class);
+ generateByNewInstance(NodeAttributeInfo.class);
generateByNewInstance(NodesToAttributesMappingRequest.class);
-
}
@Test
@@ -1250,11 +1261,29 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
}
@Test
+ public void testNodeAttributeKeyPBImpl() throws Exception {
+ validatePBImplRecord(NodeAttributeKeyPBImpl.class,
+ NodeAttributeKeyProto.class);
+ }
+
+ @Test
+ public void testNodeToAttributeValuePBImpl() throws Exception {
+ validatePBImplRecord(NodeToAttributeValuePBImpl.class,
+ NodeToAttributeValueProto.class);
+ }
+
+ @Test
public void testNodeAttributePBImpl() throws Exception {
validatePBImplRecord(NodeAttributePBImpl.class, NodeAttributeProto.class);
}
@Test
+ public void testNodeAttributeInfoPBImpl() throws Exception {
+ validatePBImplRecord(NodeAttributeInfoPBImpl.class,
+ NodeAttributeInfoProto.class);
+ }
+
+ @Test
public void testNodeToAttributesPBImpl() throws Exception {
validatePBImplRecord(NodeToAttributesPBImpl.class,
NodeToAttributesProto.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
index d4384b4..bad74d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestConfigurationNodeAttributesProvider.java
@@ -22,6 +22,7 @@ import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.BeforeClass;
@@ -120,7 +121,8 @@ public class TestConfigurationNodeAttributesProvider {
while(times>0) {
Set<NodeAttribute> current = spyProvider.getDescriptors();
Assert.assertEquals(1, current.size());
- String attributeName = current.iterator().next().getAttributeName();
+ String attributeName =
+ current.iterator().next().getAttributeKey().getAttributeName();
if ("host".equals(attributeName)){
numOfOldValue++;
} else if ("os".equals(attributeName)) {
@@ -173,7 +175,7 @@ public class TestConfigurationNodeAttributesProvider {
GenericTestUtils.waitFor(() -> {
Set<NodeAttribute> attributes = spyProvider.getDescriptors();
return "os".equalsIgnoreCase(attributes
- .iterator().next().getAttributeName());
+ .iterator().next().getAttributeKey().getAttributeName());
}, 500, 1000);
} catch (Exception e) {
// Make sure we get the timeout exception.
@@ -204,21 +206,22 @@ public class TestConfigurationNodeAttributesProvider {
Iterator<NodeAttribute> ait = attributes.iterator();
while(ait.hasNext()) {
- NodeAttribute at = ait.next();
+ NodeAttribute attr = ait.next();
+ NodeAttributeKey at = attr.getAttributeKey();
if (at.getAttributeName().equals("hostname")) {
Assert.assertEquals("hostname", at.getAttributeName());
Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED,
at.getAttributePrefix());
Assert.assertEquals(NodeAttributeType.STRING,
- at.getAttributeType());
- Assert.assertEquals("host1234", at.getAttributeValue());
+ attr.getAttributeType());
+ Assert.assertEquals("host1234", attr.getAttributeValue());
} else if (at.getAttributeName().equals("uptime")) {
Assert.assertEquals("uptime", at.getAttributeName());
Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED,
at.getAttributePrefix());
Assert.assertEquals(NodeAttributeType.STRING,
- at.getAttributeType());
- Assert.assertEquals("321543", at.getAttributeValue());
+ attr.getAttributeType());
+ Assert.assertEquals("321543", attr.getAttributeValue());
} else {
Assert.fail("Unexpected attribute");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
index f764626..3e2e161 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/TestScriptBasedNodeAttributesProvider.java
@@ -129,7 +129,7 @@ public class TestScriptBasedNodeAttributesProvider {
.getDescriptors().iterator();
while (it.hasNext()) {
NodeAttribute att = it.next();
- switch (att.getAttributeName()) {
+ switch (att.getAttributeKey().getAttributeName()) {
case "host":
Assert.assertEquals(NodeAttributeType.STRING, att.getAttributeType());
Assert.assertEquals("host1234", att.getAttributeValue());
@@ -143,7 +143,8 @@ public class TestScriptBasedNodeAttributesProvider {
Assert.assertEquals("10.0.0.1", att.getAttributeValue());
break;
default:
- Assert.fail("Unexpected attribute name " + att.getAttributeName());
+ Assert.fail("Unexpected attribute name "
+ + att.getAttributeKey().getAttributeName());
break;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 12f8aaf..77f8f66 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -1035,7 +1035,7 @@ public class AdminService extends CompositeService implements
List<NodeAttribute> nodeAttributes = nodeToAttributes.getNodeAttributes();
if (!nodeAttributes.stream()
.allMatch(nodeAttribute -> NodeAttribute.PREFIX_CENTRALIZED
- .equals(nodeAttribute.getAttributePrefix()))) {
+ .equals(nodeAttribute.getAttributeKey().getAttributePrefix()))) {
throw new IOException("Invalid Attribute Mapping for the node " + node
+ ". Prefix should be " + NodeAttribute.PREFIX_CENTRALIZED);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 3f24355..ad796f6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -29,12 +29,14 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.Collectors;
import org.apache.commons.cli.UnrecognizedOptionException;
import org.apache.commons.lang3.Range;
@@ -134,8 +136,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationTimeoutType;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
+import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeToAttributeValue;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
@@ -155,6 +160,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.nodelabels.AttributeValue;
import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
@@ -1851,9 +1857,23 @@ public class ClientRMService extends AbstractService implements
GetAttributesToNodesRequest request) throws YarnException, IOException {
NodeAttributesManager attributesManager =
rmContext.getNodeAttributesManager();
- GetAttributesToNodesResponse response = GetAttributesToNodesResponse
- .newInstance(attributesManager
- .getAttributesToNodes(request.getNodeAttributes()));
+ Map<NodeAttributeKey, List<NodeToAttributeValue>> attrToNodesWithStrVal =
+ new HashMap<>();
+ Map<NodeAttributeKey, Map<String, AttributeValue>> attributesToNodes =
+ attributesManager.getAttributesToNodes(request.getNodeAttributes());
+ for (Map.Entry<NodeAttributeKey, Map<String, AttributeValue>> attrib :
+ attributesToNodes.entrySet()) {
+ Map<String, AttributeValue> nodesToVal = attrib.getValue();
+ List<NodeToAttributeValue> nodeToAttrValList = new ArrayList<>();
+ for (Map.Entry<String, AttributeValue> nodeToVal : nodesToVal
+ .entrySet()) {
+ nodeToAttrValList.add(NodeToAttributeValue
+ .newInstance(nodeToVal.getKey(), nodeToVal.getValue().getValue()));
+ }
+ attrToNodesWithStrVal.put(attrib.getKey(), nodeToAttrValList);
+ }
+ GetAttributesToNodesResponse response =
+ GetAttributesToNodesResponse.newInstance(attrToNodesWithStrVal);
return response;
}
@@ -1865,8 +1885,11 @@ public class ClientRMService extends AbstractService implements
rmContext.getNodeAttributesManager();
Set<NodeAttribute> attributes =
attributesManager.getClusterNodeAttributes(null);
+
GetClusterNodeAttributesResponse response =
- GetClusterNodeAttributesResponse.newInstance(attributes);
+ GetClusterNodeAttributesResponse.newInstance(
+ attributes.stream().map(attr -> NodeAttributeInfo.newInstance(attr))
+ .collect(Collectors.toSet()));
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index 4f4400f..cbb5ecf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -660,7 +660,7 @@ public class ResourceTrackerService extends AbstractService implements
// Validate attributes
if (!nodeAttributes.stream().allMatch(
nodeAttribute -> NodeAttribute.PREFIX_DISTRIBUTED
- .equals(nodeAttribute.getAttributePrefix()))) {
+ .equals(nodeAttribute.getAttributeKey().getAttributePrefix()))) {
// All attributes must be in same prefix: nm.yarn.io.
// Since we have the checks in NM to make sure attributes reported
// in HB are with correct prefix, so it should not reach here.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
index 67e1f38..328910f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeAttributesManagerImpl.java
@@ -19,29 +19,28 @@
package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentHashMap.KeySetView;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-import java.util.ArrayList;
-import java.util.List;
-import com.google.common.base.Strings;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -58,6 +57,8 @@ import org.apache.hadoop.yarn.nodelabels.StringAttributeValue;
import org.apache.hadoop.yarn.server.api.protocolrecords.AttributeMappingOperationType;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeToAttributes;
+import com.google.common.base.Strings;
+
/**
* Manager holding the attributes to Labels.
*/
@@ -75,8 +76,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
// TODO may be we can have a better collection here.
// this will be updated to get the attributeName to NM mapping
- private ConcurrentHashMap<NodeAttribute, RMNodeAttribute> clusterAttributes =
- new ConcurrentHashMap<>();
+ private ConcurrentHashMap<NodeAttributeKey, RMNodeAttribute> clusterAttributes
+ = new ConcurrentHashMap<>();
// hostname -> (Map (attributeName -> NodeAttribute))
// Instead of NodeAttribute, plan to have it in future as AttributeValue
@@ -149,7 +150,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void internalUpdateAttributesOnNodes(
Map<String, Map<NodeAttribute, AttributeValue>> nodeAttributeMapping,
AttributeMappingOperationType op,
- Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded,
+ Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded,
String attributePrefix) {
try {
writeLock.lock();
@@ -210,13 +211,14 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void removeNodeFromAttributes(String nodeHost,
Set<NodeAttribute> attributeMappings) {
for (NodeAttribute rmAttribute : attributeMappings) {
- RMNodeAttribute host = clusterAttributes.get(rmAttribute);
+ RMNodeAttribute host =
+ clusterAttributes.get(rmAttribute.getAttributeKey());
if (host != null) {
host.removeNode(nodeHost);
// If there is no other host has such attribute,
// remove it from the global mapping.
if (host.getAssociatedNodeIds().isEmpty()) {
- clusterAttributes.remove(rmAttribute);
+ clusterAttributes.remove(rmAttribute.getAttributeKey());
}
}
}
@@ -224,12 +226,16 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
private void addNodeToAttribute(String nodeHost,
Map<NodeAttribute, AttributeValue> attributeMappings) {
- for (NodeAttribute attribute : attributeMappings.keySet()) {
- RMNodeAttribute rmNodeAttribute = clusterAttributes.get(attribute);
+ for (Entry<NodeAttribute, AttributeValue> attributeEntry : attributeMappings
+ .entrySet()) {
+
+ RMNodeAttribute rmNodeAttribute =
+ clusterAttributes.get(attributeEntry.getKey().getAttributeKey());
if (rmNodeAttribute != null) {
- rmNodeAttribute.addNode(nodeHost);
+ rmNodeAttribute.addNode(nodeHost, attributeEntry.getValue());
} else {
- clusterAttributes.put(attribute, new RMNodeAttribute(attribute));
+ clusterAttributes.put(attributeEntry.getKey().getAttributeKey(),
+ new RMNodeAttribute(attributeEntry.getKey()));
}
}
}
@@ -257,7 +263,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
*/
protected Map<String, Map<NodeAttribute, AttributeValue>> validate(
Map<String, Set<NodeAttribute>> nodeAttributeMapping,
- Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded,
+ Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded,
boolean isRemoveOperation) throws IOException {
Map<String, Map<NodeAttribute, AttributeValue>> nodeToAttributesMap =
new TreeMap<>();
@@ -274,19 +280,21 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
// validate for attributes
for (NodeAttribute attribute : nodeToAttrMappingEntry.getValue()) {
- String attributeName = attribute.getAttributeName().trim();
+ NodeAttributeKey attributeKey = attribute.getAttributeKey();
+ String attributeName = attributeKey.getAttributeName().trim();
NodeLabelUtil.checkAndThrowLabelName(attributeName);
NodeLabelUtil
- .checkAndThrowAttributePrefix(attribute.getAttributePrefix());
+ .checkAndThrowAttributePrefix(attributeKey.getAttributePrefix());
// ensure trimmed values are set back
- attribute.setAttributeName(attributeName);
- attribute.setAttributePrefix(attribute.getAttributePrefix().trim());
+ attributeKey.setAttributeName(attributeName);
+ attributeKey
+ .setAttributePrefix(attributeKey.getAttributePrefix().trim());
// verify for type against prefix/attributeName
if (validateForAttributeTypeMismatch(isRemoveOperation, attribute,
newAttributesToBeAdded)) {
- newAttributesToBeAdded.put(attribute,
+ newAttributesToBeAdded.put(attribute.getAttributeKey(),
new RMNodeAttribute(attribute));
}
// TODO type based value setting needs to be done using a factory
@@ -310,9 +318,11 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
*/
private boolean validateForAttributeTypeMismatch(boolean isRemoveOperation,
NodeAttribute attribute,
- Map<NodeAttribute, RMNodeAttribute> newAttributes)
+ Map<NodeAttributeKey, RMNodeAttribute> newAttributes)
throws IOException {
- if (isRemoveOperation && !clusterAttributes.containsKey(attribute)) {
+ NodeAttributeKey attributeKey = attribute.getAttributeKey();
+ if (isRemoveOperation
+ && !clusterAttributes.containsKey(attributeKey)) {
// no need to validate anything as its remove operation and attribute
// doesn't exist.
return false; // no need to add as its remove operation
@@ -320,10 +330,10 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
// already existing or attribute is mapped to another Node in the
// current command, then check whether the attribute type is matching
NodeAttribute existingAttribute =
- (clusterAttributes.containsKey((attribute))
- ? clusterAttributes.get(attribute).getAttribute()
- : (newAttributes.containsKey(attribute)
- ? newAttributes.get(attribute).getAttribute()
+ (clusterAttributes.containsKey(attributeKey)
+ ? clusterAttributes.get(attributeKey).getAttribute()
+ : (newAttributes.containsKey(attributeKey)
+ ? newAttributes.get(attributeKey).getAttribute()
: null));
if (existingAttribute == null) {
return true;
@@ -331,7 +341,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
.getAttributeType()) {
throw new IOException("Attribute name - type is not matching with "
+ "already configured mapping for the attribute "
- + attribute.getAttributeName() + " existing : "
+ + attributeKey + " existing : "
+ existingAttribute.getAttributeType() + ", new :"
+ attribute.getAttributeType());
}
@@ -347,37 +357,39 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
}
@Override
- public Set<NodeAttribute> getClusterNodeAttributes(Set<String> prefix) {
+ public Set<NodeAttribute> getClusterNodeAttributes(
+ Set<String> prefix) {
Set<NodeAttribute> attributes = new HashSet<>();
- KeySetView<NodeAttribute, RMNodeAttribute> allAttributes =
- clusterAttributes.keySet();
+ Set<Entry<NodeAttributeKey, RMNodeAttribute>> allAttributes =
+ clusterAttributes.entrySet();
// Return all if prefix is not given.
- if (prefix == null || prefix.isEmpty()) {
- attributes.addAll(allAttributes);
- return attributes;
- }
+ boolean forAllPrefix = prefix == null || prefix.isEmpty();
// Try search attributes by prefix and return valid ones.
- Iterator<NodeAttribute> iterator = allAttributes.iterator();
+ Iterator<Entry<NodeAttributeKey, RMNodeAttribute>> iterator =
+ allAttributes.iterator();
while (iterator.hasNext()) {
- NodeAttribute current = iterator.next();
- if (prefix.contains(current.getAttributePrefix())) {
- attributes.add(current);
+ Entry<NodeAttributeKey, RMNodeAttribute> current = iterator.next();
+ NodeAttributeKey attrID = current.getKey();
+ RMNodeAttribute rmAttr = current.getValue();
+ if (forAllPrefix || prefix.contains(attrID.getAttributePrefix())) {
+ attributes.add(rmAttr.getAttribute());
}
}
return attributes;
}
@Override
- public Map<NodeAttribute, Set<String>> getAttributesToNodes(
- Set<NodeAttribute> attributes) {
+ public Map<NodeAttributeKey, Map<String, AttributeValue>> getAttributesToNodes(
+ Set<NodeAttributeKey> attributes) {
try {
readLock.lock();
boolean fetchAllAttributes = (attributes == null || attributes.isEmpty());
- Map<NodeAttribute, Set<String>> attributesToNodes = new HashMap<>();
- for (Entry<NodeAttribute, RMNodeAttribute> attributeEntry :
+ Map<NodeAttributeKey, Map<String, AttributeValue>> attributesToNodes =
+ new HashMap<>();
+ for (Entry<NodeAttributeKey, RMNodeAttribute> attributeEntry :
clusterAttributes.entrySet()) {
- if (fetchAllAttributes || attributes
- .contains(attributeEntry.getKey())) {
+ if (fetchAllAttributes
+ || attributes.contains(attributeEntry.getKey())) {
attributesToNodes.put(attributeEntry.getKey(),
attributeEntry.getValue().getAssociatedNodeIds());
}
@@ -391,8 +403,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
public Resource getResourceByAttribute(NodeAttribute attribute) {
try {
readLock.lock();
- return clusterAttributes.containsKey(attribute)
- ? clusterAttributes.get(attribute).getResource()
+ return clusterAttributes.containsKey(attribute.getAttributeKey())
+ ? clusterAttributes.get(attribute.getAttributeKey()).getResource()
: Resource.newInstance(0, 0);
} finally {
readLock.unlock();
@@ -425,7 +437,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
attrs = new ArrayList<>();
for (Entry<NodeAttribute, AttributeValue> nodeAttr : v.attributes
.entrySet()) {
- if (prefix.contains(nodeAttr.getKey().getAttributePrefix())) {
+ if (prefix.contains(
+ nodeAttr.getKey().getAttributeKey().getAttributePrefix())) {
attrs.add(nodeAttr.getKey());
}
}
@@ -473,7 +486,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
}
host.activateNode(resource);
for (NodeAttribute attribute : host.getAttributes().keySet()) {
- clusterAttributes.get(attribute).removeNode(resource);
+ clusterAttributes.get(attribute.getAttributeKey()).removeNode(resource);
}
} finally {
writeLock.unlock();
@@ -485,7 +498,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
writeLock.lock();
Host host = nodeCollections.get(nodeId.getHost());
for (NodeAttribute attribute : host.getAttributes().keySet()) {
- clusterAttributes.get(attribute).removeNode(host.getResource());
+ clusterAttributes.get(attribute.getAttributeKey())
+ .removeNode(host.getResource());
}
host.deactivateNode();
} finally {
@@ -531,7 +545,8 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
this.attributes.entrySet().iterator();
while (it.hasNext()) {
Entry<NodeAttribute, AttributeValue> current = it.next();
- if (prefix.equals(current.getKey().getAttributePrefix())) {
+ if (prefix.equals(
+ current.getKey().getAttributeKey().getAttributePrefix())) {
it.remove();
}
}
@@ -659,7 +674,7 @@ public class NodeAttributesManagerImpl extends NodeAttributesManager {
Map<String, Set<NodeAttribute>> nodeAttributeMapping,
AttributeMappingOperationType mappingType, String attributePrefix)
throws IOException {
- Map<NodeAttribute, RMNodeAttribute> newAttributesToBeAdded =
+ Map<NodeAttributeKey, RMNodeAttribute> newAttributesToBeAdded =
new HashMap<>();
Map<String, Map<NodeAttribute, AttributeValue>> validMapping =
validate(nodeAttributeMapping, newAttributesToBeAdded, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java
index 1645d13..93b901e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NodeLabelsUtils.java
@@ -21,9 +21,11 @@ package org.apache.hadoop.yarn.server.resourcemanager.nodelabels;
import java.io.IOException;
import java.util.HashSet;
import java.util.Set;
+import java.util.stream.Collectors;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
import org.apache.hadoop.yarn.api.records.NodeLabel;
/**
@@ -56,4 +58,21 @@ public final class NodeLabelsUtils {
throw new IOException(msg);
}
}
+
+ /**
+ * Returns a set of node attributes whose name exists in the provided
+ * <code>attributeNames</code> list.
+ *
+ * @param attributeNames For this given list of attribute names get the
+ * cluster NodeAttributes
+ * @param clusterNodeAttributes set of node Attributes
+ * @return set of Node Attributes which maps to the give attributes names
+ */
+ public static Set <NodeAttribute> getNodeAttributesByName(
+ Set<String> attributeNames, Set<NodeAttribute> clusterNodeAttributes) {
+ return clusterNodeAttributes.stream()
+ .filter(attribute -> attributeNames
+ .contains(attribute.getAttributeKey().getAttributeName()))
+ .collect(Collectors.toSet());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
index bbc2ec3..8384312 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeAttributeInfo.java
@@ -41,8 +41,8 @@ public class NodeAttributeInfo {
}
public NodeAttributeInfo(NodeAttribute nodeAttribute) {
- this.prefix = nodeAttribute.getAttributePrefix();
- this.name = nodeAttribute.getAttributeName();
+ this.prefix = nodeAttribute.getAttributeKey().getAttributePrefix();
+ this.name = nodeAttribute.getAttributeKey().getAttributeName();
this.type = nodeAttribute.getAttributeType().toString();
this.value = nodeAttribute.getAttributeValue();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 95ad35a..39892f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -18,16 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
-import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
-
-import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse;
-import org.apache.hadoop.yarn.api.records.NodeAttribute;
-import org.apache.hadoop.yarn.api.records.NodeAttributeType;
-import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.any;
@@ -78,6 +68,10 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetAttributesToNodesResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeAttributesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodesRequest;
@@ -88,6 +82,8 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainersResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetLabelsToNodesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNewReservationRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesRequest;
+import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToAttributesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.protocolrecords.GetQueueInfoRequest;
@@ -117,10 +113,15 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.NodeAttribute;
+import org.apache.hadoop.yarn.api.records.NodeAttributeInfo;
+import org.apache.hadoop.yarn.api.records.NodeAttributeKey;
+import org.apache.hadoop.yarn.api.records.NodeAttributeType;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.NodeToAttributeValue;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueConfigurations;
@@ -142,6 +143,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.nodelabels.NodeAttributesManager;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
@@ -163,7 +165,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.Capacity
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.security.QueueACLsManager;
import org.apache.hadoop.yarn.server.resourcemanager.timelineservice.RMTimelineCollectorManager;
-
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -176,11 +177,11 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Test;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
public class TestClientRMService {
@@ -2048,11 +2049,12 @@ public class TestClientRMService {
GetClusterNodeAttributesRequest.newInstance();
GetClusterNodeAttributesResponse response =
client.getClusterNodeAttributes(request);
- Set<NodeAttribute> attributes = response.getNodeAttributes();
+ Set<NodeAttributeInfo> attributes = response.getNodeAttributes();
Assert.assertEquals("Size not correct", 3, attributes.size());
- Assert.assertTrue(attributes.contains(gpu));
- Assert.assertTrue(attributes.contains(os));
- Assert.assertTrue(attributes.contains(docker));
+ Assert.assertTrue(attributes.contains(NodeAttributeInfo.newInstance(gpu)));
+ Assert.assertTrue(attributes.contains(NodeAttributeInfo.newInstance(os)));
+ Assert
+ .assertTrue(attributes.contains(NodeAttributeInfo.newInstance(docker)));
rpc.stopProxy(client, conf);
rm.close();
}
@@ -2071,17 +2073,17 @@ public class TestClientRMService {
NodeAttributesManager mgr = rm.getRMContext().getNodeAttributesManager();
String node1 = "host1";
String node2 = "host2";
- NodeAttribute gpu = NodeAttribute
- .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
- NodeAttributeType.STRING, "nvida");
- NodeAttribute os = NodeAttribute
- .newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
+ NodeAttribute gpu =
+ NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "GPU",
+ NodeAttributeType.STRING, "nvidia");
+ NodeAttribute os =
+ NodeAttribute.newInstance(NodeAttribute.PREFIX_CENTRALIZED, "OS",
NodeAttributeType.STRING, "windows64");
- NodeAttribute docker = NodeAttribute
- .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
+ NodeAttribute docker =
+ NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "DOCKER",
NodeAttributeType.STRING, "docker0");
- NodeAttribute dist = NodeAttribute
- .newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION",
+ NodeAttribute dist =
+ NodeAttribute.newInstance(NodeAttribute.PREFIX_DISTRIBUTED, "VERSION",
NodeAttributeType.STRING, "3_0_2");
Map<String, Set<NodeAttribute>> nodes = new HashMap<>();
nodes.put(node1, ImmutableSet.of(gpu, os, dist));
@@ -2099,35 +2101,55 @@ public class TestClientRMService {
GetAttributesToNodesRequest.newInstance();
GetAttributesToNodesResponse response =
client.getAttributesToNodes(request);
- Map<NodeAttribute, Set<String>> attrs = response.getAttributesToNodes();
+ Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs =
+ response.getAttributesToNodes();
Assert.assertEquals(response.getAttributesToNodes().size(), 4);
- Assert.assertEquals(attrs.get(dist).size(), 2);
- Assert.assertEquals(attrs.get(os).size(), 1);
- Assert.assertEquals(attrs.get(gpu).size(), 1);
- Assert.assertTrue(attrs.get(dist).contains(node1));
- Assert.assertTrue(attrs.get(dist).contains(node2));
- Assert.assertTrue(attrs.get(docker).contains(node2));
-
- GetAttributesToNodesRequest request2 =
- GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker));
+ Assert.assertEquals(attrs.get(dist.getAttributeKey()).size(), 2);
+ Assert.assertEquals(attrs.get(os.getAttributeKey()).size(), 1);
+ Assert.assertEquals(attrs.get(gpu.getAttributeKey()).size(), 1);
+ Assert.assertTrue(findHostnameAndValInMapping(node1, "3_0_2",
+ attrs.get(dist.getAttributeKey())));
+ Assert.assertTrue(findHostnameAndValInMapping(node2, "3_0_2",
+ attrs.get(dist.getAttributeKey())));
+ Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0",
+ attrs.get(docker.getAttributeKey())));
+
+ GetAttributesToNodesRequest request2 = GetAttributesToNodesRequest
+ .newInstance(ImmutableSet.of(docker.getAttributeKey()));
GetAttributesToNodesResponse response2 =
client.getAttributesToNodes(request2);
- Map<NodeAttribute, Set<String>> attrs2 = response2.getAttributesToNodes();
- Assert.assertEquals(response2.getAttributesToNodes().size(), 1);
- Assert.assertTrue(attrs.get(docker).contains(node2));
+ Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs2 =
+ response2.getAttributesToNodes();
+ Assert.assertEquals(attrs2.size(), 1);
+ Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0",
+ attrs2.get(docker.getAttributeKey())));
GetAttributesToNodesRequest request3 =
- GetAttributesToNodesRequest.newInstance(ImmutableSet.of(docker, os));
+ GetAttributesToNodesRequest.newInstance(
+ ImmutableSet.of(docker.getAttributeKey(), os.getAttributeKey()));
GetAttributesToNodesResponse response3 =
client.getAttributesToNodes(request3);
- Map<NodeAttribute, Set<String>> attrs3 = response3.getAttributesToNodes();
- Assert.assertEquals(response3.getAttributesToNodes().size(), 2);
- Assert.assertTrue(attrs.get(os).contains(node1));
- Assert.assertTrue(attrs.get(docker).contains(node2));
+ Map<NodeAttributeKey, List<NodeToAttributeValue>> attrs3 =
+ response3.getAttributesToNodes();
+ Assert.assertEquals(attrs3.size(), 2);
+ Assert.assertTrue(findHostnameAndValInMapping(node1, "windows64",
+ attrs3.get(os.getAttributeKey())));
+ Assert.assertTrue(findHostnameAndValInMapping(node2, "docker0",
+ attrs3.get(docker.getAttributeKey())));
rpc.stopProxy(client, conf);
rm.close();
}
+ private boolean findHostnameAndValInMapping(String hostname, String attrVal,
+ List<NodeToAttributeValue> mappingVals) {
+ for (NodeToAttributeValue value : mappingVals) {
+ if (value.getHostname().equals(hostname)) {
+ return attrVal.equals(value.getAttributeValue());
+ }
+ }
+ return false;
+ }
+
@Test(timeout = 120000)
public void testGetNodesToAttributes() throws IOException, YarnException {
MockRM rm = new MockRM() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index adb7fe0..e40b3c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -880,7 +880,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
.getAttributesForNode(nodeId.getHost());
Assert.assertEquals(1, attrs.size());
NodeAttribute na = attrs.keySet().iterator().next();
- Assert.assertEquals("host", na.getAttributeName());
+ Assert.assertEquals("host", na.getAttributeKey().getAttributeName());
Assert.assertEquals("host2", na.getAttributeValue());
Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType());
@@ -900,7 +900,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
attrs = attributeManager.getAttributesForNode(nodeId.getHost());
Assert.assertEquals(1, attrs.size());
na = attrs.keySet().iterator().next();
- Assert.assertEquals("host", na.getAttributeName());
+ Assert.assertEquals("host", na.getAttributeKey().getAttributeName());
Assert.assertEquals("host3", na.getAttributeValue());
Assert.assertEquals(NodeAttributeType.STRING, na.getAttributeType());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
index e2ee8b4..502f9d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestFileSystemNodeAttributeStore.java
@@ -253,8 +253,10 @@ public class TestFileSystemNodeAttributeStore {
public void checkNodeAttributeEqual(NodeAttribute atr1, NodeAttribute atr2) {
Assert.assertEquals(atr1.getAttributeType(), atr2.getAttributeType());
- Assert.assertEquals(atr1.getAttributeName(), atr2.getAttributeName());
- Assert.assertEquals(atr1.getAttributePrefix(), atr2.getAttributePrefix());
+ Assert.assertEquals(atr1.getAttributeKey().getAttributeName(),
+ atr2.getAttributeKey().getAttributeName());
+ Assert.assertEquals(atr1.getAttributeKey().getAttributePrefix(),
+ atr2.getAttributeKey().getAttributePrefix());
Assert.assertEquals(atr1.getAttributeValue(), atr2.getAttributeValue());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b3d54bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
index b8c5bc9..9bc9388 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestNodeAttributesManager.java
@@ -344,11 +344,12 @@ public class TestNodeAttributesManager {
clusterAttributes = attributesManager.getClusterNodeAttributes(
Sets.newHashSet(NodeAttribute.PREFIX_DISTRIBUTED));
Assert.assertEquals(1, clusterAttributes.size());
- NodeAttribute att = clusterAttributes.iterator().next();
- Assert.assertEquals("dist-node-attribute-v2_0", att.getAttributeName());
+ NodeAttribute attr = clusterAttributes.iterator().next();
+ Assert.assertEquals("dist-node-attribute-v2_0",
+ attr.getAttributeKey().getAttributeName());
Assert.assertEquals(NodeAttribute.PREFIX_DISTRIBUTED,
- att.getAttributePrefix());
- Assert.assertEquals("dist_v3_0", att.getAttributeValue());
+ attr.getAttributeKey().getAttributePrefix());
+ Assert.assertEquals("dist_v3_0", attr.getAttributeValue());
// Replace all attributes
toReplaceMap.put(HOSTNAMES[0],
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org