You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/06/21 23:24:10 UTC
[23/50] [abbrv] hadoop git commit: YARN-6680. Avoid locking overhead
for NO_LABEL lookups. Contributed by Daryn Sharp
YARN-6680. Avoid locking overhead for NO_LABEL lookups. Contributed by Daryn Sharp
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee89ac84
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee89ac84
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee89ac84
Branch: refs/heads/HDFS-10467
Commit: ee89ac84e68d3e181b75c63f74a0444f9d28146f
Parents: e9c2aa1
Author: Naganarasimha <na...@apache.org>
Authored: Mon Jun 19 22:42:45 2017 +0530
Committer: Naganarasimha <na...@apache.org>
Committed: Mon Jun 19 22:42:45 2017 +0530
----------------------------------------------------------------------
.../yarn/nodelabels/CommonNodeLabelsManager.java | 8 +++++++-
.../nodelabels/RMNodeLabelsManager.java | 8 ++++++--
.../resourcemanager/scheduler/ResourceUsage.java | 14 +++++++-------
.../server/resourcemanager/webapp/TestRMWebApp.java | 8 ++++++--
4 files changed, 26 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index 60ade2d..66e945f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -99,6 +99,8 @@ public class CommonNodeLabelsManager extends AbstractService {
protected ConcurrentMap<String, Host> nodeCollections =
new ConcurrentHashMap<String, Host>();
+ protected RMNodeLabel noNodeLabel;
+
protected final ReadLock readLock;
protected final WriteLock writeLock;
@@ -225,7 +227,8 @@ public class CommonNodeLabelsManager extends AbstractService {
isCentralizedNodeLabelConfiguration =
YarnConfiguration.isCentralizedNodeLabelConfiguration(conf);
- labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL));
+ noNodeLabel = new RMNodeLabel(NO_LABEL);
+ labelCollections.put(NO_LABEL, noNodeLabel);
}
/**
@@ -947,6 +950,9 @@ public class CommonNodeLabelsManager extends AbstractService {
}
public boolean isExclusiveNodeLabel(String nodeLabel) throws IOException {
+ if (nodeLabel.equals(NO_LABEL)) {
+ return noNodeLabel.getIsExclusive();
+ }
try {
readLock.lock();
RMNodeLabel label = labelCollections.get(nodeLabel);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 79b25ed..507f696 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -519,12 +519,16 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
public Resource getResourceByLabel(String label, Resource clusterResource) {
label = normalizeLabel(label);
+ if (label.equals(NO_LABEL)) {
+ return noNodeLabel.getResource();
+ }
try {
readLock.lock();
- if (null == labelCollections.get(label)) {
+ RMNodeLabel nodeLabel = labelCollections.get(label);
+ if (nodeLabel == null) {
return Resources.none();
}
- return labelCollections.get(label).getResource();
+ return nodeLabel.getResource();
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 2857379..6f0c7d2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -28,7 +28,6 @@ import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.util.resource.Resources;
/**
@@ -46,6 +45,7 @@ public class ResourceUsage {
private Map<String, UsageByLabel> usages;
// short for no-label :)
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+ private final UsageByLabel usageNoLabel;
public ResourceUsage() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@@ -53,7 +53,8 @@ public class ResourceUsage {
writeLock = lock.writeLock();
usages = new HashMap<String, UsageByLabel>();
- usages.put(NL, new UsageByLabel(NL));
+ usageNoLabel = new UsageByLabel(NL);
+ usages.put(NL, usageNoLabel);
}
// Usage enum here to make implement cleaner
@@ -323,10 +324,9 @@ public class ResourceUsage {
}
private Resource _get(String label, ResourceType type) {
- if (label == null) {
- label = RMNodeLabelsManager.NO_LABEL;
+ if (label == null || label.equals(NL)) {
+ return normalize(usageNoLabel.resArr[type.idx]);
}
-
try {
readLock.lock();
UsageByLabel usage = usages.get(label);
@@ -362,8 +362,8 @@ public class ResourceUsage {
}
private UsageByLabel getAndAddIfMissing(String label) {
- if (label == null) {
- label = RMNodeLabelsManager.NO_LABEL;
+ if (label == null || label.equals(NL)) {
+ return usageNoLabel;
}
if (!usages.containsKey(label)) {
UsageByLabel u = new UsageByLabel(label);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee89ac84/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
index 03fc206..8a65040 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContextImpl;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.applicationsmanager.MockAsm;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
@@ -229,12 +230,15 @@ public class TestRMWebApp {
setupQueueConfiguration(conf);
CapacityScheduler cs = new CapacityScheduler();
- cs.setConf(new YarnConfiguration());
+ YarnConfiguration yarnConf = new YarnConfiguration();
+ cs.setConf(yarnConf);
RMContext rmContext = new RMContextImpl(null, null, null, null, null,
null, new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
new ClientToAMTokenSecretManagerInRM(), null);
- rmContext.setNodeLabelManager(new NullRMNodeLabelsManager());
+ RMNodeLabelsManager labelManager = new NullRMNodeLabelsManager();
+ labelManager.init(yarnConf);
+ rmContext.setNodeLabelManager(labelManager);
cs.setRMContext(rmContext);
cs.init(conf);
return cs;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org