You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by bi...@apache.org on 2019/10/01 10:44:15 UTC
[hadoop] branch branch-2 updated: YARN-9858. Optimize RMContext
getExclusiveEnforcedPartitions. Contributed by Jonathan Hung.
This is an automated email from the ASF dual-hosted git repository.
bibinchundatt pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new c57e6bc3 YARN-9858. Optimize RMContext getExclusiveEnforcedPartitions. Contributed by Jonathan Hung.
c57e6bc3 is described below
commit c57e6bc385a0a8efef1767056aadc992b4af2cee
Author: bibinchundatt <bi...@apache.org>
AuthorDate: Tue Oct 1 16:13:53 2019 +0530
YARN-9858. Optimize RMContext getExclusiveEnforcedPartitions. Contributed by Jonathan Hung.
---
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 16 ++++++++++++++++
.../server/resourcemanager/DefaultAMSProcessor.java | 6 ++++--
.../yarn/server/resourcemanager/RMAppManager.java | 3 ++-
.../hadoop/yarn/server/resourcemanager/RMContext.java | 4 ----
.../yarn/server/resourcemanager/RMContextImpl.java | 19 -------------------
.../yarn/server/resourcemanager/TestAppManager.java | 1 +
6 files changed, 23 insertions(+), 26 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 7139818..b8aa1ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -23,7 +23,9 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
+import java.util.HashSet;
import java.util.List;
+import java.util.Set;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -3213,6 +3215,20 @@ public class YarnConfiguration extends Configuration {
public static final String EXCLUSIVE_ENFORCED_PARTITIONS = NODE_LABELS_PREFIX
+ EXCLUSIVE_ENFORCED_PARTITIONS_SUFFIX;
+ @Private
+ public static Set<String> getExclusiveEnforcedPartitions(
+ Configuration conf) {
+ Set<String> exclusiveEnforcedPartitions = new HashSet<>();
+ String[] configuredPartitions = conf.getStrings(
+ EXCLUSIVE_ENFORCED_PARTITIONS);
+ if (configuredPartitions != null) {
+ for (String partition : configuredPartitions) {
+ exclusiveEnforcedPartitions.add(partition);
+ }
+ }
+ return exclusiveEnforcedPartitions;
+ }
+
public static final String MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY =
YARN_PREFIX + "cluster.max-application-priority";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 65bbaca..a706f86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -101,11 +101,14 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
RecordFactoryProvider.getRecordFactory(null);
private RMContext rmContext;
+ private Set<String> exclusiveEnforcedPartitions;
@Override
public void init(ApplicationMasterServiceContext amsContext,
ApplicationMasterServiceProcessor nextProcessor) {
this.rmContext = (RMContext)amsContext;
+ this.exclusiveEnforcedPartitions = YarnConfiguration
+ .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration());
}
@Override
@@ -209,8 +212,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
}
if (ResourceRequest.ANY.equals(req.getResourceName())) {
SchedulerUtils.enforcePartitionExclusivity(req,
- getRmContext().getExclusiveEnforcedPartitions(),
- asc.getNodeLabelExpression());
+ exclusiveEnforcedPartitions, asc.getNodeLabelExpression());
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 5b50623..4be5e2a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -112,7 +112,8 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
this.maxCompletedAppsInStateStore = this.maxCompletedAppsInMemory;
}
this.authorizer = YarnAuthorizationProvider.getInstance(conf);
- this.exclusiveEnforcedPartitions = context.getExclusiveEnforcedPartitions();
+ this.exclusiveEnforcedPartitions = YarnConfiguration
+ .getExclusiveEnforcedPartitions(rmContext.getYarnConfiguration());
}
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index b5cc5a7..b255a30 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager;
import java.nio.ByteBuffer;
-import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
@@ -162,7 +161,4 @@ public interface RMContext extends ApplicationMasterServiceContext {
ResourceManager getResourceManager();
String getAppProxyUrl(Configuration conf, ApplicationId applicationId);
-
- Set<String> getExclusiveEnforcedPartitions();
-
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 6717d1b..d7c624d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
-import java.util.HashSet;
-import java.util.Set;
import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
@@ -35,7 +33,6 @@ import org.apache.hadoop.yarn.LocalConfigurationProvider;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWriter;
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
@@ -585,20 +582,4 @@ public class RMContextImpl implements RMContext {
return UNAVAILABLE;
}
}
-
- public Set<String> getExclusiveEnforcedPartitions() {
- Set<String> exclusiveEnforcedPartitions = new HashSet<>();
- Configuration conf = getYarnConfiguration();
- if (conf == null) {
- return new HashSet<>();
- }
- String[] configuredPartitions = conf.getStrings(
- YarnConfiguration.EXCLUSIVE_ENFORCED_PARTITIONS);
- if (configuredPartitions != null) {
- for (String partition : configuredPartitions) {
- exclusiveEnforcedPartitions.add(partition);
- }
- }
- return exclusiveEnforcedPartitions;
- }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index e7bb2d3..1a04e6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -154,6 +154,7 @@ public class TestAppManager{
metricsPublisher = mock(SystemMetricsPublisher.class);
context.setSystemMetricsPublisher(metricsPublisher);
context.setRMApplicationHistoryWriter(writer);
+ ((RMContextImpl) context).setYarnConfiguration(new YarnConfiguration());
return context;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org