You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ep...@apache.org on 2020/09/04 18:25:06 UTC
[hadoop] branch branch-2.10 updated: YARN-10177: Backport YARN-7307
to branch-2.10 Allow client/AM update supported resource types via YARN
APIs
This is an automated email from the ASF dual-hosted git repository.
epayne pushed a commit to branch branch-2.10
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.10 by this push:
new e5bd8d2 YARN-10177: Backport YARN-7307 to branch-2.10 Allow client/AM update supported resource types via YARN APIs
e5bd8d2 is described below
commit e5bd8d28400baa7c9c0062adb697b466a36b7501
Author: Eric E Payne <er...@verizonmedia.com>
AuthorDate: Fri Sep 4 18:23:08 2020 +0000
YARN-10177: Backport YARN-7307 to branch-2.10 Allow client/AM update supported resource types via YARN APIs
---
.../RegisterApplicationMasterResponse.java | 20 +++++
.../yarn/api/records/ResourceInformation.java | 6 ++
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +++
.../hadoop/yarn/util/resource/ResourceUtils.java | 23 +++++-
.../src/main/proto/yarn_service_protos.proto | 1 +
.../yarn/client/api/impl/YarnClientImpl.java | 16 ++++
.../RegisterApplicationMasterResponsePBImpl.java | 95 ++++++++++++++++++++--
.../src/main/resources/yarn-default.xml | 9 ++
.../resourcemanager/DefaultAMSProcessor.java | 2 +
9 files changed, 173 insertions(+), 10 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
index 0b886dd..e8805e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/RegisterApplicationMasterResponse.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.util.Records;
@@ -204,4 +205,23 @@ public abstract class RegisterApplicationMasterResponse {
@Unstable
public abstract void setSchedulerResourceTypes(
EnumSet<SchedulerResourceTypes> types);
+
+ /**
+ * Get available resource types supported by RM.
+ *
+ * @return a Map of RM settings
+ */
+ @Public
+ @Unstable
+ public abstract List<ResourceTypeInfo> getResourceTypes();
+
+ /**
+ * Set the resource types used by RM.
+ *
+ * @param types
+ * a set of the resource types supported by RM.
+ */
+ @Private
+ @Unstable
+ public abstract void setResourceTypes(List<ResourceTypeInfo> types);
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 08746a9..e699e96 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -220,6 +220,12 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
ResourceTypes.COUNTABLE, minRes, maxRes);
}
+ public static ResourceInformation newInstance(String name, String units,
+ ResourceTypes resourceType) {
+ return ResourceInformation.newInstance(name, units, 0L, resourceType, 0L,
+ Long.MAX_VALUE);
+ }
+
public static ResourceInformation newInstance(String name, long value) {
return ResourceInformation
.newInstance(name, "", value, ResourceTypes.COUNTABLE, 0L,
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 814e4d9..854f6a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -990,6 +990,17 @@ public class YarnConfiguration extends Configuration {
public static final String RM_PROXY_USER_PREFIX = RM_PREFIX + "proxyuser.";
/**
+ * Enable/disable loading resource-types.xml at client side.
+ */
+ @Public
+ @Unstable
+ public static final String YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = YARN_PREFIX
+ + "client.load.resource-types.from-server";
+ @Public
+ @Unstable
+ public static final boolean DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER = false;
+
+ /**
* Timeout in seconds for YARN node graceful decommission.
* This is the maximal time to wait for running containers and applications
* to complete before transition a DECOMMISSIONING node into DECOMMISSIONED.
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 011bd97..8584690 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -22,8 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -655,4 +653,25 @@ public class ResourceUtils {
return info;
}
+
+ /**
+ * Reinitialize all resource types from external source (in case of client,
+ * server will send the updated list and local resourceutils cache will be
+ * updated as per server's list of resources)
+ *
+ * @param resourceTypeInfo
+ * List of resource types
+ */
+ public static void reinitializeResources(
+ List<ResourceTypeInfo> resourceTypeInfo) {
+ Map<String, ResourceInformation> resourceInformationMap = new HashMap<>();
+
+ for (ResourceTypeInfo resourceType : resourceTypeInfo) {
+ resourceInformationMap.put(resourceType.getName(),
+ ResourceInformation.newInstance(resourceType.getName(),
+ resourceType.getDefaultUnit(), resourceType.getResourceType()));
+ }
+ ResourceUtils
+ .initializeResourcesFromResourceInformationMap(resourceInformationMap);
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
index 038d73a..26dfa41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_service_protos.proto
@@ -48,6 +48,7 @@ message RegisterApplicationMasterResponseProto {
optional string queue = 5;
repeated NMTokenProto nm_tokens_from_previous_attempts = 6;
repeated SchedulerResourceTypes scheduler_resource_types = 7;
+ repeated ResourceTypeInfoProto resource_types = 9;
}
message FinishApplicationMasterRequestProto {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
index 397566a..c0038cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/YarnClientImpl.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.EnumSet;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
@@ -103,6 +104,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.Token;
@@ -123,6 +125,7 @@ import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
@@ -146,6 +149,7 @@ public class YarnClientImpl extends YarnClient {
String timelineDTRenewer;
private boolean timelineV1ServiceEnabled;
protected boolean timelineServiceBestEffort;
+ private boolean loadResourceTypesFromServer;
private static final String ROOT = "root";
@@ -191,6 +195,11 @@ public class YarnClientImpl extends YarnClient {
timelineServiceBestEffort = conf.getBoolean(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_BEST_EFFORT,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_BEST_EFFORT);
+
+ loadResourceTypesFromServer = conf.getBoolean(
+ YarnConfiguration.YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER,
+ YarnConfiguration.DEFAULT_YARN_CLIENT_LOAD_RESOURCETYPES_FROM_SERVER);
+
super.serviceInit(conf);
}
@@ -209,6 +218,13 @@ public class YarnClientImpl extends YarnClient {
} catch (IOException e) {
throw new YarnRuntimeException(e);
}
+
+ // Reinitialize local resource types cache from list of resources pulled from
+ // RM.
+ if (loadResourceTypesFromServer) {
+ ResourceUtils.reinitializeResources(getResourceTypeInfo());
+ }
+
super.serviceStart();
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
index 1a70933..9253637 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/RegisterApplicationMasterResponsePBImpl.java
@@ -18,10 +18,8 @@
package org.apache.hadoop.yarn.api.protocolrecords.impl.pb;
-
-import java.nio.ByteBuffer;
-import java.util.*;
-
+import com.google.protobuf.ByteString;
+import com.google.protobuf.TextFormat;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse;
@@ -29,21 +27,28 @@ import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NMTokenPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceTypeInfoPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationACLMapProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceTypeInfoProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.NMTokenProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.RegisterApplicationMasterResponseProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.TextFormat;
-
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.EnumSet;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
@Private
@Unstable
@@ -59,6 +64,7 @@ public class RegisterApplicationMasterResponsePBImpl extends
private List<Container> containersFromPreviousAttempts = null;
private List<NMToken> nmTokens = null;
private EnumSet<SchedulerResourceTypes> schedulerResourceTypes = null;
+ private List<ResourceTypeInfo> resourceTypeInfo = null;
public RegisterApplicationMasterResponsePBImpl() {
builder = RegisterApplicationMasterResponseProto.newBuilder();
@@ -123,9 +129,11 @@ public class RegisterApplicationMasterResponsePBImpl extends
if(schedulerResourceTypes != null) {
addSchedulerResourceTypes();
}
+ if (resourceTypeInfo != null) {
+ addResourceTypeInfosToProto();
+ }
}
-
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = RegisterApplicationMasterResponseProto.newBuilder(proto);
@@ -456,4 +464,75 @@ public class RegisterApplicationMasterResponsePBImpl extends
private NMToken convertFromProtoFormat(NMTokenProto proto) {
return new NMTokenPBImpl(proto);
}
+
+ private ResourceTypeInfoPBImpl convertFromProtoFormat(
+ ResourceTypeInfoProto p) {
+ return new ResourceTypeInfoPBImpl(p);
+ }
+
+ private ResourceTypeInfoProto convertToProtoFormat(ResourceTypeInfo t) {
+ return ((ResourceTypeInfoPBImpl) t).getProto();
+ }
+
+ @Override
+ public List<ResourceTypeInfo> getResourceTypes() {
+ initResourceTypeInfosList();
+ return this.resourceTypeInfo;
+ }
+
+ @Override
+ public void setResourceTypes(List<ResourceTypeInfo> types) {
+ if (resourceTypeInfo == null) {
+ builder.clearResourceTypes();
+ }
+ this.resourceTypeInfo = types;
+ }
+
+ private void addResourceTypeInfosToProto() {
+ maybeInitBuilder();
+ builder.clearResourceTypes();
+ if (resourceTypeInfo == null) {
+ return;
+ }
+ Iterable<ResourceTypeInfoProto> iterable = new Iterable<ResourceTypeInfoProto>() {
+ @Override
+ public Iterator<ResourceTypeInfoProto> iterator() {
+ return new Iterator<ResourceTypeInfoProto>() {
+
+ Iterator<ResourceTypeInfo> iter = resourceTypeInfo.iterator();
+
+ @Override
+ public boolean hasNext() {
+ return iter.hasNext();
+ }
+
+ @Override
+ public ResourceTypeInfoProto next() {
+ return convertToProtoFormat(iter.next());
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException();
+
+ }
+ };
+
+ }
+ };
+ builder.addAllResourceTypes(iterable);
+ }
+
+ private void initResourceTypeInfosList() {
+ if (this.resourceTypeInfo != null) {
+ return;
+ }
+ RegisterApplicationMasterResponseProtoOrBuilder p = viaProto ? proto : builder;
+ List<ResourceTypeInfoProto> list = p.getResourceTypesList();
+ resourceTypeInfo = new ArrayList<ResourceTypeInfo>();
+
+ for (ResourceTypeInfoProto a : list) {
+ resourceTypeInfo.add(convertFromProtoFormat(a));
+ }
+ }
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 41f0431..7d4f04a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -3632,4 +3632,13 @@
<name>yarn.resourcemanager.application.max-tag.length</name>
<value>100</value>
</property>
+
+ <property>
+ <description>
+ Provides an option for client to load supported resource types from RM
+ instead of depending on local resource-types.xml file.
+ </description>
+ <name>yarn.client.load.resource-types.from-server</name>
+ <value>false</value>
+ </property>
</configuration>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index a706f86..a18b4c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -74,6 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import java.io.IOException;
@@ -181,6 +182,7 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
response.setSchedulerResourceTypes(getScheduler()
.getSchedulingResourceTypes());
+ response.setResourceTypes(ResourceUtils.getResourcesTypeInfo());
}
@Override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org