You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/09/12 20:02:50 UTC
[05/50] [abbrv] hadoop git commit: YARN-7043. Cleanup
ResourceProfileManager. (wangda)
YARN-7043. Cleanup ResourceProfileManager. (wangda)
Change-Id: I463356f37bf1f6a3f1fc3c594c79916e8c0ab913
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39240b61
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39240b61
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39240b61
Branch: refs/heads/HDFS-10467
Commit: 39240b61a163b127eec385decd30ffd96e694c28
Parents: 758b771
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Aug 21 17:20:06 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 12 09:19:12 2017 -0700
----------------------------------------------------------------------
.../ams/ApplicationMasterServiceProcessor.java | 8 ++-
.../yarn/api/ApplicationClientProtocol.java | 11 ++--
.../YARNFeatureNotEnabledException.java | 45 +++++++++++++++
.../yarn/util/resource/ResourceUtils.java | 21 +++++++
.../hadoop/yarn/client/api/YarnClient.java | 13 +++--
.../resource/DominantResourceCalculator.java | 6 +-
.../hadoop/yarn/api/TestPBImplRecords.java | 1 +
.../resourcemanager/AMSProcessingChain.java | 2 +-
.../server/resourcemanager/ClientRMService.java | 26 ++-------
.../resourcemanager/DefaultAMSProcessor.java | 12 ++--
...pportunisticContainerAllocatorAMService.java | 3 +-
.../server/resourcemanager/RMServerUtils.java | 14 -----
.../resource/ResourceProfilesManager.java | 32 +++++-----
.../resource/ResourceProfilesManagerImpl.java | 61 +++++++++++++-------
.../scheduler/AbstractYarnScheduler.java | 18 +++++-
.../TestApplicationMasterService.java | 11 ++--
.../resource/TestResourceProfiles.java | 10 +++-
17 files changed, 191 insertions(+), 103 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
index b7d925a..8e76a11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/ams/ApplicationMasterServiceProcessor.java
@@ -52,11 +52,13 @@ public interface ApplicationMasterServiceProcessor {
* @param request Register Request.
* @param response Register Response.
* @throws IOException IOException.
+ * @throws YarnException in critical situation where invalid
+ * profiles/resources are added.
*/
- void registerApplicationMaster(
- ApplicationAttemptId applicationAttemptId,
+ void registerApplicationMaster(ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException;
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException;
/**
* Allocate call.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
index 3112d65..3c4e4d0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationClientProtocol.java
@@ -81,6 +81,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnClusterMetrics;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
/**
* <p>The protocol between clients and the <code>ResourceManager</code>
@@ -604,7 +605,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* @param request request to get all the resource profiles
* @return Response containing a map of the profile name to Resource
* capabilities
- * @throws YarnException if resource profiles are not enabled on the RM
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@@ -618,8 +620,8 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* </p>
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
- * @throws YarnException if resource profiles are not enabled on the RM or
- * the profile cannot be found
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@@ -633,8 +635,7 @@ public interface ApplicationClientProtocol extends ApplicationBaseProtocol {
* </p>
* @param request request to get the details of a resource profile
* @return Response containing the details for a particular resource profile
- * @throws YarnException if resource profiles are not enabled on the RM or
- * the profile cannot be found
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java
new file mode 100644
index 0000000..62340fe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/exceptions/YARNFeatureNotEnabledException.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.exceptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * This exception is thrown when a feature is being used which is not enabled
+ * yet.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+public class YARNFeatureNotEnabledException extends YarnException {
+ private static final long serialVersionUID = 898023752676L;
+
+ public YARNFeatureNotEnabledException(Throwable cause) {
+ super(cause);
+ }
+
+ public YARNFeatureNotEnabledException(String message) {
+ super(message);
+ }
+
+ public YARNFeatureNotEnabledException(String message, Throwable cause) {
+ super(message, cause);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 997c2c0..b81c327 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.ConfigurationProvider;
import org.apache.hadoop.yarn.conf.ConfigurationProviderFactory;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -37,9 +38,12 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -561,4 +565,21 @@ public class ResourceUtils {
}
return "";
}
+
+ /**
+ * Get all resource types information from known resource types.
+ * @return List of ResourceTypeInfo
+ */
+ public static List<ResourceTypeInfo> getResourcesTypeInfo() {
+ List<ResourceTypeInfo> array = new ArrayList<>();
+ // Add all resource types
+ Collection<ResourceInformation> resourcesInfo =
+ ResourceUtils.getResourceTypes().values();
+ for (ResourceInformation resourceInfo : resourcesInfo) {
+ array.add(ResourceTypeInfo
+ .newInstance(resourceInfo.getName(), resourceInfo.getUnits(),
+ resourceInfo.getResourceType()));
+ }
+ return array;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
index 4f5b5bc..60e7813 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/YarnClient.java
@@ -72,6 +72,7 @@ import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
@@ -863,7 +864,8 @@ public abstract class YarnClient extends AbstractService {
* Get the resource profiles available in the RM.
* </p>
* @return a Map of the resource profile names to their capabilities
- * @throws YarnException if resource profiles are not enabled
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other errors
*/
@Public
@@ -876,9 +878,9 @@ public abstract class YarnClient extends AbstractService {
* Get the details of a specific resource profile from the RM.
* </p>
* @param profile the profile name
- * @return the capabilities of the resource profile
- * @throws YarnException if resource profiles are not enabled or the profile
- * cannot be found
+ * @return resource profile name with its capabilities
+ * @throws YARNFeatureNotEnabledException if resource-profile is disabled
+ * @throws YarnException if any error happens inside YARN
* @throws IOException in case of other others
*/
@Public
@@ -891,8 +893,7 @@ public abstract class YarnClient extends AbstractService {
* Get available resource types supported by RM.
* </p>
* @return list of supported resource types with detailed information
- * @throws YarnException if resource profiles are not enabled or the profile
- * cannot be found
+ * @throws YarnException if any issue happens inside YARN
* @throws IOException in case of other others
*/
@Public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 1e99bc7..ffd4fec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -528,10 +528,12 @@ public class DominantResourceCalculator extends ResourceCalculator {
long value;
if (stepFactorValue != 0) {
value = roundUp
- ? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
+ ? roundUp((long) Math.ceil((float) (rValue * by)), stepFactorValue)
: roundDown((long) (rValue * by), stepFactorValue);
} else {
- value = roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
+ value = roundUp
+ ? (long) Math.ceil((float) (rValue * by))
+ : (long) (rValue * by);
}
tmp.setValue(value);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 11e53db..1a9d4c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -431,6 +431,7 @@ public class TestPBImplRecords extends BasePBImplRecordsTest {
generateByNewInstance(ApplicationTimeout.class);
generateByNewInstance(QueueConfigurations.class);
generateByNewInstance(CollectorInfo.class);
+ generateByNewInstance(ResourceTypeInfo.class);
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
index 931b1c8..7ae23e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AMSProcessingChain.java
@@ -82,7 +82,7 @@ class AMSProcessingChain implements ApplicationMasterServiceProcessor {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse resp) throws IOException {
+ RegisterApplicationMasterResponse resp) throws IOException, YarnException {
this.head.registerApplicationMaster(applicationAttemptId, request, resp);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 0d65838..d98e558 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -143,7 +143,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
-import org.apache.hadoop.yarn.exceptions.ResourceProfilesNotEnabledException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -183,6 +182,7 @@ import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UTCClock;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
@@ -1782,42 +1782,26 @@ public class ClientRMService extends AbstractService implements
GetAllResourceProfilesRequest request) throws YarnException, IOException {
GetAllResourceProfilesResponse response =
GetAllResourceProfilesResponse.newInstance();
- response.setResourceProfiles(getResourceProfiles());
+ response.setResourceProfiles(resourceProfilesManager.getResourceProfiles());
return response;
}
@Override
public GetResourceProfileResponse getResourceProfile(
GetResourceProfileRequest request) throws YarnException, IOException {
- Map<String, Resource> profiles = getResourceProfiles();
- if (!profiles.containsKey(request.getProfileName())) {
- throw new YarnException(
- "Resource profile '" + request.getProfileName() + "' not found");
- }
GetResourceProfileResponse response =
GetResourceProfileResponse.newInstance();
- response.setResource(profiles.get(request.getProfileName()));
+ response.setResource(
+ resourceProfilesManager.getProfile(request.getProfileName()));
return response;
}
- private Map<String, Resource> getResourceProfiles() throws YarnException {
- boolean resourceProfilesEnabled = getConfig()
- .getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!resourceProfilesEnabled) {
- throw new ResourceProfilesNotEnabledException(
- "Resource profiles are not enabled");
- }
- return resourceProfilesManager.getResourceProfiles();
- }
-
@Override
public GetAllResourceTypeInfoResponse getResourceTypeInfo(
GetAllResourceTypeInfoRequest request) throws YarnException, IOException {
GetAllResourceTypeInfoResponse response =
GetAllResourceTypeInfoResponse.newInstance();
- response.setResourceTypeInfo(
- resourceProfilesManager.getAllResourceTypeInfo());
+ response.setResourceTypeInfo(ResourceUtils.getResourcesTypeInfo());
return response;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
index 9c95728..5632efe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/DefaultAMSProcessor.java
@@ -114,7 +114,8 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
RMApp app = getRmContext().getRMApps().get(
applicationAttemptId.getApplicationId());
@@ -174,10 +175,11 @@ final class DefaultAMSProcessor implements ApplicationMasterServiceProcessor {
response.setSchedulerResourceTypes(getScheduler()
.getSchedulingResourceTypes());
- if (getRmContext().getYarnConfiguration().getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
- response
- .setResourceProfiles(resourceProfilesManager.getResourceProfiles());
+ if (getRmContext().getYarnConfiguration().getBoolean(
+ YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
+ YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED)) {
+ response.setResourceProfiles(
+ resourceProfilesManager.getResourceProfiles());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 4fc2916..c3ed7d5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -127,7 +127,8 @@ public class OpportunisticContainerAllocatorAMService
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
SchedulerApplicationAttempt appAttempt = ((AbstractYarnScheduler)
getScheduler()).getApplicationAttempt(applicationAttemptId);
if (appAttempt.getOpportunisticContainerContext() == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index d3e8345..2aae3a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -628,20 +628,6 @@ public class RMServerUtils {
}
}
- public static void convertProfileToResourceCapability(
- List<ResourceRequest> asks, Configuration conf,
- ResourceProfilesManager resourceProfilesManager) throws YarnException {
- boolean profilesEnabled =
- conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
- YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!profilesEnabled) {
- return;
- }
- for (ResourceRequest req : asks) {
- convertProfileToResourceCapability(req, conf, resourceProfilesManager);
- }
- }
-
public static void convertProfileToResourceCapability(ResourceRequest ask,
Configuration conf, ResourceProfilesManager resourceProfilesManager)
throws YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
index 8963121..700f0ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManager.java
@@ -18,22 +18,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.resource;
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import java.io.IOException;
-import java.util.List;
import java.util.Map;
/**
* Interface for the resource profiles manager. Provides an interface to get
* the list of available profiles and some helper functions.
*/
-@Public
-@Unstable
public interface ResourceProfilesManager {
/**
@@ -47,14 +43,19 @@ public interface ResourceProfilesManager {
* Get the resource capability associated with given profile name.
* @param profile name of resource profile
* @return resource capability for given profile
+ *
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getProfile(String profile);
+ Resource getProfile(String profile) throws YarnException;
/**
* Get all supported resource profiles.
* @return a map of resource objects associated with each profile
+ *
+ * @throws YARNFeatureNotEnabledException when feature is disabled
*/
- Map<String, Resource> getResourceProfiles();
+ Map<String, Resource> getResourceProfiles() throws
+ YARNFeatureNotEnabledException;
/**
* Reload profiles based on updated configuration.
@@ -65,24 +66,21 @@ public interface ResourceProfilesManager {
/**
* Get default supported resource profile.
* @return resource object which is default
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getDefaultProfile();
+ Resource getDefaultProfile() throws YarnException;
/**
* Get minimum supported resource profile.
* @return resource object which is minimum
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getMinimumProfile();
+ Resource getMinimumProfile() throws YarnException;
/**
* Get maximum supported resource profile.
* @return resource object which is maximum
+ * @throws YarnException when any invalid profile name or feature is disabled
*/
- Resource getMaximumProfile();
-
- /**
- * List of ResourceTypeInfo objects which carry all resources supported by RM.
- * @return list of ResourceTypeInfo objects
- */
- List<ResourceTypeInfo> getAllResourceTypeInfo();
+ Resource getMaximumProfile() throws YarnException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
index ff4e3e4..e213a97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceProfilesManagerImpl.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.api.records.ResourceTypeInfo;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YARNFeatureNotEnabledException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.codehaus.jackson.map.ObjectMapper;
@@ -52,6 +54,7 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
private List<ResourceTypeInfo> resourceTypeInfo =
new ArrayList<ResourceTypeInfo>();
private Configuration conf;
+ private boolean profileEnabled = false;
private static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
private static final String VCORES = ResourceInformation.VCORES.getName();
@@ -65,6 +68,11 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
private static final String[] MANDATORY_PROFILES =
{ DEFAULT_PROFILE, MINIMUM_PROFILE, MAXIMUM_PROFILE };
+ private static final String FEATURE_NOT_ENABLED_MSG =
+ "Resource profile is not enabled, please "
+ + "enable resource profile feature before using its functions."
+ + " (by setting " + YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED
+ + " to true)";
public ResourceProfilesManagerImpl() {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
@@ -98,10 +106,10 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
}
private void loadProfiles() throws IOException {
- boolean profilesEnabled =
+ profileEnabled =
conf.getBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED,
YarnConfiguration.DEFAULT_RM_RESOURCE_PROFILES_ENABLED);
- if (!profilesEnabled) {
+ if (!profileEnabled) {
return;
}
String sourceFile =
@@ -131,14 +139,16 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
if (entry.getValue() instanceof Map) {
Map profileInfo = (Map) entry.getValue();
// ensure memory and vcores are specified
- if (!profileInfo.containsKey(MEMORY) || !profileInfo.containsKey(VCORES)) {
+ if (!profileInfo.containsKey(MEMORY)
+ || !profileInfo.containsKey(VCORES)) {
throw new IOException(
"Illegal resource profile definition; profile '" + profileName
+ "' must contain '" + MEMORY + "' and '" + VCORES + "'");
}
Resource resource = parseResource(profileInfo);
profiles.put(profileName, resource);
- LOG.info("Added profile '" + profileName + "' with resources " + resource);
+ LOG.info(
+ "Added profile '" + profileName + "' with resources: " + resource);
}
}
// check to make sure mandatory profiles are present
@@ -149,7 +159,7 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
+ Arrays.toString(MANDATORY_PROFILES) + " must be present");
}
}
- LOG.info("Loaded profiles " + profiles.keySet());
+ LOG.info("Loaded profiles: " + profiles.keySet());
}
private Resource parseResource(Map profileInfo) throws IOException {
@@ -182,13 +192,33 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
return resource;
}
+ private void checkAndThrowExceptionWhenFeatureDisabled()
+ throws YARNFeatureNotEnabledException {
+ if (!profileEnabled) {
+ throw new YARNFeatureNotEnabledException(FEATURE_NOT_ENABLED_MSG);
+ }
+ }
+
@Override
- public Resource getProfile(String profile) {
- return Resources.clone(profiles.get(profile));
+ public Resource getProfile(String profile) throws YarnException{
+ checkAndThrowExceptionWhenFeatureDisabled();
+
+ if (profile == null) {
+ throw new YarnException("Profile name cannot be null");
+ }
+
+ Resource profileRes = profiles.get(profile);
+ if (profileRes == null) {
+ throw new YarnException(
+ "Resource profile '" + profile + "' not found");
+ }
+ return Resources.clone(profileRes);
}
@Override
- public Map<String, Resource> getResourceProfiles() {
+ public Map<String, Resource> getResourceProfiles()
+ throws YARNFeatureNotEnabledException {
+ checkAndThrowExceptionWhenFeatureDisabled();
return Collections.unmodifiableMap(profiles);
}
@@ -200,17 +230,17 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
}
@Override
- public Resource getDefaultProfile() {
+ public Resource getDefaultProfile() throws YarnException {
return getProfile(DEFAULT_PROFILE);
}
@Override
- public Resource getMinimumProfile() {
+ public Resource getMinimumProfile() throws YarnException {
return getProfile(MINIMUM_PROFILE);
}
@Override
- public Resource getMaximumProfile() {
+ public Resource getMaximumProfile() throws YarnException {
return getProfile(MAXIMUM_PROFILE);
}
@@ -220,13 +250,4 @@ public class ResourceProfilesManagerImpl implements ResourceProfilesManager {
Long.valueOf(value.substring(0, value.length() - units.length()));
return ResourceInformation.newInstance(name, units, resourceValue);
}
-
- public List<ResourceTypeInfo> getAllResourceTypeInfo() {
- try {
- readLock.lock();
- return Collections.unmodifiableList(resourceTypeInfo);
- } finally {
- readLock.unlock();
- }
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 341be69..9254640 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -58,6 +58,7 @@ import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMAppManagerEvent;
@@ -1301,7 +1302,13 @@ public abstract class AbstractYarnScheduler
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMinimumAllocation();
} else {
- ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+ try {
+ ret = rmContext.getResourceProfilesManager().getMinimumProfile();
+ } catch (YarnException e) {
+ LOG.error(
+ "Exception while getting minimum profile from profile manager:", e);
+ throw new YarnRuntimeException(e);
+ }
}
LOG.info("Minimum allocation = " + ret);
return ret;
@@ -1323,7 +1330,14 @@ public abstract class AbstractYarnScheduler
if (!profilesEnabled) {
ret = ResourceUtils.getResourceTypesMaximumAllocation();
} else {
- ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+ try {
+ ret = rmContext.getResourceProfilesManager().getMaximumProfile();
+ } catch (YarnException e) {
+ LOG.error(
+ "Exception while getting maximum profile from ResourceProfileManager:",
+ e);
+ throw new YarnRuntimeException(e);
+ }
}
LOG.info("Maximum allocation = " + ret);
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
index f72e55d..7a2e500 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationMasterService.java
@@ -103,9 +103,11 @@ public class TestApplicationMasterService {
}
@Override
- public void registerApplicationMaster(ApplicationAttemptId
- applicationAttemptId, RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ public void registerApplicationMaster(
+ ApplicationAttemptId applicationAttemptId,
+ RegisterApplicationMasterRequest request,
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
nextProcessor.registerApplicationMaster(
applicationAttemptId, request, response);
}
@@ -145,7 +147,8 @@ public class TestApplicationMasterService {
public void registerApplicationMaster(
ApplicationAttemptId applicationAttemptId,
RegisterApplicationMasterRequest request,
- RegisterApplicationMasterResponse response) throws IOException {
+ RegisterApplicationMasterResponse response)
+ throws IOException, YarnException {
beforeRegCount.incrementAndGet();
nextProcessor.registerApplicationMaster(applicationAttemptId,
request, response);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39240b61/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
index c542ed8..b53e59c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resource/TestResourceProfiles.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.resource;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.junit.Assert;
import org.junit.Test;
@@ -36,8 +37,13 @@ public class TestResourceProfiles {
Configuration conf = new Configuration();
// be default resource profiles should not be enabled
manager.init(conf);
- Map<String, Resource> profiles = manager.getResourceProfiles();
- Assert.assertTrue(profiles.isEmpty());
+ try {
+ manager.getResourceProfiles();
+ Assert
+ .fail("Exception should be thrown as resource profile is not enabled"
+ + " and getResourceProfiles is invoked.");
+ } catch (YarnException ie) {
+ }
conf.setBoolean(YarnConfiguration.RM_RESOURCE_PROFILES_ENABLED, true);
try {
manager.init(conf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org