You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by te...@apache.org on 2017/01/07 16:25:03 UTC
hadoop git commit: YARN-6022. Revert changes of
AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton)
Repository: hadoop
Updated Branches:
refs/heads/trunk a59df1575 -> ac16400e1
YARN-6022. Revert changes of AbstractResourceRequest (Contributed by Wangda Tan via Daniel Templeton)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ac16400e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ac16400e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ac16400e
Branch: refs/heads/trunk
Commit: ac16400e1fb85a4186e5bf5bbc9cf204735ae74f
Parents: a59df15
Author: Daniel Templeton <te...@apache.org>
Authored: Sat Jan 7 08:23:06 2017 -0800
Committer: Daniel Templeton <te...@apache.org>
Committed: Sat Jan 7 08:24:30 2017 -0800
----------------------------------------------------------------------
.../api/records/AbstractResourceRequest.java | 55 --------------------
.../yarn/api/records/ResourceRequest.java | 19 ++++++-
.../api/records/UpdateContainerRequest.java | 18 ++++++-
.../server/scheduler/SchedulerRequestKey.java | 1 -
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 2 +-
.../scheduler/AbstractYarnScheduler.java | 7 ++-
.../scheduler/SchedulerUtils.java | 18 ++++---
.../scheduler/YarnScheduler.java | 6 +--
.../scheduler/fair/FairScheduler.java | 5 +-
10 files changed, 55 insertions(+), 78 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
deleted file mode 100644
index 819a607..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AbstractResourceRequest.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Public;
-import org.apache.hadoop.classification.InterfaceStability.Stable;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-
-/**
- * {@code AbstractResourceRequest} represents a generic resource request made
- * by an application to the {@code ResourceManager}.
- * <p>
- * It includes:
- * <ul>
- * <li>{@link Resource} capability required for each request.</li>
- * </ul>
- *
- * @see Resource
- */
-@Public
-@Unstable
-public abstract class AbstractResourceRequest {
-
- /**
- * Set the <code>Resource</code> capability of the request
- * @param capability <code>Resource</code> capability of the request
- */
- @Public
- @Stable
- public abstract void setCapability(Resource capability);
-
- /**
- * Get the <code>Resource</code> capability of the request.
- * @return <code>Resource</code> capability of the request
- */
- @Public
- @Stable
- public abstract Resource getCapability();
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index ce85b21..5bedc87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -58,8 +58,7 @@ import org.apache.hadoop.yarn.util.Records;
*/
@Public
@Stable
-public abstract class ResourceRequest extends AbstractResourceRequest
- implements Comparable<ResourceRequest>{
+public abstract class ResourceRequest implements Comparable<ResourceRequest> {
@Public
@Stable
@@ -506,6 +505,22 @@ public abstract class ResourceRequest extends AbstractResourceRequest
public void setAllocationRequestId(long allocationRequestID) {
throw new UnsupportedOperationException();
}
+
+ /**
+ * Set the <code>Resource</code> capability of the request.
+ * @param capability <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the <code>Resource</code> capability of the request.
+ * @return <code>Resource</code> capability of the request
+ */
+ @Public
+ @Stable
+ public abstract Resource getCapability();
@Override
public int hashCode() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
index 925a7979..2060370 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/UpdateContainerRequest.java
@@ -62,7 +62,7 @@ import org.apache.hadoop.yarn.util.Records;
*/
@InterfaceAudience.Public
@InterfaceStability.Unstable
-public abstract class UpdateContainerRequest extends AbstractResourceRequest {
+public abstract class UpdateContainerRequest {
@InterfaceAudience.Public
@InterfaceStability.Unstable
@@ -143,6 +143,22 @@ public abstract class UpdateContainerRequest extends AbstractResourceRequest {
@InterfaceStability.Unstable
public abstract void setExecutionType(ExecutionType executionType);
+ /**
+ * Set the <code>Resource</code> capability of the request.
+ * @param capability <code>Resource</code> capability of the request
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract void setCapability(Resource capability);
+
+ /**
+ * Get the <code>Resource</code> capability of the request.
+ * @return <code>Resource</code> capability of the request
+ */
+ @InterfaceAudience.Public
+ @InterfaceStability.Unstable
+ public abstract Resource getCapability();
+
@Override
public int hashCode() {
final int prime = 2153;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
index 36a9149..02539ba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/scheduler/SchedulerRequestKey.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.scheduler;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Priority;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index 9dd544e..8232b88 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -459,7 +459,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
throw e;
}
- scheduler.normalizeRequest(amReq);
+ amReq.setCapability(scheduler.getNormalizedResource(amReq.getCapability()));
return amReq;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index ebbeb0f..94bfd58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -337,7 +337,7 @@ public class RMServerUtils {
return false;
}
ResourceScheduler scheduler = rmContext.getScheduler();
- scheduler.normalizeRequest(request);
+ request.setCapability(scheduler.getNormalizedResource(request.getCapability()));
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index acfcde8..89f9ffa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -1044,8 +1043,8 @@ public abstract class AbstractYarnScheduler
}
@Override
- public void normalizeRequest(AbstractResourceRequest ask) {
- SchedulerUtils.normalizeRequest(ask,
+ public Resource getNormalizedResource(Resource requestedResource) {
+ return SchedulerUtils.getNormalizedResource(requestedResource,
getResourceCalculator(),
getMinimumResourceCapability(),
getMaximumResourceCapability(),
@@ -1059,7 +1058,7 @@ public abstract class AbstractYarnScheduler
*/
protected void normalizeRequests(List<ResourceRequest> asks) {
for (ResourceRequest ask: asks) {
- normalizeRequest(ask);
+ ask.setCapability(getNormalizedResource(ask.getCapability()));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index 5360665..148f023 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.io.IOException;
import java.util.Set;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
@@ -33,7 +34,6 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.InvalidLabelResourceRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
@@ -134,29 +134,33 @@ public class SchedulerUtils {
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of minMemory and is not zero.
*/
+ @VisibleForTesting
public static void normalizeRequest(
ResourceRequest ask,
ResourceCalculator resourceCalculator,
Resource minimumResource,
Resource maximumResource) {
- normalizeRequest(ask, resourceCalculator,
- minimumResource, maximumResource, minimumResource);
+ ask.setCapability(
+ getNormalizedResource(ask.getCapability(), resourceCalculator,
+ minimumResource, maximumResource, minimumResource));
}
/**
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of increment resource and is not zero.
+ *
+ * @return normalized resource
*/
- public static void normalizeRequest(
- AbstractResourceRequest ask,
+ public static Resource getNormalizedResource(
+ Resource ask,
ResourceCalculator resourceCalculator,
Resource minimumResource,
Resource maximumResource,
Resource incrementResource) {
Resource normalized = Resources.normalize(
- resourceCalculator, ask.getCapability(), minimumResource,
+ resourceCalculator, ask, minimumResource,
maximumResource, incrementResource);
- ask.setCapability(normalized);
+ return normalized;
}
private static void normalizeNodeLabelExpressionInRequest(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
index 3185dc1..5579efd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/YarnScheduler.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
@@ -381,7 +380,8 @@ public interface YarnScheduler extends EventHandler<SchedulerEvent> {
/**
* Normalize a resource request.
*
- * @param request the resource request to be normalized
+ * @param requestedResource the resource to be normalized
+ * @return the normalized resource
*/
- void normalizeRequest(AbstractResourceRequest request);
+ Resource getNormalizedResource(Resource requestedResource);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ac16400e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 78dfe61..6bbccc6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -27,7 +27,6 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
-import org.apache.hadoop.yarn.api.records.AbstractResourceRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
@@ -799,8 +798,8 @@ public class FairScheduler extends
}
@Override
- public void normalizeRequest(AbstractResourceRequest ask) {
- SchedulerUtils.normalizeRequest(ask,
+ public Resource getNormalizedResource(Resource requestedResource) {
+ return SchedulerUtils.getNormalizedResource(requestedResource,
DOMINANT_RESOURCE_CALCULATOR,
minimumAllocation,
getMaximumResourceCapability(),
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org