You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ac...@apache.org on 2013/01/09 06:17:34 UTC
svn commit: r1430683 [1/3] - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/...
Author: acmurthy
Date: Wed Jan 9 05:17:31 2013
New Revision: 1430683
URL: http://svn.apache.org/viewvc?rev=1430683&view=rev
Log:
Merge -c 1430682 from trunk to branch-2 to fix YARN-2. Enhanced CapacityScheduler to account for CPU alongwith memory for multi-dimensional resource scheduling. Contributed by Arun C. Murthy.
Added:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
- copied unchanged from r1430682, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DefaultResourceCalculator.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
- copied unchanged from r1430682, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/DominantResourceCalculator.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
- copied unchanged from r1430682, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/ResourceCalculator.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Resources.java
- copied unchanged from r1430682, hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/Resources.java
Removed:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resource.java
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/SchedulingAlgorithms.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/NodeManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationACLs.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMTokens.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestQueueMetrics.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed Jan 9 05:17:31 2013
@@ -15,6 +15,9 @@ Release 2.0.3-alpha - Unreleased
YARN-286. Add a YARN ApplicationClassLoader. (tomwhite)
+ YARN-2. Enhanced CapacityScheduler to account for CPU alongwith memory for
+ multi-dimensional resource scheduling. (acmurthy)
+
IMPROVEMENTS
YARN-223. Update process tree instead of getting new process trees.
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java Wed Jan 9 05:17:31 2013
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.AMRMProtocol;
@@ -54,11 +55,43 @@ public abstract class Resource implement
@Stable
public abstract void setMemory(int memory);
+
+ /**
+ * Get <em>number of virtual cpu cores</em> of the resource.
+ *
+ * We refer to <em>virtual cores</em> to clarify that these represent
+ * <em>normalized</em> cores which may have a m:n relationship w.r.t
+ * physical cores available on the compute nodes. Furthermore, they also
+ * represent <em>idealized</em> cores since the cluster might be composed
+ * of <em>heterogenous</em> nodes.
+ *
+ * @return <em>num of virtual cpu cores</em> of the resource
+ */
+ @Public
+ @Evolving
+ public abstract int getVirtualCores();
+
+ /**
+ * Set <em>number of virtual cpu cores</em> of the resource.
+ *
+ * We refer to <em>virtual cores</em> to clarify that these represent
+ * <em>normalized</em> cores which may have a m:n relationship w.r.t
+ * physical cores available on the compute nodes. Furthermore, they also
+ * represent <em>idealized</em> cores since the cluster might be composed
+ * of <em>heterogenous</em> nodes.
+ *
+ * @param vCores <em>number of virtual cpu cores</em> of the resource
+ */
+ @Public
+ @Evolving
+ public abstract void setVirtualCores(int vCores);
+
@Override
public int hashCode() {
- final int prime = 31;
- int result = 1;
- result = prime * result + getMemory();
+ final int prime = 263167;
+ int result = 3571;
+ result = 939769357 + getMemory(); // prime * result = 939769357 initially
+ result = prime * result + getVirtualCores();
return result;
}
@@ -68,16 +101,18 @@ public abstract class Resource implement
return true;
if (obj == null)
return false;
- if (getClass() != obj.getClass())
+ if (!(obj instanceof Resource))
return false;
Resource other = (Resource) obj;
- if (getMemory() != other.getMemory())
+ if (getMemory() != other.getMemory() ||
+ getVirtualCores() != other.getVirtualCores()) {
return false;
+ }
return true;
}
@Override
public String toString() {
- return "memory: " + getMemory();
+ return "<memory:" + getMemory() + ", vCores:" + getVirtualCores() + ">";
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java Wed Jan 9 05:17:31 2013
@@ -64,8 +64,24 @@ public class ResourcePBImpl extends Reso
}
@Override
+ public int getVirtualCores() {
+ ResourceProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getVirtualCores());
+ }
+
+ @Override
+ public void setVirtualCores(int vCores) {
+ maybeInitBuilder();
+ builder.setVirtualCores((vCores));
+ }
+
+ @Override
public int compareTo(Resource other) {
- return this.getMemory() - other.getMemory();
+ int diff = this.getMemory() - other.getMemory();
+ if (diff == 0) {
+ diff = this.getVirtualCores() - other.getVirtualCores();
+ }
+ return diff;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Wed Jan 9 05:17:31 2013
@@ -48,6 +48,7 @@ message ContainerIdProto {
message ResourceProto {
optional int32 memory = 1;
+ optional int32 virtual_cores = 2;
}
message PriorityProto {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Wed Jan 9 05:17:31 2013
@@ -29,10 +29,9 @@ import org.apache.hadoop.util.StringUtil
import org.apache.hadoop.yarn.api.ApplicationConstants;
import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
public class YarnConfiguration extends Configuration {
- private static final Splitter ADDR_SPLITTER = Splitter.on(':').trimResults();
+
private static final Joiner JOINER = Joiner.on("");
private static final String YARN_DEFAULT_XML_FILE = "yarn-default.xml";
@@ -111,15 +110,21 @@ public class YarnConfiguration extends C
public static final String DEFAULT_RM_SCHEDULER_ADDRESS = "0.0.0.0:" +
DEFAULT_RM_SCHEDULER_PORT;
- /** Miniumum memory request grant-able by the RM scheduler. */
+ /** Miniumum request grant-able by the RM scheduler. */
public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_MB =
YARN_PREFIX + "scheduler.minimum-allocation-mb";
public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB = 1024;
+ public static final String RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES =
+ YARN_PREFIX + "scheduler.minimum-allocation-vcores";
+ public static final int DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES = 1;
- /** Maximum memory request grant-able by the RM scheduler. */
+ /** Maximum request grant-able by the RM scheduler. */
public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_MB =
YARN_PREFIX + "scheduler.maximum-allocation-mb";
public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB = 8192;
+ public static final String RM_SCHEDULER_MAXIMUM_ALLOCATION_CORES =
+ YARN_PREFIX + "scheduler.maximum-allocation-vcores";
+ public static final int DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_CORES = 32;
/** Number of threads to handle scheduler interface.*/
public static final String RM_SCHEDULER_CLIENT_THREAD_COUNT =
@@ -408,10 +413,20 @@ public class YarnConfiguration extends C
public static final String NM_PMEM_MB = NM_PREFIX + "resource.memory-mb";
public static final int DEFAULT_NM_PMEM_MB = 8 * 1024;
+ /** Conversion ratio for physical memory to virtual memory. */
public static final String NM_VMEM_PMEM_RATIO =
NM_PREFIX + "vmem-pmem-ratio";
public static final float DEFAULT_NM_VMEM_PMEM_RATIO = 2.1f;
+ /** Number of Physical CPU Cores which can be allocated for containers.*/
+ public static final String NM_VCORES = NM_PREFIX + "resource.cpu-cores";
+ public static final int DEFAULT_NM_VCORES = 8;
+
+ /** Conversion ratio for physical cores to virtual cores. */
+ public static final String NM_VCORES_PCORES_RATIO =
+ NM_PREFIX + "vcores-pcores-ratio";
+ public static final float DEFAULT_NM_VCORES_PCORES_RATIO = 2.0f;
+
/** NM Webapp address.**/
public static final String NM_WEBAPP_ADDRESS = NM_PREFIX + "webapp.address";
public static final int DEFAULT_NM_WEBAPP_PORT = 8042;
@@ -657,4 +672,5 @@ public class YarnConfiguration extends C
public static String getRMWebAppURL(Configuration conf) {
return JOINER.join("http://", getRMWebAppHostAndPort(conf));
}
+
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Wed Jan 9 05:17:31 2013
@@ -106,6 +106,7 @@ public class ContainerTokenIdentifier ex
out.writeUTF(this.nmHostAddr);
out.writeUTF(this.appSubmitter);
out.writeInt(this.resource.getMemory());
+ out.writeInt(this.resource.getVirtualCores());
out.writeLong(this.expiryTimeStamp);
out.writeInt(this.masterKeyId);
}
@@ -120,7 +121,9 @@ public class ContainerTokenIdentifier ex
.readInt());
this.nmHostAddr = in.readUTF();
this.appSubmitter = in.readUTF();
- this.resource = BuilderUtils.newResource(in.readInt());
+ int memory = in.readInt();
+ int vCores = in.readInt();
+ this.resource = BuilderUtils.newResource(memory, vCores);
this.expiryTimeStamp = in.readLong();
this.masterKeyId = in.readInt();
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/BuilderUtils.java Wed Jan 9 05:17:31 2013
@@ -371,9 +371,10 @@ public class BuilderUtils {
return report;
}
- public static Resource newResource(int memory) {
+ public static Resource newResource(int memory, int vCores) {
Resource resource = recordFactory.newRecordInstance(Resource.class);
resource.setMemory(memory);
+ resource.setVirtualCores(vCores);
return resource;
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Wed Jan 9 05:17:31 2013
@@ -189,7 +189,7 @@
</property>
<property>
- <description>The minimum allocation size for every container request at the RM,
+ <description>The minimum allocation for every container request at the RM,
in MBs. Memory requests lower than this won't take effect,
and the specified value will get allocated at minimum.</description>
<name>yarn.scheduler.minimum-allocation-mb</name>
@@ -197,7 +197,7 @@
</property>
<property>
- <description>The maximum allocation size for every container request at the RM,
+ <description>The maximum allocation for every container request at the RM,
in MBs. Memory requests higher than this won't take effect,
and will get capped to this value.</description>
<name>yarn.scheduler.maximum-allocation-mb</name>
@@ -205,6 +205,22 @@
</property>
<property>
+ <description>The minimum allocation for every container request at the RM,
+ in terms of virtual CPU cores. Requests lower than this won't take effect,
+ and the specified value will get allocated the minimum.</description>
+ <name>yarn.scheduler.minimum-allocation-vcores</name>
+ <value>1</value>
+ </property>
+
+ <property>
+ <description>The maximum allocation for every container request at the RM,
+ in terms of virtual CPU cores. Requests higher than this won't take effect,
+ and will get capped to this value.</description>
+ <name>yarn.scheduler.maximum-allocation-vcores</name>
+ <value>32</value>
+ </property>
+
+ <property>
<description>Enable RM to recover state after starting. If true, then
yarn.resourcemanager.store.class must be specified</description>
<name>yarn.resourcemanager.recovery.enabled</name>
@@ -404,7 +420,6 @@
<value>logs</value>
</property>
-
<property>
<description>Amount of physical memory, in MB, that can be allocated
for containers.</description>
@@ -423,6 +438,21 @@
</property>
<property>
+ <description>Number of CPU cores that can be allocated
+ for containers.</description>
+ <name>yarn.nodemanager.resource.cpu-cores</name>
+ <value>8</value>
+ </property>
+
+ <property>
+ <description>Ratio between virtual cores to physical cores when
+ allocating CPU resources to containers.
+ </description>
+ <name>yarn.nodemanager.vcores-pcores-ratio</name>
+ <value>2</value>
+ </property>
+
+ <property>
<description>NM Webapp address.</description>
<name>yarn.nodemanager.webapp.address</name>
<value>0.0.0.0:8042</value>
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Wed Jan 9 05:17:31 2013
@@ -106,14 +106,37 @@ public class NodeStatusUpdaterImpl exten
this.heartBeatInterval =
conf.getLong(YarnConfiguration.NM_TO_RM_HEARTBEAT_INTERVAL_MS,
YarnConfiguration.DEFAULT_NM_TO_RM_HEARTBEAT_INTERVAL_MS);
- int memoryMb = conf.getInt(YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
+ int memoryMb =
+ conf.getInt(
+ YarnConfiguration.NM_PMEM_MB, YarnConfiguration.DEFAULT_NM_PMEM_MB);
+ float vMemToPMem =
+ conf.getFloat(
+ YarnConfiguration.NM_VMEM_PMEM_RATIO,
+ YarnConfiguration.DEFAULT_NM_VMEM_PMEM_RATIO);
+ int virtualMemoryMb = (int)Math.ceil(memoryMb * vMemToPMem);
+
+ int cpuCores =
+ conf.getInt(
+ YarnConfiguration.NM_VCORES, YarnConfiguration.DEFAULT_NM_VCORES);
+ float vCoresToPCores =
+ conf.getFloat(
+ YarnConfiguration.NM_VCORES_PCORES_RATIO,
+ YarnConfiguration.DEFAULT_NM_VCORES_PCORES_RATIO);
+ int virtualCores = (int)Math.ceil(cpuCores * vCoresToPCores);
+
this.totalResource = recordFactory.newRecordInstance(Resource.class);
this.totalResource.setMemory(memoryMb);
+ this.totalResource.setVirtualCores(virtualCores);
metrics.addResource(totalResource);
this.tokenKeepAliveEnabled = isTokenKeepAliveEnabled(conf);
this.tokenRemovalDelayMs =
conf.getInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS,
YarnConfiguration.DEFAULT_RM_NM_EXPIRY_INTERVAL_MS);
+
+ LOG.info("Initialized nodemanager for " + nodeId + ":" +
+ " physical-memory=" + memoryMb + " virtual-memory=" + virtualMemoryMb +
+ " physical-cores=" + cpuCores + " virtual-cores=" + virtualCores);
+
super.init(conf);
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java Wed Jan 9 05:17:31 2013
@@ -574,7 +574,7 @@ public class TestContainer {
when(ctxt.getUser()).thenReturn(this.user);
when(ctxt.getContainerId()).thenReturn(cId);
- Resource resource = BuilderUtils.newResource(1024);
+ Resource resource = BuilderUtils.newResource(1024, 1);
when(ctxt.getResource()).thenReturn(resource);
if (withLocalRes) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/conf/capacity-scheduler.xml Wed Jan 9 05:17:31 2013
@@ -19,6 +19,18 @@
</property>
<property>
+ <name>yarn.scheduler.capacity.resource-calculator</name>
+ <value>org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator</value>
+ <description>
+ The ResourceCalculator implementation to be used to compare
+ Resources in the scheduler.
+ The default i.e. DefaultResourceCalculator only uses Memory while
+ DominantResourceCalculator uses dominant-resource to compare
+ multi-dimensional resources such as Memory, CPU etc.
+ </description>
+ </property>
+
+ <property>
<name>yarn.scheduler.capacity.root.queues</name>
<value>default</value>
<description>
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java Wed Jan 9 05:17:31 2013
@@ -297,7 +297,7 @@ public class ApplicationMasterService ex
for(RMNode rmNode: updatedNodes) {
SchedulerNodeReport schedulerNodeReport =
rScheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0);
+ Resource used = BuilderUtils.newResource(0, 0);
int numContainers = 0;
if (schedulerNodeReport != null) {
used = schedulerNodeReport.getUsedResource();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java Wed Jan 9 05:17:31 2013
@@ -441,7 +441,7 @@ public class ClientRMService extends Abs
private NodeReport createNodeReports(RMNode rmNode) {
SchedulerNodeReport schedulerNodeReport =
scheduler.getNodeReport(rmNode.getNodeID());
- Resource used = BuilderUtils.newResource(0);
+ Resource used = BuilderUtils.newResource(0, 0);
int numContainers = 0;
if (schedulerNodeReport != null) {
used = schedulerNodeReport.getUsedResource();
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java Wed Jan 9 05:17:31 2013
@@ -193,7 +193,7 @@ public class ResourceTrackerService exte
LOG.info("NodeManager from node " + host + "(cmPort: " + cmPort
+ " httpPort: " + httpPort + ") " + "registered with capability: "
- + capability.getMemory() + ", assigned nodeId " + nodeId);
+ + capability + ", assigned nodeId " + nodeId);
regResponse.setNodeAction(NodeAction.NORMAL);
response.setRegistrationResponse(regResponse);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/resource/Resources.java Wed Jan 9 05:17:31 2013
@@ -41,15 +41,34 @@ public class Resources {
}
@Override
+ public int getVirtualCores() {
+ return 0;
+ }
+
+ @Override
+ public void setVirtualCores(int cores) {
+ throw new RuntimeException("NONE cannot be modified!");
+ }
+
+ @Override
public int compareTo(Resource o) {
- return (0 - o.getMemory());
+ int diff = 0 - o.getMemory();
+ if (diff == 0) {
+ diff = 0 - o.getVirtualCores();
+ }
+ return diff;
}
};
public static Resource createResource(int memory) {
+ return createResource(memory, (memory > 0) ? 1 : 0);
+ }
+
+ public static Resource createResource(int memory, int cores) {
Resource resource = Records.newRecord(Resource.class);
resource.setMemory(memory);
+ resource.setVirtualCores(cores);
return resource;
}
@@ -58,11 +77,12 @@ public class Resources {
}
public static Resource clone(Resource res) {
- return createResource(res.getMemory());
+ return createResource(res.getMemory(), res.getVirtualCores());
}
public static Resource addTo(Resource lhs, Resource rhs) {
lhs.setMemory(lhs.getMemory() + rhs.getMemory());
+ lhs.setVirtualCores(lhs.getVirtualCores() + rhs.getVirtualCores());
return lhs;
}
@@ -72,6 +92,7 @@ public class Resources {
public static Resource subtractFrom(Resource lhs, Resource rhs) {
lhs.setMemory(lhs.getMemory() - rhs.getMemory());
+ lhs.setVirtualCores(lhs.getVirtualCores() - rhs.getVirtualCores());
return lhs;
}
@@ -83,50 +104,107 @@ public class Resources {
return subtract(NONE, resource);
}
- public static Resource multiplyTo(Resource lhs, int by) {
- lhs.setMemory(lhs.getMemory() * by);
+ public static Resource multiplyTo(Resource lhs, double by) {
+ lhs.setMemory((int)(lhs.getMemory() * by));
+ lhs.setVirtualCores((int)(lhs.getVirtualCores() * by));
return lhs;
}
- public static Resource multiply(Resource lhs, int by) {
+ public static Resource multiply(Resource lhs, double by) {
return multiplyTo(clone(lhs), by);
}
- /**
- * Mutliply a resource by a {@code double}. Note that integral
- * resource quantites are subject to rounding during cast.
- */
- public static Resource multiply(Resource lhs, double by) {
+ public static Resource multiplyAndNormalizeUp(
+ ResourceCalculator calculator,Resource lhs, double by, Resource factor) {
+ return calculator.multiplyAndNormalizeUp(lhs, by, factor);
+ }
+
+ public static Resource multiplyAndNormalizeDown(
+ ResourceCalculator calculator,Resource lhs, double by, Resource factor) {
+ return calculator.multiplyAndNormalizeDown(lhs, by, factor);
+ }
+
+ public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- out.setMemory((int) (lhs.getMemory() * by));
+ out.setMemory((int)(lhs.getMemory() * by));
+ out.setVirtualCores((int)(lhs.getVirtualCores() * by));
return out;
}
-
+
+ public static Resource normalize(
+ ResourceCalculator calculator, Resource lhs, Resource factor) {
+ return calculator.normalize(lhs, factor);
+ }
+
+ public static Resource roundUp(
+ ResourceCalculator calculator, Resource lhs, Resource factor) {
+ return calculator.roundUp(lhs, factor);
+ }
+
+ public static Resource roundDown(
+ ResourceCalculator calculator, Resource lhs, Resource factor) {
+ return calculator.roundDown(lhs, factor);
+ }
+
+ public static float ratio(
+ ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) {
+ return resourceCalculator.ratio(lhs, rhs);
+ }
+
+ public static float divide(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource, Resource lhs, Resource rhs) {
+ return resourceCalculator.divide(clusterResource, lhs, rhs);
+ }
+
+ public static Resource divideAndCeil(
+ ResourceCalculator resourceCalculator, Resource lhs, int rhs) {
+ return resourceCalculator.divideAndCeil(lhs, rhs);
+ }
+
public static boolean equals(Resource lhs, Resource rhs) {
- return lhs.getMemory() == rhs.getMemory();
+ return lhs.equals(rhs);
}
- public static boolean lessThan(Resource lhs, Resource rhs) {
- return lhs.getMemory() < rhs.getMemory();
+ public static boolean lessThan(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource lhs, Resource rhs) {
+ return (resourceCalculator.compare(clusterResource, lhs, rhs) < 0);
}
- public static boolean lessThanOrEqual(Resource lhs, Resource rhs) {
- return lhs.getMemory() <= rhs.getMemory();
+ public static boolean lessThanOrEqual(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource lhs, Resource rhs) {
+ return (resourceCalculator.compare(clusterResource, lhs, rhs) <= 0);
}
- public static boolean greaterThan(Resource lhs, Resource rhs) {
- return lhs.getMemory() > rhs.getMemory();
+ public static boolean greaterThan(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource lhs, Resource rhs) {
+ return resourceCalculator.compare(clusterResource, lhs, rhs) > 0;
}
- public static boolean greaterThanOrEqual(Resource lhs, Resource rhs) {
- return lhs.getMemory() >= rhs.getMemory();
+ public static boolean greaterThanOrEqual(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource lhs, Resource rhs) {
+ return resourceCalculator.compare(clusterResource, lhs, rhs) >= 0;
}
- public static Resource min(Resource lhs, Resource rhs) {
- return (lhs.getMemory() < rhs.getMemory()) ? lhs : rhs;
+ public static Resource min(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource lhs, Resource rhs) {
+ return resourceCalculator.compare(clusterResource, lhs, rhs) <= 0 ? lhs : rhs;
}
- public static Resource max(Resource lhs, Resource rhs) {
- return (lhs.getMemory() > rhs.getMemory()) ? lhs : rhs;
+ public static Resource max(
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource lhs, Resource rhs) {
+ return resourceCalculator.compare(clusterResource, lhs, rhs) >= 0 ? lhs : rhs;
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java Wed Jan 9 05:17:31 2013
@@ -207,8 +207,8 @@ public class RMAppImpl implements RMApp,
private static final ApplicationResourceUsageReport
DUMMY_APPLICATION_RESOURCE_USAGE_REPORT =
BuilderUtils.newApplicationResourceUsageReport(-1, -1,
- Resources.createResource(-1), Resources.createResource(-1),
- Resources.createResource(-1));
+ Resources.createResource(-1, -1), Resources.createResource(-1, -1),
+ Resources.createResource(-1, -1));
private static final int DUMMY_APPLICATION_ATTEMPT_NUMBER = -1;
public RMAppImpl(ApplicationId applicationId, RMContext rmContext,
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java Wed Jan 9 05:17:31 2013
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -591,8 +592,9 @@ public class RMAppAttemptImpl implements
try {
int numUsedContainers = 0;
int numReservedContainers = 0;
- int reservedResources = 0;
- int currentConsumption = 0;
+ Resource currentConsumption = Resources.createResource(0, 0);
+ Resource reservedResources = Resources.createResource(0, 0);
+
SchedulerAppReport schedApp =
scheduler.getSchedulerAppInfo(this.getAppAttemptId());
Collection<RMContainer> liveContainers;
@@ -603,22 +605,21 @@ public class RMAppAttemptImpl implements
if (liveContainers != null) {
numUsedContainers = liveContainers.size();
for (RMContainer lc : liveContainers) {
- currentConsumption += lc.getContainer().getResource().getMemory();
+ Resources.addTo(currentConsumption, lc.getContainer().getResource());
}
}
if (reservedContainers != null) {
numReservedContainers = reservedContainers.size();
for (RMContainer rc : reservedContainers) {
- reservedResources += rc.getContainer().getResource().getMemory();
+ Resources.addTo(reservedResources, rc.getContainer().getResource());
}
}
}
return BuilderUtils.newApplicationResourceUsageReport(
numUsedContainers, numReservedContainers,
- Resources.createResource(currentConsumption),
- Resources.createResource(reservedResources),
- Resources.createResource(currentConsumption + reservedResources));
+ currentConsumption, reservedResources,
+ Resources.add(currentConsumption, reservedResources));
} finally {
this.readLock.unlock();
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/QueueMetrics.java Wed Jan 9 05:17:31 2013
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import static org.apache.hadoop.metrics2.lib.Interns.info;
-import static org.apache.hadoop.yarn.server.resourcemanager.resource.Resources.multiply;
import java.util.ArrayList;
import java.util.HashMap;
@@ -42,6 +41,7 @@ import org.apache.hadoop.metrics2.lib.Mu
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -325,7 +325,7 @@ public class QueueMetrics implements Met
allocatedContainers.incr(containers);
aggregateContainersAllocated.incr(containers);
allocatedMB.incr(res.getMemory() * containers);
- _decrPendingResources(containers, multiply(res, containers));
+ _decrPendingResources(containers, Resources.multiply(res, containers));
QueueMetrics userMetrics = getUserMetrics(user);
if (userMetrics != null) {
userMetrics.allocateResources(user, containers, res);
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java Wed Jan 9 05:17:31 2013
@@ -24,10 +24,13 @@ import org.apache.hadoop.classification.
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
/**
* Utilities shared by schedulers.
@@ -81,32 +84,31 @@ public class SchedulerUtils {
/**
* Utility method to normalize a list of resource requests, by insuring that
* the memory for each request is a multiple of minMemory and is not zero.
- *
- * @param asks
- * a list of resource requests.
- * @param minMemory
- * the configured minimum memory allocation.
*/
- public static void normalizeRequests(List<ResourceRequest> asks,
- int minMemory) {
+ public static void normalizeRequests(
+ List<ResourceRequest> asks,
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource minimumResource) {
for (ResourceRequest ask : asks) {
- normalizeRequest(ask, minMemory);
+ normalizeRequest(
+ ask, resourceCalculator, clusterResource, minimumResource);
}
}
/**
* Utility method to normalize a resource request, by insuring that the
* requested memory is a multiple of minMemory and is not zero.
- *
- * @param ask
- * the resource request.
- * @param minMemory
- * the configured minimum memory allocation.
*/
- public static void normalizeRequest(ResourceRequest ask, int minMemory) {
- int memory = Math.max(ask.getCapability().getMemory(), minMemory);
- ask.getCapability().setMemory(
- minMemory * ((memory / minMemory) + (memory % minMemory > 0 ? 1 : 0)));
+ public static void normalizeRequest(
+ ResourceRequest ask,
+ ResourceCalculator resourceCalculator,
+ Resource clusterResource,
+ Resource minimumResource) {
+ Resource normalized =
+ Resources.normalize(
+ resourceCalculator, ask.getCapability(), minimumResource);
+ ask.setCapability(normalized);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java Wed Jan 9 05:17:31 2013
@@ -19,6 +19,7 @@ package org.apache.hadoop.yarn.server.re
import org.apache.hadoop.yarn.Lock;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
class CSQueueUtils {
@@ -51,15 +52,19 @@ class CSQueueUtils {
return (parentAbsMaxCapacity * maximumCapacity);
}
- public static int computeMaxActiveApplications(Resource clusterResource,
- Resource minimumAllocation, float maxAMResourcePercent,
- float absoluteMaxCapacity) {
- return
+ public static int computeMaxActiveApplications(
+ ResourceCalculator calculator,
+ Resource clusterResource, Resource minimumAllocation,
+ float maxAMResourcePercent, float absoluteMaxCapacity) {
+ return
Math.max(
(int)Math.ceil(
- ((float)clusterResource.getMemory() /
- minimumAllocation.getMemory()) *
- maxAMResourcePercent * absoluteMaxCapacity),
+ Resources.ratio(
+ calculator,
+ clusterResource,
+ minimumAllocation) *
+ maxAMResourcePercent * absoluteMaxCapacity
+ ),
1);
}
@@ -73,36 +78,43 @@ class CSQueueUtils {
@Lock(CSQueue.class)
public static void updateQueueStatistics(
+ final ResourceCalculator calculator,
final CSQueue childQueue, final CSQueue parentQueue,
final Resource clusterResource, final Resource minimumAllocation) {
- final int clusterMemory = clusterResource.getMemory();
- final int usedMemory = childQueue.getUsedResources().getMemory();
+ Resource queueLimit = Resources.none();
+ Resource usedResources = childQueue.getUsedResources();
- float queueLimit = 0.0f;
float absoluteUsedCapacity = 0.0f;
float usedCapacity = 0.0f;
- if (clusterMemory > 0) {
- queueLimit = clusterMemory * childQueue.getAbsoluteCapacity();
- absoluteUsedCapacity = ((float)usedMemory / (float)clusterMemory);
- usedCapacity = (queueLimit == 0) ? 0 : (usedMemory / queueLimit);
+
+ if (Resources.greaterThan(
+ calculator, clusterResource, clusterResource, Resources.none())) {
+ queueLimit =
+ Resources.multiply(clusterResource, childQueue.getAbsoluteCapacity());
+ absoluteUsedCapacity =
+ Resources.divide(calculator, clusterResource,
+ usedResources, clusterResource);
+ usedCapacity =
+ Resources.equals(queueLimit, Resources.none()) ? 0 :
+ Resources.divide(calculator, clusterResource,
+ usedResources, queueLimit);
}
childQueue.setUsedCapacity(usedCapacity);
childQueue.setAbsoluteUsedCapacity(absoluteUsedCapacity);
- int available =
- Math.max((roundUp(minimumAllocation, (int)queueLimit) - usedMemory), 0);
+ Resource available =
+ Resources.roundUp(
+ calculator,
+ Resources.subtract(queueLimit, usedResources),
+ minimumAllocation);
childQueue.getMetrics().setAvailableResourcesToQueue(
- Resources.createResource(available));
- }
-
- public static int roundUp(Resource minimumAllocation, int memory) {
- int minMemory = minimumAllocation.getMemory();
- return LeafQueue.divideAndCeil(memory, minMemory) * minMemory;
- }
-
- public static int roundDown(Resource minimumAllocation, int memory) {
- int minMemory = minimumAllocation.getMemory();
- return (memory / minMemory) * minMemory;
- }
+ Resources.max(
+ calculator,
+ clusterResource,
+ available,
+ Resources.none()
+ )
+ );
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java Wed Jan 9 05:17:31 2013
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.server.res
import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstants;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
@@ -141,6 +142,8 @@ implements ResourceScheduler, CapacitySc
private boolean initialized = false;
+ private ResourceCalculator calculator;
+
public CapacityScheduler() {}
@Override
@@ -173,6 +176,21 @@ implements ResourceScheduler, CapacitySc
}
@Override
+ public Comparator<FiCaSchedulerApp> getApplicationComparator() {
+ return applicationComparator;
+ }
+
+ @Override
+ public ResourceCalculator getResourceCalculator() {
+ return calculator;
+ }
+
+ @Override
+ public Comparator<CSQueue> getQueueComparator() {
+ return queueComparator;
+ }
+
+ @Override
public synchronized int getNumClusterNodes() {
return numNodeManagers;
}
@@ -192,11 +210,20 @@ implements ResourceScheduler, CapacitySc
reinitialize(Configuration conf, RMContext rmContext) throws IOException {
if (!initialized) {
this.conf = new CapacitySchedulerConfiguration(conf);
+
this.minimumAllocation = this.conf.getMinimumAllocation();
this.maximumAllocation = this.conf.getMaximumAllocation();
+ this.calculator = this.conf.getResourceCalculator();
+
this.rmContext = rmContext;
+
initializeQueues(this.conf);
+
initialized = true;
+ LOG.info("Initialized CapacityScheduler with " +
+ "calculator=" + getResourceCalculator().getClass() + ", " +
+ "minimumAllocation=<" + getMinimumResourceCapability() + ">, " +
+ "maximumAllocation=<" + getMaximumResourceCapability() + ">");
} else {
CapacitySchedulerConfiguration oldConf = this.conf;
@@ -226,8 +253,8 @@ implements ResourceScheduler, CapacitySc
private void initializeQueues(CapacitySchedulerConfiguration conf)
throws IOException {
root =
- parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, queues, queues,
- queueComparator, applicationComparator, noop);
+ parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT,
+ queues, queues, noop);
LOG.info("Initialized root queue " + root);
}
@@ -237,8 +264,8 @@ implements ResourceScheduler, CapacitySc
// Parse new queues
Map<String, CSQueue> newQueues = new HashMap<String, CSQueue>();
CSQueue newRoot =
- parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT, newQueues, queues,
- queueComparator, applicationComparator, noop);
+ parseQueue(this, conf, null, CapacitySchedulerConfiguration.ROOT,
+ newQueues, queues, noop);
// Ensure all existing queues are still present
validateExistingQueues(queues, newQueues);
@@ -291,8 +318,6 @@ implements ResourceScheduler, CapacitySc
CapacitySchedulerConfiguration conf,
CSQueue parent, String queueName, Map<String, CSQueue> queues,
Map<String, CSQueue> oldQueues,
- Comparator<CSQueue> queueComparator,
- Comparator<FiCaSchedulerApp> applicationComparator,
QueueHook hook) throws IOException {
CSQueue queue;
String[] childQueueNames =
@@ -303,15 +328,14 @@ implements ResourceScheduler, CapacitySc
throw new IllegalStateException(
"Queue configuration missing child queue names for " + queueName);
}
- queue = new LeafQueue(csContext, queueName, parent, applicationComparator,
- oldQueues.get(queueName));
+ queue =
+ new LeafQueue(csContext, queueName, parent,oldQueues.get(queueName));
// Used only for unit tests
queue = hook.hook(queue);
} else {
ParentQueue parentQueue =
- new ParentQueue(csContext, queueName, queueComparator, parent,
- oldQueues.get(queueName));
+ new ParentQueue(csContext, queueName, parent,oldQueues.get(queueName));
// Used only for unit tests
queue = hook.hook(parentQueue);
@@ -320,7 +344,7 @@ implements ResourceScheduler, CapacitySc
for (String childQueueName : childQueueNames) {
CSQueue childQueue =
parseQueue(csContext, conf, queue, childQueueName,
- queues, oldQueues, queueComparator, applicationComparator, hook);
+ queues, oldQueues, hook);
childQueues.add(childQueue);
}
parentQueue.setChildQueues(childQueues);
@@ -442,7 +466,7 @@ implements ResourceScheduler, CapacitySc
}
private static final Allocation EMPTY_ALLOCATION =
- new Allocation(EMPTY_CONTAINER_LIST, Resources.createResource(0));
+ new Allocation(EMPTY_CONTAINER_LIST, Resources.createResource(0, 0));
@Override
@Lock(Lock.NoLock.class)
@@ -457,7 +481,8 @@ implements ResourceScheduler, CapacitySc
}
// Sanity check
- SchedulerUtils.normalizeRequests(ask, minimumAllocation.getMemory());
+ SchedulerUtils.normalizeRequests(
+ ask, calculator, getClusterResources(), minimumAllocation);
// Release containers
for (ContainerId releasedContainerId : release) {
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java Wed Jan 9 05:17:31 2013
@@ -26,11 +26,14 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.resource.Resources;
public class CapacitySchedulerConfiguration extends Configuration {
@@ -112,6 +115,13 @@ public class CapacitySchedulerConfigurat
PREFIX +"user-metrics.enable";
@Private public static final boolean DEFAULT_ENABLE_USER_METRICS = false;
+ /** ResourceComparator for scheduling. */
+ @Private public static final String RESOURCE_CALCULATOR_CLASS =
+ PREFIX + "resource-calculator";
+
+ @Private public static final Class<? extends ResourceCalculator>
+ DEFAULT_RESOURCE_CALCULATOR_CLASS = DefaultResourceCalculator.class;
+
@Private
public static final String ROOT = "root";
@@ -289,14 +299,20 @@ public class CapacitySchedulerConfigurat
int minimumMemory = getInt(
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB);
- return Resources.createResource(minimumMemory);
+ int minimumCores = getInt(
+ YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ return Resources.createResource(minimumMemory, minimumCores);
}
public Resource getMaximumAllocation() {
int maximumMemory = getInt(
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB);
- return Resources.createResource(maximumMemory);
+ int maximumCores = getInt(
+ YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_CORES,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_CORES);
+ return Resources.createResource(maximumMemory, maximumCores);
}
public boolean getEnableUserMetrics() {
@@ -307,4 +323,21 @@ public class CapacitySchedulerConfigurat
int delay = getInt(NODE_LOCALITY_DELAY, DEFAULT_NODE_LOCALITY_DELAY);
return (delay == DEFAULT_NODE_LOCALITY_DELAY) ? 0 : delay;
}
+
+ public ResourceCalculator getResourceCalculator() {
+ return ReflectionUtils.newInstance(
+ getClass(
+ RESOURCE_CALCULATOR_CLASS,
+ DEFAULT_RESOURCE_CALCULATOR_CLASS,
+ ResourceCalculator.class),
+ this);
+ }
+
+ public void setResourceComparator(
+ Class<? extends ResourceCalculator> resourceCalculatorClass) {
+ setClass(
+ RESOURCE_CALCULATOR_CLASS,
+ resourceCalculatorClass,
+ ResourceCalculator.class);
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java?rev=1430683&r1=1430682&r2=1430683&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java Wed Jan 9 05:17:31 2013
@@ -18,10 +18,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+import java.util.Comparator;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
+import org.apache.hadoop.yarn.server.resourcemanager.resource.ResourceCalculator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
/**
* Read-only interface to {@link CapacityScheduler} context.
@@ -45,4 +49,10 @@ public interface CapacitySchedulerContex
* Get the yarn configuration.
*/
Configuration getConf();
+
+ Comparator<FiCaSchedulerApp> getApplicationComparator();
+
+ ResourceCalculator getResourceCalculator();
+
+ Comparator<CSQueue> getQueueComparator();
}