You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/09/12 20:41:31 UTC
[16/50] [abbrv] hadoop git commit: YARN-6761. Fix build for YARN-3926
branch. Contributed by Varun Vasudev.
YARN-6761. Fix build for YARN-3926 branch. Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ca2808d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ca2808d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ca2808d8
Branch: refs/heads/YARN-5972
Commit: ca2808d8bbe49766029e5dba88ff863f6c993287
Parents: 58da546
Author: Sunil G <su...@apache.org>
Authored: Mon Jul 10 09:21:26 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 12 09:19:11 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/records/Resource.java | 112 +++++++++++++++----
.../resource/DominantResourceCalculator.java | 1 +
2 files changed, 90 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca2808d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 4356986..9a8e2ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -29,6 +29,8 @@ import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Records;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.Map;
/**
@@ -58,12 +60,17 @@ import java.util.Map;
@Stable
public abstract class Resource implements Comparable<Resource> {
+ private static Resource tmpResource = Records.newRecord(Resource.class);
+
private static class SimpleResource extends Resource {
private long memory;
private long vcores;
+ private Map<String, ResourceInformation> resourceInformationMap;
+
SimpleResource(long memory, long vcores) {
this.memory = memory;
this.vcores = vcores;
+
}
@Override
public int getMemory() {
@@ -89,17 +96,44 @@ public abstract class Resource implements Comparable<Resource> {
public void setVirtualCores(int vcores) {
this.vcores = vcores;
}
+ @Override
+ public Map<String, ResourceInformation> getResources() {
+ if (resourceInformationMap == null) {
+ resourceInformationMap = new HashMap<>();
+ resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(),
+ ResourceInformation.newInstance(ResourceInformation.MEMORY_MB));
+ resourceInformationMap.put(ResourceInformation.VCORES.getName(),
+ ResourceInformation.newInstance(ResourceInformation.VCORES));
+ }
+ resourceInformationMap.get(ResourceInformation.MEMORY_MB.getName())
+ .setValue(this.memory);
+ resourceInformationMap.get(ResourceInformation.VCORES.getName())
+ .setValue(this.vcores);
+ return Collections.unmodifiableMap(resourceInformationMap);
+ }
}
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
+ if (tmpResource.getResources().size() > 2) {
+ Resource ret = Records.newRecord(Resource.class);
+ ret.setMemorySize(memory);
+ ret.setVirtualCores(vCores);
+ return ret;
+ }
return new SimpleResource(memory, vCores);
}
@Public
@Stable
public static Resource newInstance(long memory, int vCores) {
+ if (tmpResource.getResources().size() > 2) {
+ Resource ret = Records.newRecord(Resource.class);
+ ret.setMemorySize(memory);
+ ret.setVirtualCores(vCores);
+ return ret;
+ }
return new SimpleResource(memory, vCores);
}
@@ -116,13 +150,7 @@ public abstract class Resource implements Comparable<Resource> {
public static void copy(Resource source, Resource dest) {
for (Map.Entry<String, ResourceInformation> entry : source.getResources()
.entrySet()) {
- try {
- ResourceInformation.copy(entry.getValue(),
- dest.getResourceInformation(entry.getKey()));
- } catch (YarnException ye) {
- dest.setResourceInformation(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
- }
+ dest.setResourceInformation(entry.getKey(), entry.getValue());
}
}
@@ -234,8 +262,15 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract ResourceInformation getResourceInformation(String resource)
- throws YarnException;
+ public ResourceInformation getResourceInformation(String resource)
+ throws YarnException {
+ if (getResources().containsKey(resource)) {
+ return getResources().get(resource);
+ }
+ throw new YarnException(
+ "Unknown resource '" + resource + "'. Known resources are "
+ + getResources().keySet());
+ }
/**
* Get the value for a specified resource. No information about the units is
@@ -247,7 +282,14 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract Long getResourceValue(String resource) throws YarnException;
+ public Long getResourceValue(String resource) throws YarnException {
+ if (getResources().containsKey(resource)) {
+ return getResources().get(resource).getValue();
+ }
+ throw new YarnException(
+ "Unknown resource '" + resource + "'. Known resources are "
+ + getResources().keySet());
+ }
/**
* Set the ResourceInformation object for a particular resource.
@@ -258,8 +300,25 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract void setResourceInformation(String resource,
- ResourceInformation resourceInformation) throws ResourceNotFoundException;
+ public void setResourceInformation(String resource,
+ ResourceInformation resourceInformation) throws ResourceNotFoundException {
+ if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+ this.setMemorySize(resourceInformation.getValue());
+ return;
+ }
+ if (resource.equals(ResourceInformation.VCORES.getName())) {
+ this.setVirtualCores((int) resourceInformation.getValue());
+ return;
+ }
+ if (getResources().containsKey(resource)) {
+ ResourceInformation
+ .copy(resourceInformation, getResources().get(resource));
+ return;
+ }
+ throw new ResourceNotFoundException(
+ "Unknown resource '" + resource + "'. Known resources are "
+ + getResources().keySet());
+ }
/**
* Set the value of a resource in the ResourceInformation object. The unit of
@@ -271,8 +330,24 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract void setResourceValue(String resource, Long value)
- throws ResourceNotFoundException;
+ public void setResourceValue(String resource, Long value)
+ throws ResourceNotFoundException {
+ if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+ this.setMemorySize(value);
+ return;
+ }
+ if (resource.equals(ResourceInformation.VCORES.getName())) {
+ this.setVirtualCores(value.intValue());
+ return;
+ }
+ if (getResources().containsKey(resource)) {
+ getResources().get(resource).setValue(value);
+ return;
+ }
+ throw new ResourceNotFoundException(
+ "Unknown resource '" + resource + "'. Known resources are "
+ + getResources().keySet());
+ }
@Override
public int hashCode() {
@@ -312,15 +387,6 @@ public abstract class Resource implements Comparable<Resource> {
}
@Override
- public int compareTo(Resource other) {
- long diff = this.getMemorySize() - other.getMemorySize();
- if (diff == 0) {
- diff = this.getVirtualCores() - other.getVirtualCores();
- }
- return diff == 0 ? 0 : (diff > 0 ? 1 : -1);
- }
-
- @Override
public String toString() {
StringBuilder sb = new StringBuilder();
sb.append("<memory:").append(getMemorySize()).append(", vCores:")
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ca2808d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 76bc364..79bb03d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -306,6 +306,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
value = roundUp(value, stepFactorValue);
}
tmp.setValue(Math.min(value, maximumValue));
+ ret.setResourceInformation(resource, tmp);
} catch (YarnException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org