You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2019/03/19 23:02:10 UTC
[hadoop] 15/47: YARN-6788. [YARN-3926] Improve performance of
resource profile branch (Contributed by Sunil Govindan via Daniel
Templeton)
This is an automated email from the ASF dual-hosted git repository.
jhung pushed a commit to branch YARN-8200
in repository https://gitbox.apache.org/repos/asf/hadoop.git
commit 8182da5dd88f0554b4401dc9b0b3d4ff41d7a06a
Author: Daniel Templeton <te...@apache.org>
AuthorDate: Fri Aug 4 08:42:34 2017 -0700
YARN-6788. [YARN-3926] Improve performance of resource profile branch
(Contributed by Sunil Govindan via Daniel Templeton)
(cherry picked from commit 3aeaafecb823ef6c175ea5d0d9cb726faacaf32d)
---
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 18 ++
.../apache/hadoop/yarn/api/records/Resource.java | 234 ++++++++-------------
.../yarn/api/records/ResourceInformation.java | 13 +-
.../hadoop/yarn/api/records/impl/BaseResource.java | 133 ++++++++++++
.../hadoop/yarn/api/records/impl/package-info.java | 22 ++
.../hadoop/yarn/util/UnitsConversionUtil.java | 8 +-
.../hadoop/yarn/util/resource/ResourceUtils.java | 214 +++++++++++--------
.../hadoop/yarn/util/resource/package-info.java | 22 ++
.../yarn/api/records/impl/pb/ProtoUtils.java | 5 +-
.../yarn/api/records/impl/pb/ResourcePBImpl.java | 110 +++++-----
.../util/resource/DominantResourceCalculator.java | 67 +++---
.../hadoop/yarn/util/resource/Resources.java | 194 ++++++++---------
.../yarn/util/resource/TestResourceUtils.java | 14 +-
.../hadoop/yarn/util/resource/TestResources.java | 7 +-
.../rmapp/attempt/RMAppAttemptMetrics.java | 11 +-
.../scheduler/SchedulerApplicationAttempt.java | 9 +-
.../resourcemanager/webapp/dao/SchedulerInfo.java | 3 +-
.../server/resourcemanager/TestAppManager.java | 1 +
18 files changed, 627 insertions(+), 458 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index c129203..e086fbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -615,4 +615,22 @@
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Ignore MS_EXPOSE_REP -->
+ <Match>
+ <Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
+ <Method name="getResourceTypesArray" />
+ <Bug pattern="MS_EXPOSE_REP" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.util.resource.ResourceUtils" />
+ <Method name="getResourceNamesArray" />
+ <Bug pattern="MS_EXPOSE_REP" />
+ </Match>
+
+ <Match>
+ <Class name="org.apache.hadoop.yarn.api.records.impl.BaseResource" />
+ <Method name="getResources" />
+ <Bug pattern="EI_EXPOSE_REP" />
+ </Match>
</FindBugsFilter>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index fb31745..bbd4c87 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.api.records;
+import java.util.Arrays;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Public;
@@ -25,13 +27,10 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.classification.InterfaceStability.Stable;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.Records;
-
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import org.apache.hadoop.yarn.util.resource.ResourceUtils;
/**
* <p><code>Resource</code> models a set of computer resources in the
@@ -60,97 +59,49 @@ import java.util.Map;
@Stable
public abstract class Resource implements Comparable<Resource> {
- private static Resource tmpResource = Records.newRecord(Resource.class);
-
- private static class SimpleResource extends Resource {
- private long memory;
- private long vcores;
- private Map<String, ResourceInformation> resourceInformationMap;
-
- SimpleResource(long memory, long vcores) {
- this.memory = memory;
- this.vcores = vcores;
-
- }
- @Override
- public int getMemory() {
- return castToIntSafely(memory);
- }
- @Override
- public void setMemory(int memory) {
- this.memory = memory;
- }
- @Override
- public long getMemorySize() {
- return memory;
- }
- @Override
- public void setMemorySize(long memory) {
- this.memory = memory;
- }
- @Override
- public int getVirtualCores() {
- return castToIntSafely(vcores);
- }
- @Override
- public void setVirtualCores(int vcores) {
- this.vcores = vcores;
- }
- @Override
- public Map<String, ResourceInformation> getResources() {
- if (resourceInformationMap == null) {
- resourceInformationMap = new HashMap<>();
- resourceInformationMap.put(ResourceInformation.MEMORY_MB.getName(),
- ResourceInformation.newInstance(ResourceInformation.MEMORY_MB));
- resourceInformationMap.put(ResourceInformation.VCORES.getName(),
- ResourceInformation.newInstance(ResourceInformation.VCORES));
- }
- resourceInformationMap.get(ResourceInformation.MEMORY_MB.getName())
- .setValue(this.memory);
- resourceInformationMap.get(ResourceInformation.VCORES.getName())
- .setValue(this.vcores);
- return Collections.unmodifiableMap(resourceInformationMap);
- }
- }
+ protected static final String MEMORY = ResourceInformation.MEMORY_MB.getName();
+ protected static final String VCORES = ResourceInformation.VCORES.getName();
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
- if (tmpResource.getResources().size() > 2) {
+ if (ResourceUtils.getResourceTypesArray().length > 2) {
Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory);
ret.setVirtualCores(vCores);
return ret;
}
- return new SimpleResource(memory, vCores);
+ return new BaseResource(memory, vCores);
}
@Public
@Stable
public static Resource newInstance(long memory, int vCores) {
- if (tmpResource.getResources().size() > 2) {
+ if (ResourceUtils.getResourceTypesArray().length > 2) {
Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory);
ret.setVirtualCores(vCores);
return ret;
}
- return new SimpleResource(memory, vCores);
+ return new BaseResource(memory, vCores);
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static Resource newInstance(Resource resource) {
- Resource ret = Resource.newInstance(0, 0);
- Resource.copy(resource, ret);
+ Resource ret = Resource.newInstance(resource.getMemorySize(),
+ resource.getVirtualCores());
+ if (ResourceUtils.getResourceTypesArray().length > 2) {
+ Resource.copy(resource, ret);
+ }
return ret;
}
@InterfaceAudience.Private
@InterfaceStability.Unstable
public static void copy(Resource source, Resource dest) {
- for (Map.Entry<String, ResourceInformation> entry : source.getResources()
- .entrySet()) {
- dest.setResourceInformation(entry.getKey(), entry.getValue());
+ for (ResourceInformation entry : source.getResources()) {
+ dest.setResourceInformation(entry.getName(), entry);
}
}
@@ -251,25 +202,26 @@ public abstract class Resource implements Comparable<Resource> {
*/
@Public
@Evolving
- public abstract Map<String, ResourceInformation> getResources();
+ public abstract ResourceInformation[] getResources();
/**
* Get ResourceInformation for a specified resource.
*
* @param resource name of the resource
* @return the ResourceInformation object for the resource
- * @throws YarnException if the resource can't be found
+ * @throws ResourceNotFoundException if the resource can't be found
*/
@Public
@Evolving
public ResourceInformation getResourceInformation(String resource)
- throws YarnException {
- if (getResources().containsKey(resource)) {
- return getResources().get(resource);
+ throws ResourceNotFoundException {
+ Integer index = ResourceUtils.getResourceTypeIndex().get(resource);
+ ResourceInformation[] resources = getResources();
+ if (index != null) {
+ return resources[index];
}
- throw new YarnException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+ throw new ResourceNotFoundException("Unknown resource '" + resource
+ + "'. Known resources are " + Arrays.toString(resources));
}
/**
@@ -278,17 +230,13 @@ public abstract class Resource implements Comparable<Resource> {
*
* @param resource name of the resource
* @return the value for the resource
- * @throws YarnException if the resource can't be found
+ * @throws ResourceNotFoundException if the resource can't be found
*/
@Public
@Evolving
- public Long getResourceValue(String resource) throws YarnException {
- if (getResources().containsKey(resource)) {
- return getResources().get(resource).getValue();
- }
- throw new YarnException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+ public long getResourceValue(String resource)
+ throws ResourceNotFoundException {
+ return getResourceInformation(resource).getValue();
}
/**
@@ -301,23 +249,18 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Evolving
public void setResourceInformation(String resource,
- ResourceInformation resourceInformation) throws ResourceNotFoundException {
- if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+ ResourceInformation resourceInformation)
+ throws ResourceNotFoundException {
+ if (resource.equals(MEMORY)) {
this.setMemorySize(resourceInformation.getValue());
return;
}
- if (resource.equals(ResourceInformation.VCORES.getName())) {
+ if (resource.equals(VCORES)) {
this.setVirtualCores((int) resourceInformation.getValue());
return;
}
- if (getResources().containsKey(resource)) {
- ResourceInformation
- .copy(resourceInformation, getResources().get(resource));
- return;
- }
- throw new ResourceNotFoundException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+ ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ ResourceInformation.copy(resourceInformation, storedResourceInfo);
}
/**
@@ -332,21 +275,17 @@ public abstract class Resource implements Comparable<Resource> {
@Evolving
public void setResourceValue(String resource, Long value)
throws ResourceNotFoundException {
- if (resource.equals(ResourceInformation.MEMORY_MB.getName())) {
+ if (resource.equals(MEMORY)) {
this.setMemorySize(value);
return;
}
- if (resource.equals(ResourceInformation.VCORES.getName())) {
+ if (resource.equals(VCORES)) {
this.setVirtualCores(value.intValue());
return;
}
- if (getResources().containsKey(resource)) {
- getResources().get(resource).setValue(value);
- return;
- }
- throw new ResourceNotFoundException(
- "Unknown resource '" + resource + "'. Known resources are "
- + getResources().keySet());
+
+ ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ storedResourceInfo.setValue(value);
}
@Override
@@ -356,13 +295,10 @@ public abstract class Resource implements Comparable<Resource> {
int result = (int) (939769357
+ getMemorySize()); // prime * result = 939769357 initially
result = prime * result + getVirtualCores();
- for (Map.Entry<String, ResourceInformation> entry : getResources()
- .entrySet()) {
- if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
- || entry.getKey().equals(ResourceInformation.VCORES.getName())) {
- continue;
+ for (ResourceInformation entry : getResources()) {
+ if (!entry.getName().equals(MEMORY) && !entry.getName().equals(VCORES)) {
+ result = prime * result + entry.hashCode();
}
- result = prime * result + entry.getValue().hashCode();
}
return result;
}
@@ -379,11 +315,26 @@ public abstract class Resource implements Comparable<Resource> {
return false;
}
Resource other = (Resource) obj;
- if (getMemorySize() != other.getMemorySize() || getVirtualCores() != other
- .getVirtualCores()) {
+ if (getMemorySize() != other.getMemorySize()
+ || getVirtualCores() != other.getVirtualCores()) {
+ return false;
+ }
+
+ ResourceInformation[] myVectors = getResources();
+ ResourceInformation[] otherVectors = other.getResources();
+
+ if (myVectors.length != otherVectors.length) {
return false;
}
- return this.getResources().equals(other.getResources());
+
+ for (int i = 0; i < myVectors.length; i++) {
+ ResourceInformation a = myVectors[i];
+ ResourceInformation b = otherVectors[i];
+ if ((a != b) && ((a == null) || !a.equals(b))) {
+ return false;
+ }
+ }
+ return true;
}
@Override
@@ -391,21 +342,20 @@ public abstract class Resource implements Comparable<Resource> {
StringBuilder sb = new StringBuilder();
sb.append("<memory:").append(getMemorySize()).append(", vCores:")
.append(getVirtualCores());
- for (Map.Entry<String, ResourceInformation> entry : getResources()
- .entrySet()) {
- if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
- && entry.getValue().getUnits()
+ for (ResourceInformation entry : getResources()) {
+ if (entry.getName().equals(MEMORY)
+ && entry.getUnits()
.equals(ResourceInformation.MEMORY_MB.getUnits())) {
continue;
}
- if (entry.getKey().equals(ResourceInformation.VCORES.getName())
- && entry.getValue().getUnits()
+ if (entry.getName().equals(VCORES)
+ && entry.getUnits()
.equals(ResourceInformation.VCORES.getUnits())) {
continue;
}
- sb.append(", ").append(entry.getKey()).append(": ")
- .append(entry.getValue().getValue())
- .append(entry.getValue().getUnits());
+ sb.append(", ").append(entry.getName()).append(": ")
+ .append(entry.getValue())
+ .append(entry.getUnits());
}
sb.append(">");
return sb.toString();
@@ -413,28 +363,30 @@ public abstract class Resource implements Comparable<Resource> {
@Override
public int compareTo(Resource other) {
- Map<String, ResourceInformation> thisResources, otherResources;
- thisResources = this.getResources();
- otherResources = other.getResources();
- long diff = thisResources.size() - otherResources.size();
+ ResourceInformation[] thisResources = this.getResources();
+ ResourceInformation[] otherResources = other.getResources();
+
+ // compare memory and vcores first(in that order) to preserve
+ // existing behaviour
+ long diff = this.getMemorySize() - other.getMemorySize();
if (diff == 0) {
- // compare memory and vcores first(in that order) to preserve
- // existing behaviour
- if (thisResources.keySet().equals(otherResources.keySet())) {
- diff = this.getMemorySize() - other.getMemorySize();
- if (diff == 0) {
- diff = this.getVirtualCores() - other.getVirtualCores();
- }
- if (diff == 0) {
- for (Map.Entry<String, ResourceInformation> entry : thisResources
- .entrySet()) {
- if (entry.getKey().equals(ResourceInformation.MEMORY_MB.getName())
- || entry.getKey()
- .equals(ResourceInformation.VCORES.getName())) {
- continue;
- }
- diff =
- entry.getValue().compareTo(otherResources.get(entry.getKey()));
+ diff = this.getVirtualCores() - other.getVirtualCores();
+ }
+ if (diff == 0) {
+ diff = thisResources.length - otherResources.length;
+ if (diff == 0) {
+ int maxLength = ResourceUtils.getResourceTypesArray().length;
+ for (int i = 0; i < maxLength; i++) {
+ // For memory and vcores, we can skip the loop as it's already
+ // compared.
+ if (i < 2) {
+ continue;
+ }
+
+ ResourceInformation entry = thisResources[i];
+ ResourceInformation otherEntry = otherResources[i];
+ if (entry.getName().equals(otherEntry.getName())) {
+ diff = entry.compareTo(otherEntry);
if (diff != 0) {
break;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
index 7c0f055..4717d82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceInformation.java
@@ -242,10 +242,15 @@ public class ResourceInformation implements Comparable<ResourceInformation> {
return false;
}
ResourceInformation r = (ResourceInformation) obj;
- int cmp =
- UnitsConversionUtil.compare(this.units, this.value, r.units, r.value);
- return this.name.equals(r.getName()) && this.resourceType
- .equals(r.getResourceType()) && (cmp == 0);
+ if (!this.name.equals(r.getName())
+ || !this.resourceType.equals(r.getResourceType())) {
+ return false;
+ }
+ if (this.units.equals(r.units)) {
+ return this.value == r.value;
+ }
+ return (UnitsConversionUtil.compare(this.units, this.value, r.units,
+ r.value) == 0);
}
@Override
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
new file mode 100644
index 0000000..83db542
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
@@ -0,0 +1,133 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
+
+import java.util.Arrays;
+
+/**
+ * <p>
+ * <code>BaseResource</code> extends Resource to handle base resources such
+ * as memory and CPU.
+ * TODO: We have a long term plan to use AbstractResource when additional
+ * resource types are to be handled as well.
+ * </p>
+ *
+ * <p>
+ * Currently it models both <em>memory</em> and <em>CPU</em>.
+ * </p>
+ *
+ * <p>
+ * The unit for memory is megabytes. CPU is modeled with virtual cores (vcores),
+ * a unit for expressing parallelism. A node's capacity should be configured
+ * with virtual cores equal to its number of physical cores. A container should
+ * be requested with the number of cores it can saturate, i.e. the average
+ * number of threads it expects to have runnable at a time.
+ * </p>
+ *
+ * <p>
+ * Virtual cores take integer values and thus currently CPU-scheduling is very
+ * coarse. A complementary axis for CPU requests that represents processing
+ * power will likely be added in the future to enable finer-grained resource
+ * configuration.
+ * </p>
+ *
+ * @see Resource
+ */
+@Public
+@Unstable
+public class BaseResource extends Resource {
+
+ private ResourceInformation memoryResInfo;
+ private ResourceInformation vcoresResInfo;
+ protected ResourceInformation[] resources = null;
+ protected ResourceInformation[] readOnlyResources = null;
+
+ protected enum MandatoryResources {
+ MEMORY(0), VCORES(1);
+
+ private final int id;
+
+ MandatoryResources(int id) {
+ this.id = id;
+ }
+
+ public int getId() {
+ return this.id;
+ }
+ }
+
+ public BaseResource() {
+ // Base constructor.
+ }
+
+ public BaseResource(long memory, long vcores) {
+ this.memoryResInfo = ResourceInformation.newInstance(MEMORY,
+ ResourceInformation.MEMORY_MB.getUnits(), memory);
+ this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
+
+ resources = new ResourceInformation[MandatoryResources.values().length];
+ readOnlyResources = new ResourceInformation[MandatoryResources
+ .values().length];
+ resources[MandatoryResources.MEMORY.id] = memoryResInfo;
+ resources[MandatoryResources.VCORES.id] = vcoresResInfo;
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public int getMemory() {
+ return (int) memoryResInfo.getValue();
+ }
+
+ @Override
+ @SuppressWarnings("deprecation")
+ public void setMemory(int memory) {
+ this.memoryResInfo.setValue(memory);
+ }
+
+ @Override
+ public long getMemorySize() {
+ return memoryResInfo.getValue();
+ }
+
+ @Override
+ public void setMemorySize(long memory) {
+ this.memoryResInfo.setValue(memory);
+ }
+
+ @Override
+ public int getVirtualCores() {
+ return (int) vcoresResInfo.getValue();
+ }
+
+ @Override
+ public void setVirtualCores(int vcores) {
+ this.vcoresResInfo.setValue(vcores);
+ }
+
+ @Override
+ public ResourceInformation[] getResources() {
+ return readOnlyResources;
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
new file mode 100644
index 0000000..d04e339
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.api.records.impl;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
index c7663de..7b737bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/UnitsConversionUtil.java
@@ -186,11 +186,11 @@ public class UnitsConversionUtil {
if (!KNOWN_UNITS.contains(unitB)) {
throw new IllegalArgumentException("Unknown unit '" + unitB + "'");
}
- Converter unitAC = getConverter(unitA);
- Converter unitBC = getConverter(unitB);
if (unitA.equals(unitB)) {
- return Long.valueOf(valueA).compareTo(valueB);
+ return Long.compare(valueA, valueB);
}
+ Converter unitAC = getConverter(unitA);
+ Converter unitBC = getConverter(unitB);
int unitAPos = SORTED_UNITS.indexOf(unitA);
int unitBPos = SORTED_UNITS.indexOf(unitB);
try {
@@ -201,7 +201,7 @@ public class UnitsConversionUtil {
} else {
tmpA = convert(unitA, unitB, valueA);
}
- return Long.valueOf(tmpA).compareTo(tmpB);
+ return Long.compare(tmpA, tmpB);
} catch (IllegalArgumentException ie) {
BigInteger tmpA = BigInteger.valueOf(valueA);
BigInteger tmpB = BigInteger.valueOf(valueB);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
similarity index 72%
rename from hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
rename to hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 86cf872..49feafa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -42,6 +42,7 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
/**
* Helper class to read the resource-types to be supported by the system.
@@ -65,11 +66,14 @@ public class ResourceUtils {
DISALLOWED_NAMES.add(VCORES);
}
- private static volatile Object lock;
- private static Map<String, ResourceInformation> readOnlyResources;
- private static volatile Object nodeLock;
- private static Map<String, ResourceInformation> readOnlyNodeResources;
-
+ private static volatile boolean initializedResources = false;
+ private static final Map<String, Integer> RESOURCE_NAME_TO_INDEX =
+ new ConcurrentHashMap<String, Integer>();
+ private static volatile Map<String, ResourceInformation> resourceTypes;
+ private static volatile String[] resourceNamesArray;
+ private static volatile ResourceInformation[] resourceTypesArray;
+ private static volatile boolean initializedNodeResources = false;
+ private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
static final Log LOG = LogFactory.getLog(ResourceUtils.class);
@@ -127,21 +131,17 @@ public class ResourceUtils {
private static void setMinimumAllocationForMandatoryResources(
Map<String, ResourceInformation> res, Configuration conf) {
- String[][] resourceTypesKeys =
- {
- { ResourceInformation.MEMORY_MB.getName(),
+ String[][] resourceTypesKeys = {
+ {ResourceInformation.MEMORY_MB.getName(),
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB,
String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
- ResourceInformation.MEMORY_MB.getName()
- },
- { ResourceInformation.VCORES.getName(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_MB),
+ ResourceInformation.MEMORY_MB.getName()},
+ {ResourceInformation.VCORES.getName(),
YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES,
String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
- ResourceInformation.VCORES.getName()
- }
- };
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES),
+ ResourceInformation.VCORES.getName()}};
for (String[] arr : resourceTypesKeys) {
String resourceTypesKey =
YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MINIMUM_ALLOCATION;
@@ -166,23 +166,17 @@ public class ResourceUtils {
private static void setMaximumAllocationForMandatoryResources(
Map<String, ResourceInformation> res, Configuration conf) {
- String[][] resourceTypesKeys =
- {
- {
- ResourceInformation.MEMORY_MB.getName(),
+ String[][] resourceTypesKeys = {
+ {ResourceInformation.MEMORY_MB.getName(),
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB,
String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
- ResourceInformation.MEMORY_MB.getName()
- },
- {
- ResourceInformation.VCORES.getName(),
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_MB),
+ ResourceInformation.MEMORY_MB.getName()},
+ {ResourceInformation.VCORES.getName(),
YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES,
String.valueOf(
- YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
- ResourceInformation.VCORES.getName()
- }
- };
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES),
+ ResourceInformation.VCORES.getName()}};
for (String[] arr : resourceTypesKeys) {
String resourceTypesKey =
YarnConfiguration.RESOURCE_TYPES + "." + arr[0] + MAXIMUM_ALLOCATION;
@@ -251,7 +245,50 @@ public class ResourceUtils {
addManadtoryResources(resourceInformationMap);
setMinimumAllocationForMandatoryResources(resourceInformationMap, conf);
setMaximumAllocationForMandatoryResources(resourceInformationMap, conf);
- readOnlyResources = Collections.unmodifiableMap(resourceInformationMap);
+ resourceTypes = Collections.unmodifiableMap(resourceInformationMap);
+ updateKnownResources();
+ updateResourceTypeIndex();
+ }
+
+ private static void updateKnownResources() {
+ // Update resource names.
+ resourceNamesArray = new String[resourceTypes.size()];
+ resourceTypesArray = new ResourceInformation[resourceTypes.size()];
+
+ int index = 2;
+ for (ResourceInformation resInfo : resourceTypes.values()) {
+ if (resInfo.getName().equals(MEMORY)) {
+ resourceTypesArray[0] = ResourceInformation
+ .newInstance(resourceTypes.get(MEMORY));
+ resourceNamesArray[0] = MEMORY;
+ } else if (resInfo.getName().equals(VCORES)) {
+ resourceTypesArray[1] = ResourceInformation
+ .newInstance(resourceTypes.get(VCORES));
+ resourceNamesArray[1] = VCORES;
+ } else {
+ resourceTypesArray[index] = ResourceInformation.newInstance(resInfo);
+ resourceNamesArray[index] = resInfo.getName();
+ index++;
+ }
+ }
+ }
+
+ private static void updateResourceTypeIndex() {
+ RESOURCE_NAME_TO_INDEX.clear();
+
+ for (int index = 0; index < resourceTypesArray.length; index++) {
+ ResourceInformation resInfo = resourceTypesArray[index];
+ RESOURCE_NAME_TO_INDEX.put(resInfo.getName(), index);
+ }
+ }
+
+ /**
+ * Get associate index of resource types such memory, cpu etc.
+ * This could help to access each resource types in a resource faster.
+ * @return Index map for all Resource Types.
+ */
+ public static Map<String, Integer> getResourceTypeIndex() {
+ return RESOURCE_NAME_TO_INDEX;
}
/**
@@ -264,6 +301,22 @@ public class ResourceUtils {
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
}
+ /**
+ * Get resource names array, this is mostly for performance perspective. Never
+ * modify returned array.
+ *
+ * @return resourceNamesArray
+ */
+ public static String[] getResourceNamesArray() {
+ getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ return resourceNamesArray;
+ }
+
+ public static ResourceInformation[] getResourceTypesArray() {
+ getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ return resourceTypesArray;
+ }
+
private static Map<String, ResourceInformation> getResourceTypes(
Configuration conf) {
return getResourceTypes(conf,
@@ -272,30 +325,28 @@ public class ResourceUtils {
private static Map<String, ResourceInformation> getResourceTypes(
Configuration conf, String resourceFile) {
- if (lock == null) {
+ if (!initializedResources) {
synchronized (ResourceUtils.class) {
- if (lock == null) {
- synchronized (ResourceUtils.class) {
- Map<String, ResourceInformation> resources = new HashMap<>();
- if (conf == null) {
- conf = new YarnConfiguration();
- }
- try {
- addResourcesFileToConf(resourceFile, conf);
- LOG.debug("Found " + resourceFile + ", adding to configuration");
- initializeResourcesMap(conf, resources);
- lock = new Object();
- } catch (FileNotFoundException fe) {
- LOG.info("Unable to find '" + resourceFile
- + "'. Falling back to memory and vcores as resources", fe);
- initializeResourcesMap(conf, resources);
- lock = new Object();
- }
+ if (!initializedResources) {
+ Map<String, ResourceInformation> resources = new HashMap<>();
+ if (conf == null) {
+ conf = new YarnConfiguration();
+ }
+ try {
+ addResourcesFileToConf(resourceFile, conf);
+ LOG.debug("Found " + resourceFile + ", adding to configuration");
+ initializeResourcesMap(conf, resources);
+ initializedResources = true;
+ } catch (FileNotFoundException fe) {
+ LOG.info("Unable to find '" + resourceFile
+ + "'. Falling back to memory and vcores as resources", fe);
+ initializeResourcesMap(conf, resources);
+ initializedResources = true;
}
}
}
}
- return readOnlyResources;
+ return resourceTypes;
}
private static InputStream getConfInputStream(String resourceFile,
@@ -341,13 +392,15 @@ public class ResourceUtils {
}
@VisibleForTesting
- static void resetResourceTypes() {
- lock = null;
+ synchronized static void resetResourceTypes() {
+ initializedResources = false;
}
@VisibleForTesting
public static void resetResourceTypes(Configuration conf) {
- lock = null;
+ synchronized (ResourceUtils.class) {
+ initializedResources = false;
+ }
getResourceTypes(conf);
}
@@ -375,17 +428,17 @@ public class ResourceUtils {
*/
public static Map<String, ResourceInformation> getNodeResourceInformation(
Configuration conf) {
- if (nodeLock == null) {
+ if (!initializedNodeResources) {
synchronized (ResourceUtils.class) {
- if (nodeLock == null) {
- synchronized (ResourceUtils.class) {
- Map<String, ResourceInformation> nodeResources =
- initializeNodeResourceInformation(conf);
- addManadtoryResources(nodeResources);
- checkMandatatoryResources(nodeResources);
- readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
- nodeLock = new Object();
- }
+ if (!initializedNodeResources) {
+ Map<String, ResourceInformation> nodeResources = initializeNodeResourceInformation(
+ conf);
+ addManadtoryResources(nodeResources);
+ checkMandatatoryResources(nodeResources);
+ setMinimumAllocationForMandatoryResources(nodeResources, conf);
+ setMaximumAllocationForMandatoryResources(nodeResources, conf);
+ readOnlyNodeResources = Collections.unmodifiableMap(nodeResources);
+ initializedNodeResources = true;
}
}
}
@@ -433,28 +486,24 @@ public class ResourceUtils {
@VisibleForTesting
synchronized public static void resetNodeResources() {
- nodeLock = null;
+ initializedNodeResources = false;
}
public static Resource getResourceTypesMinimumAllocation() {
- Map<String, ResourceInformation> resourceTypes = getResourceTypes();
Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : resourceTypes
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : resourceTypesArray) {
+ String name = entry.getName();
if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
- ret.setMemorySize(entry.getValue().getMinimumAllocation());
- continue;
- }
- if (name.equals(ResourceInformation.VCORES.getName())) {
- Long tmp = entry.getValue().getMinimumAllocation();
+ ret.setMemorySize(entry.getMinimumAllocation());
+ } else if (name.equals(ResourceInformation.VCORES.getName())) {
+ Long tmp = entry.getMinimumAllocation();
if (tmp > Integer.MAX_VALUE) {
tmp = (long) Integer.MAX_VALUE;
}
ret.setVirtualCores(tmp.intValue());
- continue;
+ } else {
+ ret.setResourceValue(name, entry.getMinimumAllocation());
}
- ret.setResourceValue(name, entry.getValue().getMinimumAllocation());
}
return ret;
}
@@ -464,24 +513,21 @@ public class ResourceUtils {
* @return a Resource object with the maximum allocation for the scheduler
*/
public static Resource getResourceTypesMaximumAllocation() {
- Map<String, ResourceInformation> resourceTypes = getResourceTypes();
Resource ret = Resource.newInstance(0, 0);
- for (Map.Entry<String, ResourceInformation> entry : resourceTypes
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : resourceTypesArray) {
+ String name = entry.getName();
if (name.equals(ResourceInformation.MEMORY_MB.getName())) {
- ret.setMemorySize(entry.getValue().getMaximumAllocation());
- continue;
- }
- if (name.equals(ResourceInformation.VCORES.getName())) {
- Long tmp = entry.getValue().getMaximumAllocation();
+ ret.setMemorySize(entry.getMaximumAllocation());
+ } else if (name.equals(ResourceInformation.VCORES.getName())) {
+ Long tmp = entry.getMaximumAllocation();
if (tmp > Integer.MAX_VALUE) {
tmp = (long) Integer.MAX_VALUE;
}
ret.setVirtualCores(tmp.intValue());
continue;
+ } else {
+ ret.setResourceValue(name, entry.getMaximumAllocation());
}
- ret.setResourceValue(name, entry.getValue().getMaximumAllocation());
}
return ret;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
new file mode 100644
index 0000000..01c7eb0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/package-info.java
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Unstable
+package org.apache.hadoop.yarn.util.resource;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
index e8b0594..8176068 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ProtoUtils.java
@@ -472,9 +472,8 @@ public class ProtoUtils {
List<YarnProtos.StringLongMapProto> pList) {
Resource tmp = Resource.newInstance(0, 0);
Map<String, Long> ret = new HashMap<>();
- for (Map.Entry<String, ResourceInformation> entry : tmp.getResources()
- .entrySet()) {
- ret.put(entry.getKey(), 0L);
+ for (ResourceInformation entry : tmp.getResources()) {
+ ret.put(entry.getName(), 0L);
}
if (pList != null) {
for (YarnProtos.StringLongMapProto p : pList) {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
index 6e51efa..00be77a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourcePBImpl.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.protocolrecords.ResourceTypes;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
@@ -33,14 +34,13 @@ import org.apache.hadoop.yarn.proto.YarnProtos.ResourceInformationProto;
import org.apache.hadoop.yarn.util.resource.ResourceUtils;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.HashMap;
+import java.util.Arrays;
import java.util.Map;
-import java.util.Collections;
@Private
@Unstable
-public class ResourcePBImpl extends Resource {
+public class ResourcePBImpl extends BaseResource {
private static final Log LOG = LogFactory.getLog(ResourcePBImpl.class);
@@ -48,10 +48,6 @@ public class ResourcePBImpl extends Resource {
ResourceProto.Builder builder = null;
boolean viaProto = false;
- private Map<String, ResourceInformation> resources;
- private Map<String, ResourceInformation> readOnlyResources;
-
-
// call via ProtoUtils.convertToProtoFormat(Resource)
static ResourceProto getProto(Resource r) {
final ResourcePBImpl pb;
@@ -72,8 +68,6 @@ public class ResourcePBImpl extends Resource {
public ResourcePBImpl(ResourceProto proto) {
this.proto = proto;
viaProto = true;
- this.readOnlyResources = null;
- this.resources = null;
initResources();
}
@@ -101,11 +95,13 @@ public class ResourcePBImpl extends Resource {
public long getMemorySize() {
// memory should always be present
initResources();
- ResourceInformation ri =
- this.getResourceInformation(ResourceInformation.MEMORY_MB.getName());
- return UnitsConversionUtil
- .convert(ri.getUnits(), ResourceInformation.MEMORY_MB.getUnits(),
- ri.getValue());
+ ResourceInformation ri = resources[MandatoryResources.MEMORY.getId()];
+
+ if (ri.getUnits().isEmpty()) {
+ return ri.getValue();
+ }
+ return UnitsConversionUtil.convert(ri.getUnits(),
+ ResourceInformation.MEMORY_MB.getUnits(), ri.getValue());
}
@Override
@@ -117,23 +113,20 @@ public class ResourcePBImpl extends Resource {
@Override
public void setMemorySize(long memory) {
maybeInitBuilder();
- getResourceInformation(ResourceInformation.MEMORY_MB.getName())
- .setValue(memory);
+ getResourceInformation(MEMORY).setValue(memory);
}
@Override
public int getVirtualCores() {
// vcores should always be present
initResources();
- return this.getResourceValue(ResourceInformation.VCORES.getName())
- .intValue();
+ return (int) resources[MandatoryResources.VCORES.getId()].getValue();
}
@Override
public void setVirtualCores(int vCores) {
maybeInitBuilder();
- getResourceInformation(ResourceInformation.VCORES.getName())
- .setValue(vCores);
+ getResourceInformation(VCORES).setValue(vCores);
}
private void initResources() {
@@ -142,6 +135,7 @@ public class ResourcePBImpl extends Resource {
}
ResourceProtoOrBuilder p = viaProto ? proto : builder;
initResourcesMap();
+ Map<String, Integer> indexMap = ResourceUtils.getResourceTypeIndex();
for (ResourceInformationProto entry : p.getResourceValueMapList()) {
ResourceTypes type =
entry.hasType() ? ProtoUtils.convertFromProtoFormat(entry.getType()) :
@@ -150,14 +144,16 @@ public class ResourcePBImpl extends Resource {
long value = entry.hasValue() ? entry.getValue() : 0L;
ResourceInformation ri = ResourceInformation
.newInstance(entry.getKey(), units, value, type, 0L, Long.MAX_VALUE);
- if (resources.containsKey(ri.getName())) {
- resources.get(ri.getName()).setResourceType(ri.getResourceType());
- resources.get(ri.getName()).setUnits(ri.getUnits());
- resources.get(ri.getName()).setValue(value);
- } else {
+ Integer index = indexMap.get(entry.getKey());
+ if (index == null) {
LOG.warn("Got unknown resource type: " + ri.getName() + "; skipping");
+ } else {
+ resources[index].setResourceType(ri.getResourceType());
+ resources[index].setUnits(ri.getUnits());
+ resources[index].setValue(value);
}
}
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
this.setMemorySize(p.getMemory());
this.setVirtualCores(p.getVirtualCores());
}
@@ -173,79 +169,67 @@ public class ResourcePBImpl extends Resource {
if (!resource.equals(resourceInformation.getName())) {
resourceInformation.setName(resource);
}
- initResources();
- if (resources.containsKey(resource)) {
- ResourceInformation.copy(resourceInformation, resources.get(resource));
- }
+ ResourceInformation storedResourceInfo = getResourceInformation(resource);
+ ResourceInformation.copy(resourceInformation, storedResourceInfo);
}
@Override
public void setResourceValue(String resource, Long value)
throws ResourceNotFoundException {
maybeInitBuilder();
- initResources();
if (resource == null) {
throw new IllegalArgumentException("resource type object cannot be null");
}
- if (resources == null || (!resources.containsKey(resource))) {
- throw new ResourceNotFoundException(
- "Resource " + resource + " not found");
- }
- resources.get(resource).setValue(value);
+ getResourceInformation(resource).setValue(value);
}
@Override
- public Map<String, ResourceInformation> getResources() {
+ public ResourceInformation[] getResources() {
initResources();
- return readOnlyResources;
+ return super.getResources();
}
@Override
- public ResourceInformation getResourceInformation(String resource) {
+ public ResourceInformation getResourceInformation(String resource)
+ throws ResourceNotFoundException {
initResources();
- if (this.resources.containsKey(resource)) {
- return this.resources.get(resource);
- }
- throw new ResourceNotFoundException("Could not find entry for " + resource);
+ return super.getResourceInformation(resource);
}
@Override
- public Long getResourceValue(String resource) {
+ public long getResourceValue(String resource)
+ throws ResourceNotFoundException {
initResources();
- if (this.resources.containsKey(resource)) {
- return this.resources.get(resource).getValue();
- }
- throw new ResourceNotFoundException("Could not find entry for " + resource);
+ return super.getResourceValue(resource);
}
private void initResourcesMap() {
if (resources == null) {
- resources = new HashMap<>();
- Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
if (types == null) {
throw new YarnRuntimeException(
"Got null return value from ResourceUtils.getResourceTypes()");
}
- for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
- resources.put(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
+
+ resources = new ResourceInformation[types.length];
+ readOnlyResources = new ResourceInformation[types.length];
+ for (ResourceInformation entry : types) {
+ int index = ResourceUtils.getResourceTypeIndex().get(entry.getName());
+ resources[index] = ResourceInformation.newInstance(entry);
}
- readOnlyResources = Collections.unmodifiableMap(resources);
}
}
synchronized private void mergeLocalToBuilder() {
builder.clearResourceValueMap();
- if (resources != null && !resources.isEmpty()) {
- for (Map.Entry<String, ResourceInformation> entry :
- resources.entrySet()) {
- ResourceInformationProto.Builder e =
- ResourceInformationProto.newBuilder();
- e.setKey(entry.getKey());
- e.setUnits(entry.getValue().getUnits());
- e.setType(
- ProtoUtils.converToProtoFormat(entry.getValue().getResourceType()));
- e.setValue(entry.getValue().getValue());
+ if(resources != null && resources.length != 0) {
+ for (ResourceInformation resInfo : resources) {
+ ResourceInformationProto.Builder e = ResourceInformationProto
+ .newBuilder();
+ e.setKey(resInfo.getName());
+ e.setUnits(resInfo.getUnits());
+ e.setType(ProtoUtils.converToProtoFormat(resInfo.getResourceType()));
+ e.setValue(resInfo.getValue());
builder.addResourceValueMap(e);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 79bb03d..ea5c8a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -23,10 +23,9 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
-import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.Set;
/**
* A {@link ResourceCalculator} which uses the concept of
@@ -56,10 +55,10 @@ public class DominantResourceCalculator extends ResourceCalculator {
LogFactory.getLog(DominantResourceCalculator.class);
- private Set<String> resourceNames;
+ private String[] resourceNames;
public DominantResourceCalculator() {
- resourceNames = ResourceUtils.getResourceTypes().keySet();
+ resourceNames = ResourceUtils.getResourceNamesArray();
}
/**
@@ -88,7 +87,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
} else if (diff <= -1) {
rhsGreater = true;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + rName, ye);
}
@@ -163,7 +162,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
.getValue();
min = min < tmp ? min : tmp;
max = max > tmp ? max : tmp;
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -187,7 +186,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
long tmp = availableResource.getValue() / requiredResourceValue;
min = min < tmp ? min : tmp;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -206,14 +205,9 @@ public class DominantResourceCalculator extends ResourceCalculator {
@Override
public boolean isInvalidDivisor(Resource r) {
- for (String resource : resourceNames) {
- try {
- if (r.getResourceValue(resource).equals(0L)) {
- return true;
- }
- } catch (YarnException ye) {
- throw new IllegalArgumentException(
- "Error getting resource value for " + resource, ye);
+ for (ResourceInformation res : r.getResources()) {
+ if (res.getValue() == 0L) {
+ return true;
}
}
return false;
@@ -235,7 +229,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
float tmp =
(float) aResourceInformation.getValue() / (float) bResourceValue;
ratio = ratio > tmp ? ratio : tmp;
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -256,7 +250,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
ret.getResourceInformation(resource);
resourceInformation.setValue(
divideAndCeil(resourceInformation.getValue(), denominator));
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -307,7 +301,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
tmp.setValue(Math.min(value, maximumValue));
ret.setResourceInformation(resource, tmp);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -347,7 +341,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
ResourceInformation
.copy(rResourceInformation, ret.getResourceInformation(resource));
ret.getResourceInformation(resource).setValue(value);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -372,28 +366,29 @@ public class DominantResourceCalculator extends ResourceCalculator {
Resource ret = Resource.newInstance(r);
for (String resource : resourceNames) {
try {
- ResourceInformation rResourceInformation =
- r.getResourceInformation(resource);
- ResourceInformation stepFactorResourceInformation =
- stepFactor.getResourceInformation(resource);
+ ResourceInformation rResourceInformation = r
+ .getResourceInformation(resource);
+ ResourceInformation stepFactorResourceInformation = stepFactor
+ .getResourceInformation(resource);
ResourceInformation tmp = ret.getResourceInformation(resource);
- Long rValue = rResourceInformation.getValue();
- Long stepFactorValue = UnitsConversionUtil
- .convert(stepFactorResourceInformation.getUnits(),
- rResourceInformation.getUnits(),
- stepFactorResourceInformation.getValue());
- Long value;
+ long rValue = rResourceInformation.getValue();
+ long stepFactorValue = UnitsConversionUtil.convert(
+ stepFactorResourceInformation.getUnits(),
+ rResourceInformation.getUnits(),
+ stepFactorResourceInformation.getValue());
+ long value;
if (stepFactorValue != 0) {
- value = roundUp ?
- roundUp((long) Math.ceil(rValue * by), stepFactorValue) :
- roundDown((long) (rValue * by), stepFactorValue);
+ value = roundUp
+ ? roundUp((long) Math.ceil(rValue * by), stepFactorValue)
+ : roundDown((long) (rValue * by), stepFactorValue);
} else {
- value =
- roundUp ? (long) Math.ceil(rValue * by) : (long) (rValue * by);
+ value = roundUp
+ ? (long) Math.ceil(rValue * by)
+ : (long) (rValue * by);
}
tmp.setValue(value);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
throw new IllegalArgumentException(
"Error getting resource information for " + resource, ye);
}
@@ -416,7 +411,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
if(sResourceValue > bResourceInformation.getValue()) {
return false;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
return false;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index d143e93..f62114d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -18,30 +18,31 @@
package org.apache.hadoop.yarn.util.resource;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceInformation;
+import org.apache.hadoop.yarn.api.records.impl.BaseResource;
import org.apache.hadoop.yarn.exceptions.ResourceNotFoundException;
-import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.hadoop.yarn.util.UnitsConversionUtil;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.Map;
+import java.util.Arrays;
@InterfaceAudience.LimitedPrivate({ "YARN", "MapReduce" })
@Unstable
public class Resources {
+ private static final Log LOG =
+ LogFactory.getLog(Resources.class);
+
/**
* Helper class to create a resource with a fixed value for all resource
* types. For example, a NONE resource which returns 0 for any resource type.
*/
- static class FixedValueResource extends Resource {
+ static class FixedValueResource extends BaseResource {
- private Map<String, ResourceInformation> resources;
private long resourceValue;
private String name;
@@ -53,7 +54,7 @@ public class Resources {
FixedValueResource(String rName, long value) {
this.resourceValue = value;
this.name = rName;
- resources = initResourceMap();
+ initResourceMap();
}
private int resourceValueToInt() {
@@ -96,31 +97,6 @@ public class Resources {
}
@Override
- public Map<String, ResourceInformation> getResources() {
- return Collections.unmodifiableMap(this.resources);
- }
-
- @Override
- public ResourceInformation getResourceInformation(String resource)
- throws YarnException {
- if (resources.containsKey(resource)) {
- ResourceInformation value = this.resources.get(resource);
- ResourceInformation ret = ResourceInformation.newInstance(value);
- ret.setValue(resourceValue);
- return ret;
- }
- throw new YarnException("" + resource + " not found");
- }
-
- @Override
- public Long getResourceValue(String resource) throws YarnException {
- if (resources.containsKey(resource)) {
- return resourceValue;
- }
- throw new YarnException("" + resource + " not found");
- }
-
- @Override
public void setResourceInformation(String resource,
ResourceInformation resourceInformation)
throws ResourceNotFoundException {
@@ -133,24 +109,24 @@ public class Resources {
throw new RuntimeException(name + " cannot be modified!");
}
- private Map<String, ResourceInformation> initResourceMap() {
- Map<String, ResourceInformation> tmp = new HashMap<>();
- Map<String, ResourceInformation> types = ResourceUtils.getResourceTypes();
+ private void initResourceMap() {
+ ResourceInformation[] types = ResourceUtils.getResourceTypesArray();
if (types != null) {
- for (Map.Entry<String, ResourceInformation> entry : types.entrySet()) {
- tmp.put(entry.getKey(),
- ResourceInformation.newInstance(entry.getValue()));
- tmp.get(entry.getKey()).setValue(resourceValue);
+ resources = new ResourceInformation[types.length];
+ readOnlyResources = new ResourceInformation[types.length];
+ for (int index = 0; index < types.length; index++) {
+ resources[index] = ResourceInformation.newInstance(types[index]);
+ resources[index].setValue(resourceValue);
+
+ // this is a fix for getVirtualCores returning an int
+ if (resourceValue > Integer.MAX_VALUE && ResourceInformation.VCORES
+ .getName().equals(resources[index].getName())) {
+ resources[index].setValue((long) Integer.MAX_VALUE);
+ }
}
}
- // this is a fix for getVirtualCores returning an int
- if (resourceValue > Integer.MAX_VALUE) {
- tmp.get(ResourceInformation.VCORES.getName())
- .setValue((long) Integer.MAX_VALUE);
- }
- return tmp;
+ readOnlyResources = Arrays.copyOf(resources, resources.length);
}
-
}
public static Resource createResource(int memory) {
@@ -197,17 +173,19 @@ public class Resources {
}
public static Resource addTo(Resource lhs, Resource rhs) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -219,17 +197,19 @@ public class Resources {
}
public static Resource subtractFrom(Resource lhs, Resource rhs) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
lhs.setResourceValue(name, lhsValue.getValue() - convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -263,10 +243,9 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
- ResourceInformation lhsValue = entry.getValue();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
+ ResourceInformation lhsValue = entry;
lhs.setResourceValue(name, (long) (lhsValue.getValue() * by));
}
return lhs;
@@ -282,17 +261,21 @@ public class Resources {
*/
public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) {
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = (long) (UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue()) * by);
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (long) (((rhsValue.getUnits()
+ .equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue()))
+ * by);
lhs.setResourceValue(name, lhsValue.getValue() + convertedRhs);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -311,10 +294,9 @@ public class Resources {
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- for (Map.Entry<String, ResourceInformation> entry : out.getResources()
- .entrySet()) {
- String name = entry.getKey();
- ResourceInformation lhsValue = entry.getValue();
+ for (ResourceInformation entry : out.getResources()) {
+ String name = entry.getName();
+ ResourceInformation lhsValue = entry;
out.setResourceValue(name, (long) (lhsValue.getValue() * by));
}
return out;
@@ -416,19 +398,21 @@ public class Resources {
}
public static boolean fitsIn(Resource smaller, Resource bigger) {
- for (Map.Entry<String, ResourceInformation> entry : smaller.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : smaller.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = bigger.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
if(lhsValue.getValue() > convertedRhs) {
return false;
}
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
return false;
}
}
@@ -442,19 +426,21 @@ public class Resources {
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
ResourceInformation outInfo =
lhsValue.getValue() < convertedRhs ? lhsValue : rhsValue;
ret.setResourceInformation(name, outInfo);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
@@ -463,19 +449,21 @@ public class Resources {
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- for (Map.Entry<String, ResourceInformation> entry : lhs.getResources()
- .entrySet()) {
- String name = entry.getKey();
+ for (ResourceInformation entry : lhs.getResources()) {
+ String name = entry.getName();
try {
ResourceInformation rhsValue = rhs.getResourceInformation(name);
- ResourceInformation lhsValue = entry.getValue();
- long convertedRhs = UnitsConversionUtil
- .convert(rhsValue.getUnits(), lhsValue.getUnits(),
- rhsValue.getValue());
+ ResourceInformation lhsValue = entry;
+
+ long convertedRhs = (rhsValue.getUnits().equals(lhsValue.getUnits()))
+ ? rhsValue.getValue()
+ : UnitsConversionUtil.convert(rhsValue.getUnits(),
+ lhsValue.getUnits(), rhsValue.getValue());
ResourceInformation outInfo =
lhsValue.getValue() > convertedRhs ? lhsValue : rhsValue;
ret.setResourceInformation(name, outInfo);
- } catch (YarnException ye) {
+ } catch (ResourceNotFoundException ye) {
+ LOG.warn("Resource is missing:" + ye.getMessage());
continue;
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
index 38554b6..b530150 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceUtils.java
@@ -276,13 +276,17 @@ public class TestResourceUtils {
String resourceFile = entry.getKey();
ResourceUtils.resetNodeResources();
File dest;
- File source =
- new File(conf.getClassLoader().getResource(resourceFile).getFile());
+ File source = new File(
+ conf.getClassLoader().getResource(resourceFile).getFile());
dest = new File(source.getParent(), "node-resources.xml");
FileUtils.copyFile(source, dest);
- Map<String, ResourceInformation> actual =
- ResourceUtils.getNodeResourceInformation(conf);
- Assert.assertEquals(entry.getValue().getResources(), actual);
+ Map<String, ResourceInformation> actual = ResourceUtils
+ .getNodeResourceInformation(conf);
+ Assert.assertEquals(actual.size(),
+ entry.getValue().getResources().length);
+ for (ResourceInformation resInfo : entry.getValue().getResources()) {
+ Assert.assertEquals(resInfo, actual.get(resInfo.getName()));
+ }
dest.delete();
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
index 1555e55..a8404fb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResources.java
@@ -105,7 +105,7 @@ public class TestResources {
unboundedClone.compareTo(createResource(0, Integer.MAX_VALUE)) > 0);
}
- @Test(timeout=10000)
+ @Test(timeout = 10000)
public void testCompareToWithNoneResource() {
assertTrue(Resources.none().compareTo(createResource(0, 0)) == 0);
assertTrue(Resources.none().compareTo(createResource(1, 0)) < 0);
@@ -114,7 +114,6 @@ public class TestResources {
assertTrue(Resources.none().compareTo(createResource(1, 0, 0)) < 0);
assertTrue(Resources.none().compareTo(createResource(0, 1, 0)) < 0);
assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
- assertTrue(Resources.none().compareTo(createResource(0, 0, 1)) < 0);
}
@Test(timeout=10000)
@@ -246,7 +245,9 @@ public class TestResources {
}
@Test
- public void testMultiplyAndAddTo() {
+ public void testMultiplyAndAddTo() throws Exception {
+ unsetExtraResourceType();
+ setupExtraResourceType();
assertEquals(createResource(6, 4),
multiplyAndAddTo(createResource(3, 1), createResource(2, 2), 1.5));
assertEquals(createResource(6, 4, 0),
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index 8ef6d99..0982ef9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -178,16 +178,15 @@ public class RMAppAttemptMetrics {
private void updateUsageMap(Resource allocated, long deltaUsedMillis,
Map<String, AtomicLong> targetMap) {
- for (Map.Entry<String, ResourceInformation> entry : allocated.getResources()
- .entrySet()) {
+ for (ResourceInformation entry : allocated.getResources()) {
AtomicLong resourceUsed;
- if (!targetMap.containsKey(entry.getKey())) {
+ if (!targetMap.containsKey(entry.getName())) {
resourceUsed = new AtomicLong(0);
- targetMap.put(entry.getKey(), resourceUsed);
+ targetMap.put(entry.getName(), resourceUsed);
}
- resourceUsed = targetMap.get(entry.getKey());
- resourceUsed.addAndGet((entry.getValue().getValue() * deltaUsedMillis)
+ resourceUsed = targetMap.get(entry.getName());
+ resourceUsed.addAndGet((entry.getValue() * deltaUsedMillis)
/ DateUtils.MILLIS_PER_SECOND);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 1785dfb..1e1f327 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -1006,13 +1006,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
for (RMContainer rmContainer : this.liveContainers.values()) {
long usedMillis = currentTimeMillis - rmContainer.getCreationTime();
Resource resource = rmContainer.getContainer().getResource();
- for (Map.Entry<String, ResourceInformation> entry : resource
- .getResources().entrySet()) {
+ for (ResourceInformation entry : resource.getResources()) {
long value = RMServerUtils
- .getOrDefault(resourceSecondsMap, entry.getKey(), 0L);
- value += entry.getValue().getValue() * usedMillis
+ .getOrDefault(resourceSecondsMap, entry.getName(), 0L);
+ value += entry.getValue() * usedMillis
/ DateUtils.MILLIS_PER_SECOND;
- resourceSecondsMap.put(entry.getKey(), value);
+ resourceSecondsMap.put(entry.getName(), value);
}
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
index 887b854..81491b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/SchedulerInfo.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+import java.util.Arrays;
import java.util.EnumSet;
import javax.xml.bind.annotation.XmlRootElement;
@@ -73,7 +74,7 @@ public class SchedulerInfo {
}
public String getSchedulerResourceTypes() {
- return minAllocResource.getResource().getResources().keySet().toString();
+ return Arrays.toString(minAllocResource.getResource().getResources());
}
public int getMaxClusterLevelAppPriority() {
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index 8ae5d7d..187fa3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -249,6 +249,7 @@ public class TestAppManager{
ResourceScheduler scheduler = mockResourceScheduler();
((RMContextImpl)rmContext).setScheduler(scheduler);
Configuration conf = new Configuration();
+ conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
((RMContextImpl) rmContext).setYarnConfiguration(conf);
ApplicationMasterService masterService =
new ApplicationMasterService(rmContext, scheduler);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org