You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by in...@apache.org on 2017/09/12 20:03:02 UTC
[17/50] [abbrv] hadoop git commit: YARN-7030. [YARN-3926] Performance
optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan.
YARN-7030. [YARN-3926] Performance optimizations in Resource and ResourceUtils class. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a6989af1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a6989af1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a6989af1
Branch: refs/heads/HDFS-10467
Commit: a6989af1d91d8b6795862c39d154823f8488c8de
Parents: a0030c8
Author: Sunil G <su...@apache.org>
Authored: Thu Aug 17 21:14:51 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Tue Sep 12 09:19:12 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/records/Resource.java | 6 ++---
.../yarn/api/records/impl/BaseResource.java | 10 +++++---
.../yarn/util/resource/ResourceUtils.java | 25 ++++++++++++++++----
.../hadoop/yarn/util/resource/Resources.java | 16 ++++++-------
4 files changed, 39 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6989af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
index 1e9f213..04579c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/Resource.java
@@ -65,7 +65,7 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Stable
public static Resource newInstance(int memory, int vCores) {
- if (ResourceUtils.getResourceTypesArray().length > 2) {
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory);
ret.setVirtualCores(vCores);
@@ -77,7 +77,7 @@ public abstract class Resource implements Comparable<Resource> {
@Public
@Stable
public static Resource newInstance(long memory, int vCores) {
- if (ResourceUtils.getResourceTypesArray().length > 2) {
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
Resource ret = Records.newRecord(Resource.class);
ret.setMemorySize(memory);
ret.setVirtualCores(vCores);
@@ -91,7 +91,7 @@ public abstract class Resource implements Comparable<Resource> {
public static Resource newInstance(Resource resource) {
Resource ret = Resource.newInstance(resource.getMemorySize(),
resource.getVirtualCores());
- if (ResourceUtils.getResourceTypesArray().length > 2) {
+ if (ResourceUtils.getNumberOfKnownResourceTypes() > 2) {
Resource.copy(resource, ret);
}
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6989af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
index 83db542..b5cc4d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/BaseResource.java
@@ -63,6 +63,11 @@ public class BaseResource extends Resource {
protected ResourceInformation[] resources = null;
protected ResourceInformation[] readOnlyResources = null;
+ // Number of mandatory resources, this is added to avoid invoke
+ // MandatoryResources.values().length, since values() internally will
+ // copy array, etc.
+ private static final int NUM_MANDATORY_RESOURCES = 2;
+
protected enum MandatoryResources {
MEMORY(0), VCORES(1);
@@ -86,9 +91,8 @@ public class BaseResource extends Resource {
ResourceInformation.MEMORY_MB.getUnits(), memory);
this.vcoresResInfo = ResourceInformation.newInstance(VCORES, "", vcores);
- resources = new ResourceInformation[MandatoryResources.values().length];
- readOnlyResources = new ResourceInformation[MandatoryResources
- .values().length];
+ resources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
+ readOnlyResources = new ResourceInformation[NUM_MANDATORY_RESOURCES];
resources[MandatoryResources.MEMORY.id] = memoryResInfo;
resources[MandatoryResources.VCORES.id] = vcoresResInfo;
readOnlyResources = Arrays.copyOf(resources, resources.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6989af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
index 49feafa..06a5414 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceUtils.java
@@ -74,6 +74,7 @@ public class ResourceUtils {
private static volatile ResourceInformation[] resourceTypesArray;
private static volatile boolean initializedNodeResources = false;
private static volatile Map<String, ResourceInformation> readOnlyNodeResources;
+ private static volatile int numKnownResourceTypes = -1;
static final Log LOG = LogFactory.getLog(ResourceUtils.class);
@@ -308,23 +309,33 @@ public class ResourceUtils {
* @return resourceNamesArray
*/
public static String[] getResourceNamesArray() {
- getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ initializeResourceTypesIfNeeded(null,
+ YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
return resourceNamesArray;
}
public static ResourceInformation[] getResourceTypesArray() {
- getResourceTypes(null, YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ initializeResourceTypesIfNeeded(null,
+ YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
return resourceTypesArray;
}
+ public static int getNumberOfKnownResourceTypes() {
+ if (numKnownResourceTypes < 0) {
+ initializeResourceTypesIfNeeded(null,
+ YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
+ }
+ return numKnownResourceTypes;
+ }
+
private static Map<String, ResourceInformation> getResourceTypes(
Configuration conf) {
return getResourceTypes(conf,
YarnConfiguration.RESOURCE_TYPES_CONFIGURATION_FILE);
}
- private static Map<String, ResourceInformation> getResourceTypes(
- Configuration conf, String resourceFile) {
+ private static void initializeResourceTypesIfNeeded(Configuration conf,
+ String resourceFile) {
if (!initializedResources) {
synchronized (ResourceUtils.class) {
if (!initializedResources) {
@@ -346,6 +357,12 @@ public class ResourceUtils {
}
}
}
+ numKnownResourceTypes = resourceTypes.size();
+ }
+
+ private static Map<String, ResourceInformation> getResourceTypes(
+ Configuration conf, String resourceFile) {
+ initializeResourceTypesIfNeeded(conf, resourceFile);
return resourceTypes;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a6989af1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
index 702c1c8..54ef036 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
@@ -173,7 +173,7 @@ public class Resources {
}
public static Resource addTo(Resource lhs, Resource rhs) {
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -197,7 +197,7 @@ public class Resources {
}
public static Resource subtractFrom(Resource lhs, Resource rhs) {
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -243,7 +243,7 @@ public class Resources {
}
public static Resource multiplyTo(Resource lhs, double by) {
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation lhsValue = lhs.getResourceInformation(i);
@@ -266,7 +266,7 @@ public class Resources {
*/
public static Resource multiplyAndAddTo(
Resource lhs, Resource rhs, double by) {
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -299,7 +299,7 @@ public class Resources {
public static Resource multiplyAndRoundDown(Resource lhs, double by) {
Resource out = clone(lhs);
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation lhsValue = lhs.getResourceInformation(i);
@@ -408,7 +408,7 @@ public class Resources {
}
public static boolean fitsIn(Resource smaller, Resource bigger) {
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation rhsValue = bigger.getResourceInformation(i);
@@ -436,7 +436,7 @@ public class Resources {
public static Resource componentwiseMin(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
@@ -460,7 +460,7 @@ public class Resources {
public static Resource componentwiseMax(Resource lhs, Resource rhs) {
Resource ret = createResource(0);
- int maxLength = ResourceUtils.getResourceTypesArray().length;
+ int maxLength = ResourceUtils.getNumberOfKnownResourceTypes();
for (int i = 0; i < maxLength; i++) {
try {
ResourceInformation rhsValue = rhs.getResourceInformation(i);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org