You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2017/12/08 03:06:48 UTC
[02/13] hadoop git commit: YARN-7564. Cleanup to fix checkstyle
issues of YARN-5881 branch. Contributed by Sunil G.
YARN-7564. Cleanup to fix checkstyle issues of YARN-5881 branch. Contributed by Sunil G.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/daa1cdd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/daa1cdd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/daa1cdd0
Branch: refs/heads/trunk
Commit: daa1cdd062657a47acbf4b23f895860296241199
Parents: 1012b90
Author: Sunil G <su...@apache.org>
Authored: Tue Nov 28 14:07:09 2017 +0530
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Dec 7 18:56:54 2017 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/util/StringUtils.java | 18 ++++++-------
.../impl/pb/QueueConfigurationsPBImpl.java | 10 +++----
.../resource/DominantResourceCalculator.java | 2 +-
.../yarn/util/resource/ResourceCalculator.java | 12 ++++-----
.../scheduler/AbstractResourceUsage.java | 8 +++++-
.../scheduler/ResourceUsage.java | 6 -----
.../scheduler/capacity/CSQueue.java | 28 +++++++++++++++++---
.../CapacitySchedulerConfiguration.java | 28 ++++++++++++--------
.../scheduler/capacity/TestReservations.java | 3 +--
9 files changed, 70 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
index 1be8a08..ebe7013 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
@@ -1171,16 +1171,16 @@ public class StringUtils {
* @return <code>true</code> if only contains letters, and is non-null
*/
public static boolean isAlpha(String str) {
- if (str == null) {
- return false;
- }
- int sz = str.length();
- for (int i = 0; i < sz; i++) {
- if (Character.isLetter(str.charAt(i)) == false) {
- return false;
- }
+ if (str == null) {
+ return false;
+ }
+ int sz = str.length();
+ for (int i = 0; i < sz; i++) {
+ if (!Character.isLetter(str.charAt(i))) {
+ return false;
}
- return true;
+ }
+ return true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
index 80ef4b2..8266013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueConfigurationsPBImpl.java
@@ -236,12 +236,12 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
}
@Override
- public void setConfiguredMinCapacity(Resource configuredMinResource) {
+ public void setConfiguredMinCapacity(Resource minResource) {
maybeInitBuilder();
- if (configuredMinResource == null) {
+ if (minResource == null) {
builder.clearConfiguredMinCapacity();
}
- this.configuredMinResource = configuredMinResource;
+ this.configuredMinResource = minResource;
}
@Override
@@ -259,11 +259,11 @@ public class QueueConfigurationsPBImpl extends QueueConfigurations {
}
@Override
- public void setConfiguredMaxCapacity(Resource configuredMaxResource) {
+ public void setConfiguredMaxCapacity(Resource maxResource) {
maybeInitBuilder();
if (configuredMaxResource == null) {
builder.clearConfiguredMaxCapacity();
}
- this.configuredMaxResource = configuredMaxResource;
+ this.configuredMaxResource = maxResource;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 26ddc18..6fed23b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -515,7 +515,7 @@ public class DominantResourceCalculator extends ResourceCalculator {
}
return ret;
}
-
+
@Override
public Resource multiplyAndNormalizeUp(Resource r, double by,
Resource stepFactor) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
index dfd6684..1c42126 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
@@ -127,12 +127,12 @@ public abstract class ResourceCalculator {
Resource r, double by, Resource stepFactor);
/**
- * Multiply resource <code>r</code> by factor <code>by</code>
+ * Multiply resource <code>r</code> by factor <code>by</code>
* and normalize up using step-factor <code>stepFactor</code>.
- *
+ *
* @param r resource to be multiplied
* @param by multiplier array for all resource types
- * @param stepFactor factor by which to normalize up
+ * @param stepFactor factor by which to normalize up
* @return resulting normalized resource
*/
public abstract Resource multiplyAndNormalizeUp(
@@ -149,7 +149,7 @@ public abstract class ResourceCalculator {
*/
public abstract Resource multiplyAndNormalizeDown(
Resource r, double by, Resource stepFactor);
-
+
/**
* Normalize resource <code>r</code> given the base
* <code>minimumResource</code> and verify against max allowed
@@ -162,9 +162,7 @@ public abstract class ResourceCalculator {
* @return normalized resource
*/
public abstract Resource normalize(Resource r, Resource minimumResource,
- Resource maximumResource,
- Resource stepFactor);
-
+ Resource maximumResource, Resource stepFactor);
/**
* Round-up resource <code>r</code> given factor <code>stepFactor</code>.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
index 3203543..1406fec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractResourceUsage.java
@@ -55,7 +55,10 @@ public class AbstractResourceUsage {
usages.put(CommonNodeLabelsManager.NO_LABEL, noLabelUsages);
}
- // Usage enum here to make implement cleaner
+ /**
+ * Use enum here to make implementation more cleaner and readable.
+ * Indicates array index for each resource usage type.
+ */
public enum ResourceType {
// CACHED_USED and CACHED_PENDING may be read by anyone, but must only
// be written by ordering policies
@@ -70,6 +73,9 @@ public class AbstractResourceUsage {
}
}
+ /**
+ * UsageByLabel stores resource array for all resource usage types.
+ */
public static class UsageByLabel {
// usage by label, contains all UsageType
private final AtomicReferenceArray<Resource> resArr;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index ede4aec..711a468 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -18,13 +18,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
-import java.util.HashMap;
-import java.util.Map;
import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 624fcc7..6d79b6a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -366,7 +366,7 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
public QueueResourceQuotas getQueueResourceQuotas();
/**
- * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE
+ * Get CapacityConfigType as PERCENTAGE or ABSOLUTE_RESOURCE.
* @return CapacityConfigType
*/
public CapacityConfigType getCapacityConfigType();
@@ -374,24 +374,46 @@ public interface CSQueue extends SchedulerQueue<CSQueue> {
/**
* Get effective capacity of queue. If min/max resource is configured,
* preference will be given to absolute configuration over normal capacity.
- * Also round down the result to normalizeDown.
*
* @param label
* partition
* @return effective queue capacity
*/
Resource getEffectiveCapacity(String label);
+
+ /**
+ * Get effective capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @param factor
+ * factor to normalize down
+ * @return effective queue capacity
+ */
Resource getEffectiveCapacityDown(String label, Resource factor);
/**
* Get effective max capacity of queue. If min/max resource is configured,
* preference will be given to absolute configuration over normal capacity.
- * Also round down the result to normalizeDown.
*
* @param label
* partition
* @return effective max queue capacity
*/
Resource getEffectiveMaxCapacity(String label);
+
+ /**
+ * Get effective max capacity of queue. If min/max resource is configured,
+ * preference will be given to absolute configuration over normal capacity.
+ * Also round down the result to normalizeDown.
+ *
+ * @param label
+ * partition
+ * @param factor
+ * factor to normalize down
+ * @return effective max queue capacity
+ */
Resource getEffectiveMaxCapacityDown(String label, Resource factor);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index e47c8d0..a33d81a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.ReservationACL;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceInformation;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
@@ -320,7 +321,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
@Private
public static final int DEFAULT_MAX_ASSIGN_PER_HEARTBEAT = -1;
- /** Configuring absolute min/max resources in a queue **/
+ /** Configuring absolute min/max resources in a queue. **/
@Private
public static final String MINIMUM_RESOURCE = "min-resource";
@@ -333,6 +334,9 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
private static final Pattern RESOURCE_PATTERN = Pattern.compile(PATTERN_FOR_ABSOLUTE_RESOURCE);
+ /**
+ * Different resource types supported.
+ */
public enum AbsoluteResourceType {
MEMORY, VCORES;
}
@@ -1825,7 +1829,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
// Define resource here.
- Resource resource = Resource.newInstance(0l, 0);
+ Resource resource = Resource.newInstance(0L, 0);
Matcher matcher = RESOURCE_PATTERN.matcher(resourceString);
/*
@@ -1852,7 +1856,7 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
}
// Memory has to be configured always.
- if (resource.getMemorySize() == 0l) {
+ if (resource.getMemorySize() == 0L) {
return Resources.none();
}
@@ -1884,14 +1888,16 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
AbsoluteResourceType resType = AbsoluteResourceType
.valueOf(StringUtils.toUpperCase(splits[0].trim()));
switch (resType) {
- case MEMORY :
- resource.setMemorySize(resourceValue);
- break;
- case VCORES :
- resource.setVirtualCores(resourceValue.intValue());
- break;
- default :
- break;
+ case MEMORY :
+ resource.setMemorySize(resourceValue);
+ break;
+ case VCORES :
+ resource.setVirtualCores(resourceValue.intValue());
+ break;
+ default :
+ resource.setResourceInformation(splits[0].trim(), ResourceInformation
+ .newInstance(splits[0].trim(), units, resourceValue));
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/daa1cdd0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 0855720..8705887 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -904,12 +904,11 @@ public class TestReservations {
String host_1 = "host_1";
FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
8 * GB);
-
+
Resource clusterResource = Resources.createResource(2 * 8 * GB);
root.updateClusterResource(clusterResource,
new ResourceLimits(clusterResource));
-
// Setup resource-requests
Priority p = TestUtils.createMockPriority(5);
SchedulerRequestKey priorityMap = toSchedulerKey(p);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org