You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2018/04/26 05:15:19 UTC
[1/2] hadoop git commit: HADOOP-15411. AuthenticationFilter should
use Configuration.getPropsWithPrefix instead of iterator. (Suma Shivaprasad
via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 96a57c967 -> af986b442
HADOOP-15411. AuthenticationFilter should use Configuration.getPropsWithPrefix instead of iterator. (Suma Shivaprasad via wangda)
Change-Id: Ia19458a43aceb4ffdb1e8eccc519655235d92ba9
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3559d8b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3559d8b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3559d8b1
Branch: refs/heads/trunk
Commit: 3559d8b1dacf5cf207424de37cb6ba8865d26ffe
Parents: 96a57c9
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Apr 25 22:09:37 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 25 22:09:37 2018 -0700
----------------------------------------------------------------------
.../hadoop/security/AuthenticationFilterInitializer.java | 10 +++-------
1 file changed, 3 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3559d8b1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
index ca221f5..7e6b3a8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/AuthenticationFilterInitializer.java
@@ -69,14 +69,10 @@ public class AuthenticationFilterInitializer extends FilterInitializer {
//setting the cookie path to root '/' so it is used for all resources.
filterConfig.put(AuthenticationFilter.COOKIE_PATH, "/");
+ Map<String, String> propsWithPrefix = conf.getPropsWithPrefix(prefix);
- for (Map.Entry<String, String> entry : conf) {
- String name = entry.getKey();
- if (name.startsWith(prefix)) {
- String value = conf.get(name);
- name = name.substring(prefix.length());
- filterConfig.put(name, value);
- }
+ for (Map.Entry<String, String> entry : propsWithPrefix.entrySet()) {
+ filterConfig.put(entry.getKey(), entry.getValue());
}
//Resolve _HOST into bind address
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-8193. YARN RM hangs abruptly (stops
allocating resources) when running successive applications. (Zian Chen via
wangda)
Posted by wa...@apache.org.
YARN-8193. YARN RM hangs abruptly (stops allocating resources) when running successive applications. (Zian Chen via wangda)
Change-Id: Ia83dd2499ee9000b9e09ae5a932f21a13c0ddee6
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af986b44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af986b44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af986b44
Branch: refs/heads/trunk
Commit: af986b442b7be0f2fad1241ca9e267a15c20bf43
Parents: 3559d8b
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Apr 25 22:10:18 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Apr 25 22:10:18 2018 -0700
----------------------------------------------------------------------
.../allocator/RegularContainerAllocator.java | 42 +++++++++++++++-----
1 file changed, 31 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af986b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index afa468b..99deb1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -179,11 +179,22 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// This is to make sure non-partitioned-resource-request will prefer
// to be allocated to non-partitioned nodes
int missedNonPartitionedRequestSchedulingOpportunity = 0;
+ AppPlacementAllocator appPlacementAllocator =
+ appInfo.getAppPlacementAllocator(schedulerKey);
+ if (null == appPlacementAllocator){
+ // This is possible when #pending resource decreased by a different
+ // thread.
+ ActivitiesLogger.APP.recordSkippedAppActivityWithoutAllocation(
+ activitiesManager, node, application, priority,
+ ActivityDiagnosticConstant.PRIORITY_SKIPPED_BECAUSE_NULL_ANY_REQUEST);
+ return ContainerAllocation.PRIORITY_SKIPPED;
+ }
+ String requestPartition =
+ appPlacementAllocator.getPrimaryRequestedNodePartition();
+
// Only do this when request associated with given scheduler key accepts
// NO_LABEL under RESPECT_EXCLUSIVITY mode
- if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL,
- appInfo.getAppPlacementAllocator(schedulerKey)
- .getPrimaryRequestedNodePartition())) {
+ if (StringUtils.equals(RMNodeLabelsManager.NO_LABEL, requestPartition)) {
missedNonPartitionedRequestSchedulingOpportunity =
application.addMissedNonPartitionedRequestSchedulingOpportunity(
schedulerKey);
@@ -261,12 +272,9 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
return result;
}
- public float getLocalityWaitFactor(
- SchedulerRequestKey schedulerKey, int clusterNodes) {
+ public float getLocalityWaitFactor(int uniqAsks, int clusterNodes) {
// Estimate: Required unique resources (i.e. hosts + racks)
- int requiredResources = Math.max(
- application.getAppPlacementAllocator(schedulerKey)
- .getUniqueLocationAsks() - 1, 0);
+ int requiredResources = Math.max(uniqAsks - 1, 0);
// waitFactor can't be more than '1'
// i.e. no point skipping more than clustersize opportunities
@@ -296,10 +304,16 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
if (rmContext.getScheduler().getNumClusterNodes() == 0) {
return false;
}
+
+ int uniqLocationAsks = 0;
+ AppPlacementAllocator appPlacementAllocator =
+ application.getAppPlacementAllocator(schedulerKey);
+ if (appPlacementAllocator != null) {
+ uniqLocationAsks = appPlacementAllocator.getUniqueLocationAsks();
+ }
// If we have only ANY requests for this schedulerKey, we should not
// delay its scheduling.
- if (application.getAppPlacementAllocator(schedulerKey)
- .getUniqueLocationAsks() == 1) {
+ if (uniqLocationAsks == 1) {
return true;
}
@@ -313,7 +327,7 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
} else {
long requiredContainers =
application.getOutstandingAsksCount(schedulerKey);
- float localityWaitFactor = getLocalityWaitFactor(schedulerKey,
+ float localityWaitFactor = getLocalityWaitFactor(uniqLocationAsks,
rmContext.getScheduler().getNumClusterNodes());
// Cap the delay by the number of nodes in the cluster.
return (Math.min(rmContext.getScheduler().getNumClusterNodes(),
@@ -806,6 +820,12 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
application.getAppSchedulingInfo().getAppPlacementAllocator(
schedulerKey);
+ // This could be null when #pending request decreased by another thread.
+ if (schedulingPS == null) {
+ return new ContainerAllocation(reservedContainer, null,
+ AllocationState.QUEUE_SKIPPED);
+ }
+
result = ContainerAllocation.PRIORITY_SKIPPED;
Iterator<FiCaSchedulerNode> iter = schedulingPS.getPreferredNodeIterator(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org