You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2018/08/09 19:03:51 UTC
[1/2] hadoop git commit: YARN-8136. Add version attribute to site doc
examples and quickstart. (Eric Yang via wangda)
Repository: hadoop
Updated Branches:
refs/heads/trunk 5326a7906 -> 8244abb7a
YARN-8136. Add version attribute to site doc examples and quickstart. (Eric Yang via wangda)
Change-Id: I4541b239f490ca0a6edf9698e0d3deaf83669151
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8244abb7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8244abb7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8244abb7
Branch: refs/heads/trunk
Commit: 8244abb7aeb768b73682b8c9a26516a9cf06bca5
Parents: 344c335
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Aug 9 11:03:46 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Aug 9 11:04:02 2018 -0700
----------------------------------------------------------------------
.../hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md | 5 ++++-
.../src/site/markdown/yarn-service/QuickStart.md | 1 +
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8244abb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
index 03fec79..73e00b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/Examples.md
@@ -30,6 +30,7 @@ Note this example requires registry DNS.
```
{
"name": "httpd-service",
+ "version": "1.0",
"lifetime": "3600",
"components": [
{
@@ -169,9 +170,10 @@ Then visit port 8080 for each IP to view the pages.
Docker images may have built with ENTRYPOINT to enable start up of docker image without any parameters.
When passing parameters to ENTRYPOINT enabled image, `launch_command` is delimited by comma (,).
+```
{
"name": "sleeper-service",
- "version": "1",
+ "version": "1.0",
"components" :
[
{
@@ -198,3 +200,4 @@ When passing parameters to ENTRYPOINT enabled image, `launch_command` is delimit
}
]
}
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8244abb7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
index e144320..bba9bb8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/yarn-service/QuickStart.md
@@ -40,6 +40,7 @@ Below is a simple service definition that launches sleep containers on YARN by w
```
{
"name": "sleeper-service",
+ "version": "1.0",
"components" :
[
{
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[2/2] hadoop git commit: YARN-8588. Logging improvements for better
debuggability. (Suma Shivaprasad via wangda)
Posted by wa...@apache.org.
YARN-8588. Logging improvements for better debuggability. (Suma Shivaprasad via wangda)
Change-Id: I66aa4b0ec031ae5ce0fae558e2f8cbcbbfebc442
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/344c335a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/344c335a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/344c335a
Branch: refs/heads/trunk
Commit: 344c335a920e6f32a35ebace0a118a9dc4a22fb7
Parents: 5326a79
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Aug 9 11:03:00 2018 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Thu Aug 9 11:04:02 2018 -0700
----------------------------------------------------------------------
.../capacity/AutoCreatedLeafQueueConfig.java | 5 ++
.../capacity/QueueManagementChange.java | 2 +-
.../QueueManagementDynamicEditPolicy.java | 36 ++++++--------
.../GuaranteedOrZeroCapacityOverTimePolicy.java | 50 ++++++++++++--------
4 files changed, 52 insertions(+), 41 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
index 5952250..87ef1c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AutoCreatedLeafQueueConfig.java
@@ -63,4 +63,9 @@ public class AutoCreatedLeafQueueConfig {
public CapacitySchedulerConfiguration getLeafQueueConfigs() {
return leafQueueConfigs;
}
+
+ @Override public String toString() {
+ return "AutoCreatedLeafQueueConfig{" + "queueCapacities=" + queueCapacities
+ + ", leafQueueConfigs=" + leafQueueConfigs + '}';
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
index 74d9b23..64ba578 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementChange.java
@@ -124,7 +124,7 @@ public abstract class QueueManagementChange {
@Override
public String toString() {
- return "QueueManagementChange{" + "queue=" + queue
+ return "QueueManagementChange{" + "queue=" + queue.getQueueName()
+ ", updatedEntitlementsByPartition=" + queueTemplateUpdate
+ ", queueAction=" + queueAction + ", transitionToQueueState="
+ transitionToQueueState + '}';
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
index 9b0cf7b..ea43ac8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueManagementDynamicEditPolicy.java
@@ -19,8 +19,9 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
@@ -50,8 +51,8 @@ import java.util.Set;
*/
public class QueueManagementDynamicEditPolicy implements SchedulingEditPolicy {
- private static final Log LOG =
- LogFactory.getLog(QueueManagementDynamicEditPolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(QueueManagementDynamicEditPolicy.class);
private Clock clock;
@@ -90,7 +91,7 @@ public class QueueManagementDynamicEditPolicy implements SchedulingEditPolicy {
@Override
public void init(final Configuration config, final RMContext context,
final ResourceScheduler sched) {
- LOG.info("Queue Management Policy monitor:" + this.
+ LOG.info("Queue Management Policy monitor: {}" + this.
getClass().getCanonicalName());
assert null == scheduler : "Unexpected duplicate call to init";
if (!(sched instanceof CapacityScheduler)) {
@@ -189,13 +190,7 @@ public class QueueManagementDynamicEditPolicy implements SchedulingEditPolicy {
parentQueue.getAutoCreatedQueueManagementPolicy();
long startTime = 0;
try {
- if (LOG.isDebugEnabled()) {
- LOG.debug(MessageFormat
- .format("Trying to use {0} to compute preemption "
- + "candidates",
- policyClazz.getClass().getName()));
- startTime = clock.getTime();
- }
+ startTime = clock.getTime();
queueManagementChanges = policyClazz.computeQueueManagementChanges();
@@ -209,15 +204,14 @@ public class QueueManagementDynamicEditPolicy implements SchedulingEditPolicy {
}
if (LOG.isDebugEnabled()) {
- LOG.debug(MessageFormat.format("{0} uses {1} millisecond"
- + " to run",
- policyClazz.getClass().getName(), clock.getTime()
- - startTime));
+ LOG.debug("{} uses {} millisecond" + " to run",
+ policyClazz.getClass().getName(), clock.getTime() - startTime);
if (queueManagementChanges.size() > 0) {
- LOG.debug(" Updated queue management updates for parent queue"
- + " ["
- + parentQueue.getQueueName() + ": [\n" + queueManagementChanges
- .toString() + "\n]");
+ LOG.debug(" Updated queue management changes for parent queue" + " "
+ + "{}: [{}]", parentQueue.getQueueName(),
+ queueManagementChanges.size() < 25 ?
+ queueManagementChanges.toString() :
+ queueManagementChanges.size());
}
}
} catch (YarnException e) {
@@ -232,7 +226,7 @@ public class QueueManagementDynamicEditPolicy implements SchedulingEditPolicy {
"Skipping queue management updates for parent queue "
+ parentQueue
.getQueuePath() + " "
- + "since configuration for auto creating queue's beyond "
+ + "since configuration for auto creating queues beyond "
+ "parent's "
+ "guaranteed capacity is disabled");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/344c335a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
index b2301fd..faa6e6f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/queuemanagement/GuaranteedOrZeroCapacityOverTimePolicy.java
@@ -19,8 +19,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
.queuemanagement;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity
+ .QueueManagementDynamicEditPolicy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler
.SchedulerDynamicEditException;
@@ -81,8 +83,8 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
private CapacitySchedulerContext scheduler;
private ManagedParentQueue managedParentQueue;
- private static final Log LOG = LogFactory.getLog(
- GuaranteedOrZeroCapacityOverTimePolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(GuaranteedOrZeroCapacityOverTimePolicy.class);
private ReentrantReadWriteLock.WriteLock writeLock;
@@ -380,6 +382,17 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
deactivateLeafQueuesIfInActive(managedParentQueue, nodeLabel,
leafQueueEntitlements);
+ if (LOG.isDebugEnabled()) {
+ if ( deactivatedLeafQueues.size() > 0) {
+ LOG.debug("Parent queue = {}, " +
+ ", nodeLabel = {}, deactivated leaf queues = [{}] ",
+ managedParentQueue.getQueueName(), nodeLabel,
+ deactivatedLeafQueues.size() > 25 ? deactivatedLeafQueues
+ .size() : deactivatedLeafQueues);
+
+ }
+ }
+
float deactivatedCapacity = getTotalDeactivatedCapacity(
deactivatedLeafQueues, nodeLabel);
@@ -392,7 +405,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
+ deactivatedCapacity + EPSILON;
if (LOG.isDebugEnabled()) {
- LOG.debug("Parent queue : " + managedParentQueue.getQueueName()
+ LOG.debug("Parent queue = " + managedParentQueue.getQueueName()
+ ", nodeLabel = " + nodeLabel + ", absCapacity = "
+ parentAbsoluteCapacity + ", leafQueueAbsoluteCapacity = "
+ leafQueueTemplateAbsoluteCapacity + ", deactivatedCapacity = "
@@ -409,7 +422,8 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
pendingApps.size());
if (LOG.isDebugEnabled()) {
- LOG.debug("Found " + maxLeafQueuesTobeActivated + " leaf queues"
+ LOG.debug("Parent queue = " + managedParentQueue.getQueueName()
+ + " : Found " + maxLeafQueuesTobeActivated + " leaf queues"
+ " to be activated with " + pendingApps.size() + " apps ");
}
@@ -424,8 +438,9 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
if (LOG.isDebugEnabled()) {
if (leafQueuesToBeActivated.size() > 0) {
- LOG.debug("Activated leaf queues : [" + leafQueuesToBeActivated
- + "]");
+ LOG.debug("Activated leaf queues : [{}]",
+ leafQueuesToBeActivated.size() < 25 ?
+ leafQueuesToBeActivated : leafQueuesToBeActivated.size());
}
}
}
@@ -492,8 +507,9 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
String partition = e.getKey();
if (!newPartitions.contains(partition)) {
itr.remove();
- LOG.info(
- "Removed partition " + partition + " from leaf queue " + "state");
+ LOG.info(managedParentQueue.getQueueName() +
+ " : Removed partition " + partition + " from leaf queue " +
+ "state");
} else{
Map<String, LeafQueueStatePerPartition> queues = e.getValue();
for (
@@ -502,7 +518,9 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
String queue = queueItr.next().getKey();
if (!newQueues.contains(queue)) {
queueItr.remove();
- LOG.info("Removed queue " + queue + " from leaf queue "
+ LOG.info(managedParentQueue.getQueueName() + " : Removed queue"
+ + queue + " from "
+ + "leaf queue "
+ "state from partition " + partition);
}
}
@@ -582,12 +600,6 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
updateToZeroCapacity(capacities, nodeLabel);
deactivatedQueues.put(leafQueue.getQueueName(),
leafQueueTemplateCapacities);
- } else{
- if (LOG.isDebugEnabled()) {
- LOG.debug(" Leaf queue has pending applications or is " + "inactive"
- + " : " + leafQueue.getNumApplications()
- + ".Skipping deactivation for " + leafQueue);
- }
}
} else{
LOG.warn("Could not find queue in scheduler while trying" + " to "
@@ -659,7 +671,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
if (isActive(leafQueue, nodeLabel)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Queue is already active." + " Skipping activation : "
- + queue.getQueuePath());
+ + leafQueue.getQueueName());
}
} else{
activate(leafQueue, nodeLabel);
@@ -668,7 +680,7 @@ public class GuaranteedOrZeroCapacityOverTimePolicy
if (!isActive(leafQueue, nodeLabel)) {
if (LOG.isDebugEnabled()) {
LOG.debug("Queue is already de-activated. Skipping "
- + "de-activation : " + leafQueue.getQueuePath());
+ + "de-activation : " + leafQueue.getQueueName());
}
} else{
deactivate(leafQueue, nodeLabel);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org