You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sj...@apache.org on 2015/09/26 18:05:08 UTC
[01/50] [abbrv] hadoop git commit: HADOOP-12280. Skip unit tests
based on maven profile rather than NativeCodeLoader.isNativeCodeLoaded
(Masatake Iwasaki via Colin P. McCabe)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 934886486 -> 1828ba00b (forced update)
HADOOP-12280. Skip unit tests based on maven profile rather than NativeCodeLoader.isNativeCodeLoaded (Masatake Iwasaki via Colin P. McCabe)
(cherry picked from commit 6f83274afc1eba1159427684d72d8f13778c5a88)
(cherry picked from commit e92107b18f82b3501deaa6170d322a0fb512ec71)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3bd9b745
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3bd9b745
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3bd9b745
Branch: refs/heads/branch-2.6
Commit: 3bd9b7459bfe2e4d81d60498832dc297cd01e003
Parents: d8ddfea
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Aug 4 13:51:04 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 15:02:33 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/crypto/TestCryptoCodec.java | 11 +++--------
.../TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 2 ++
.../org/apache/hadoop/io/TestSequenceFileAppend.java | 5 +++++
.../java/org/apache/hadoop/test/GenericTestUtils.java | 11 +++++++++++
5 files changed, 24 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bd9b745/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 57c7eae..c4735a0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -11,6 +11,9 @@ Release 2.6.1 - UNRELEASED
HADOOP-7139. Allow appending to existing SequenceFiles
(kanaka kumar avvaru via vinayakumarb)
+ HADOOP-12280. Skip unit tests based on maven profile rather than
+ NativeCodeLoader.isNativeCodeLoaded (Masatake Iwasaki via Colin P. McCabe)
+
OPTIMIZATIONS
HADOOP-11238. Update the NameNode's Group Cache in the background when
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bd9b745/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
index 08231f9..e13df57 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoCodec.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputBuffer;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.RandomDatum;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.NativeCodeLoader;
import org.apache.hadoop.util.ReflectionUtils;
import org.junit.Assert;
@@ -63,10 +64,7 @@ public class TestCryptoCodec {
@Test(timeout=120000)
public void testJceAesCtrCryptoCodec() throws Exception {
- if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
- LOG.warn("Skipping since test was not run with -Pnative flag");
- Assume.assumeTrue(false);
- }
+ GenericTestUtils.assumeInNativeProfile();
if (!NativeCodeLoader.buildSupportsOpenssl()) {
LOG.warn("Skipping test since openSSL library not loaded");
Assume.assumeTrue(false);
@@ -79,10 +77,7 @@ public class TestCryptoCodec {
@Test(timeout=120000)
public void testOpensslAesCtrCryptoCodec() throws Exception {
- if (!"true".equalsIgnoreCase(System.getProperty("runningWithNative"))) {
- LOG.warn("Skipping since test was not run with -Pnative flag");
- Assume.assumeTrue(false);
- }
+ GenericTestUtils.assumeInNativeProfile();
if (!NativeCodeLoader.buildSupportsOpenssl()) {
LOG.warn("Skipping test since openSSL library not loaded");
Assume.assumeTrue(false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bd9b745/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index f64e8dc..257ad5d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.BeforeClass;
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
@@ -25,6 +26,7 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
@BeforeClass
public static void init() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
Configuration conf = new Configuration();
codec = CryptoCodec.getInstance(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bd9b745/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
index 4576642..be4ab92 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestSequenceFileAppend.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.io.SequenceFile.Writer.Option;
import org.apache.hadoop.io.compress.DefaultCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.io.serializer.JavaSerializationComparator;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -140,6 +141,7 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendRecordCompression() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
Path file = new Path(ROOT_PATH, "testseqappendblockcompr.seq");
fs.delete(file, true);
@@ -173,6 +175,7 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendBlockCompression() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
Path file = new Path(ROOT_PATH, "testseqappendblockcompr.seq");
fs.delete(file, true);
@@ -247,6 +250,8 @@ public class TestSequenceFileAppend {
@Test(timeout = 30000)
public void testAppendSort() throws Exception {
+ GenericTestUtils.assumeInNativeProfile();
+
Path file = new Path(ROOT_PATH, "testseqappendSort.seq");
fs.delete(file, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3bd9b745/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
index 7be71e9..0616887 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/test/GenericTestUtils.java
@@ -41,6 +41,7 @@ import org.apache.log4j.Layout;
import org.apache.log4j.Logger;
import org.apache.log4j.WriterAppender;
import org.junit.Assert;
+import org.junit.Assume;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -368,4 +369,14 @@ public abstract class GenericTestUtils {
}
}
}
+
+ /**
+ * Skip test if native build profile of Maven is not activated.
+ * Sub-project using this must set 'runningWithNative' property to true
+ * in the definition of native profile in pom.xml.
+ */
+ public static void assumeInNativeProfile() {
+ Assume.assumeTrue(
+ Boolean.valueOf(System.getProperty("runningWithNative", "false")));
+ }
}
[11/50] [abbrv] hadoop git commit: YARN-2694. Ensure only single node
label specified in ResourceRequest. Contributed by Wangda Tan (cherry picked
from commit c1957fef29b07fea70938e971b30532a1e131fd0)
Posted by sj...@apache.org.
YARN-2694. Ensure only single node label specified in ResourceRequest. Contributed by Wangda Tan
(cherry picked from commit c1957fef29b07fea70938e971b30532a1e131fd0)
(cherry picked from commit 3ddafaa7c854dcf21ecc790c276927e7c869e62c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/637e7f9e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/637e7f9e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/637e7f9e
Branch: refs/heads/branch-2.6
Commit: 637e7f9e393db9a9e73f9683a398aec7d6aab909
Parents: 4c94f07
Author: Jian He <ji...@apache.org>
Authored: Fri Feb 6 11:34:12 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 21:07:51 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/api/records/ResourceRequest.java | 21 ++++---
.../hadoop/yarn/client/api/AMRMClient.java | 3 +-
.../yarn/client/api/impl/AMRMClientImpl.java | 37 ++++++++++++-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 23 +++++---
.../yarn/client/api/impl/TestAMRMClient.java | 58 ++++++++++++++++----
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 14 ++++-
.../nodelabels/CommonNodeLabelsManager.java | 45 +++++++++++++--
.../nodelabels/TestCommonNodeLabelsManager.java | 47 +++++++++++++---
.../ApplicationMasterService.java | 5 +-
.../scheduler/SchedulerUtils.java | 19 +++++++
.../yarn/server/resourcemanager/MockAM.java | 17 ++++--
.../nodelabels/TestRMNodeLabelsManager.java | 50 ++++++++---------
.../scheduler/TestSchedulerUtils.java | 52 +++++++++++++++---
.../capacity/TestContainerAllocation.java | 32 +++++------
.../webapp/TestRMWebServicesNodeLabels.java | 17 +++---
16 files changed, 329 insertions(+), 114 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 58c2139..2c654dc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -186,6 +186,9 @@ Release 2.6.1 - UNRELEASED
YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage
to track used-resources-by-label.(Wangda Tan via jianhe)
+ YARN-2694. Ensure only single node label specified in ResourceRequest.
+ (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
index 7f86cae..2f17ac9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceRequest.java
@@ -253,24 +253,27 @@ public abstract class ResourceRequest implements Comparable<ResourceRequest> {
/**
* Get node-label-expression for this Resource Request. If this is set, all
* containers allocated to satisfy this resource-request will be only on those
- * nodes that satisfy this node-label-expression
+ * nodes that satisfy this node-label-expression.
+ *
+ * Please note that node label expression now can only take effect when the
+ * resource request has resourceName = ANY
*
* @return node-label-expression
*/
@Public
@Evolving
- public abstract String getNodeLabelExpression();
+ public abstract String getNodeLabelExpression();
/**
- * Set node label expression of this resource request. Now only
- * support AND(&&), in the future will provide support for OR(||), NOT(!).
+ * Set node label expression of this resource request. Now only support
+ * specifying a single node label. In the future we will support more complex
+ * node label expression specification like AND(&&), OR(||), etc.
*
- * Examples:
- * - GPU && LARGE_MEM, ask for node has label GPU and LARGE_MEM together
- * - "" (empty) means ask for node doesn't have label on it, this is default
- * behavior
+ * Any please note that node label expression now can only take effect when
+ * the resource request has resourceName = ANY
*
- * @param nodelabelExpression node-label-expression of this ResourceRequest
+ * @param nodelabelExpression
+ * node-label-expression of this ResourceRequest
*/
@Public
@Evolving
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
index 6f8c65a..9923806 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/AMRMClient.java
@@ -169,7 +169,8 @@ public abstract class AMRMClient<T extends AMRMClient.ContainerRequest> extends
* If true, containers for this request may be assigned on hosts
* and racks other than the ones explicitly requested.
* @param nodeLabelsExpression
- * Set node labels to allocate resource
+ * Set node labels to allocate resource, now we only support
+ * asking for only a single node label
*/
public ContainerRequest(Resource capability, String[] nodes,
String[] racks, Priority priority, boolean relaxLocality,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
index 2fb9ebb..c823474 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/AMRMClientImpl.java
@@ -422,6 +422,8 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
checkLocalityRelaxationConflict(req.getPriority(), dedupedRacks, true);
checkLocalityRelaxationConflict(req.getPriority(), inferredRacks,
req.getRelaxLocality());
+ // check if the node label expression specified is valid
+ checkNodeLabelExpression(req);
if (req.getNodes() != null) {
HashSet<String> dedupedNodes = new HashSet<String>(req.getNodes());
@@ -587,6 +589,37 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
}
}
+ /**
+ * Valid if a node label expression specified on container request is valid or
+ * not
+ *
+ * @param containerRequest
+ */
+ private void checkNodeLabelExpression(T containerRequest) {
+ String exp = containerRequest.getNodeLabelExpression();
+
+ if (null == exp || exp.isEmpty()) {
+ return;
+ }
+
+ // Don't support specifying >= 2 node labels in a node label expression now
+ if (exp.contains("&&") || exp.contains("||")) {
+ throw new InvalidContainerRequestException(
+ "Cannot specify more than two node labels"
+ + " in a single node label expression");
+ }
+
+ // Don't allow specify node label against ANY request
+ if ((containerRequest.getRacks() != null &&
+ (!containerRequest.getRacks().isEmpty()))
+ ||
+ (containerRequest.getNodes() != null &&
+ (!containerRequest.getNodes().isEmpty()))) {
+ throw new InvalidContainerRequestException(
+ "Cannot specify node label with rack and node");
+ }
+ }
+
private void addResourceRequestToAsk(ResourceRequest remoteRequest) {
// This code looks weird but is needed because of the following scenario.
// A ResourceRequest is removed from the remoteRequestTable. A 0 container
@@ -641,7 +674,9 @@ public class AMRMClientImpl<T extends ContainerRequest> extends AMRMClient<T> {
resourceRequestInfo.containerRequests.add(req);
}
- resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
+ if (ResourceRequest.ANY.equals(resourceName)) {
+ resourceRequestInfo.remoteRequest.setNodeLabelExpression(labelExpression);
+ }
// Note this down for next interaction with ResourceManager
addResourceRequestToAsk(resourceRequestInfo.remoteRequest);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 89d87cf..285295f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RemoveFromClusterNodeLa
import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.base.Preconditions;
import com.google.common.collect.ImmutableMap;
@Private
@@ -99,8 +100,11 @@ public class RMAdminCLI extends HAAdmin {
new UsageInfo("[label1,label2,label3] (label splitted by \",\")",
"remove from cluster node labels"))
.put("-replaceLabelsOnNode",
- new UsageInfo("[node1:port,label1,label2 node2:port,label1,label2]",
- "replace labels on nodes"))
+ new UsageInfo(
+ "[node1[:port]=label1,label2 node2[:port]=label1,label2]",
+ "replace labels on nodes"
+ + " (please note that we do not support specifying multiple"
+ + " labels on a single host for now.)"))
.put("-directlyAccessNodeLabelStore",
new UsageInfo("", "Directly access node label store, "
+ "with this option, all node label related operations"
@@ -377,8 +381,7 @@ public class RMAdminCLI extends HAAdmin {
return 0;
}
- private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args)
- throws IOException {
+ private Map<NodeId, Set<String>> buildNodeLabelsFromStr(String args) {
Map<NodeId, Set<String>> map = new HashMap<NodeId, Set<String>>();
for (String nodeToLabels : args.split("[ \n]")) {
@@ -389,10 +392,9 @@ public class RMAdminCLI extends HAAdmin {
String[] splits = nodeToLabels.split(",");
String nodeIdStr = splits[0];
-
- if (nodeIdStr.trim().isEmpty()) {
- throw new IOException("node name cannot be empty");
- }
+
+ Preconditions.checkArgument(!nodeIdStr.trim().isEmpty(),
+ "node name cannot be empty");
NodeId nodeId = ConverterUtils.toNodeIdWithDefaultPort(nodeIdStr);
map.put(nodeId, new HashSet<String>());
@@ -402,6 +404,11 @@ public class RMAdminCLI extends HAAdmin {
map.get(nodeId).add(splits[i].trim().toLowerCase());
}
}
+
+ int nLabels = map.get(nodeId).size();
+ Preconditions.checkArgument(nLabels <= 1, "%d labels specified on host=%s"
+ + ", please note that we do not support specifying multiple"
+ + " labels on a single host for now.", nLabels, nodeIdStr);
}
return map;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 3c6918c..7d29d05 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.client.api.impl;
-import com.google.common.base.Supplier;
-
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -40,7 +38,6 @@ import java.util.List;
import java.util.Set;
import java.util.TreeSet;
-import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.Credentials;
@@ -75,6 +72,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.client.api.AMRMClient;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
+import org.apache.hadoop.yarn.client.api.InvalidContainerRequestException;
import org.apache.hadoop.yarn.client.api.NMTokenCache;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -89,6 +87,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
import org.junit.AfterClass;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -96,6 +95,8 @@ import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.mortbay.log.Log;
+import com.google.common.base.Supplier;
+
public class TestAMRMClient {
static Configuration conf = null;
static MiniYARNCluster yarnCluster = null;
@@ -148,7 +149,6 @@ public class TestAMRMClient {
racks = new String[]{ rack };
}
- @SuppressWarnings("deprecation")
@Before
public void startApp() throws Exception {
// submit new app
@@ -678,21 +678,57 @@ public class TestAMRMClient {
AMRMClientImpl<ContainerRequest> client =
new AMRMClientImpl<ContainerRequest>();
- // add x, y to ANY
+ // add exp=x to ANY
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
- 1), null, null, Priority.UNDEFINED, true, "x && y"));
+ 1), null, null, Priority.UNDEFINED, true, "x"));
Assert.assertEquals(1, client.ask.size());
- Assert.assertEquals("x && y", client.ask.iterator().next()
+ Assert.assertEquals("x", client.ask.iterator().next()
.getNodeLabelExpression());
- // add x, y and a, b to ANY, only a, b should be kept
+ // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
- 1), null, null, Priority.UNDEFINED, true, "x && y"));
+ 1), null, null, Priority.UNDEFINED, true, "x"));
client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
- 1), null, null, Priority.UNDEFINED, true, "a && b"));
+ 1), null, null, Priority.UNDEFINED, true, "a"));
Assert.assertEquals(1, client.ask.size());
- Assert.assertEquals("a && b", client.ask.iterator().next()
+ Assert.assertEquals("a", client.ask.iterator().next()
.getNodeLabelExpression());
+
+ // add exp=x to ANY, rack and node, only resource request has ANY resource
+ // name will be assigned the label expression
+ // add exp=x then add exp=a to ANY in same priority, only exp=a should kept
+ client.addContainerRequest(new ContainerRequest(Resource.newInstance(1024,
+ 1), null, null, Priority.UNDEFINED, true,
+ "y"));
+ Assert.assertEquals(1, client.ask.size());
+ for (ResourceRequest req : client.ask) {
+ if (ResourceRequest.ANY.equals(req.getResourceName())) {
+ Assert.assertEquals("y", req.getNodeLabelExpression());
+ } else {
+ Assert.assertNull(req.getNodeLabelExpression());
+ }
+ }
+ }
+
+ private void verifyAddRequestFailed(AMRMClient<ContainerRequest> client,
+ ContainerRequest request) {
+ try {
+ client.addContainerRequest(request);
+ } catch (InvalidContainerRequestException e) {
+ return;
+ }
+ Assert.fail();
+ }
+
+ @Test(timeout=30000)
+ public void testAskWithInvalidNodeLabels() {
+ AMRMClientImpl<ContainerRequest> client =
+ new AMRMClientImpl<ContainerRequest>();
+
+ // specified exp with more than one node labels
+ verifyAddRequestFailed(client,
+ new ContainerRequest(Resource.newInstance(1024, 1), null, null,
+ Priority.UNDEFINED, true, "x && y"));
}
private void testAllocation(final AMRMClientImpl<ContainerRequest> amClient)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index 6176a3e..c8b3a32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -470,7 +470,7 @@ public class TestRMAdminCLI {
// Successfully replace labels
dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
String[] args =
- { "-replaceLabelsOnNode", "node1,x,y node2,y",
+ { "-replaceLabelsOnNode", "node1,x node2,y",
"-directlyAccessNodeLabelStore" };
assertEquals(0, rmAdminCLI.run(args));
assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
@@ -494,7 +494,7 @@ public class TestRMAdminCLI {
// Successfully replace labels
dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
String[] args =
- { "-replaceLabelsOnNode", "node1:8000,x,y node2:8000,y",
+ { "-replaceLabelsOnNode", "node1:8000,x node2:8000,y",
"-directlyAccessNodeLabelStore" };
assertEquals(0, rmAdminCLI.run(args));
assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
@@ -502,6 +502,16 @@ public class TestRMAdminCLI {
assertTrue(dummyNodeLabelsManager.getNodeLabels().containsKey(
NodeId.newInstance("node2", 8000)));
}
+
+ @Test
+ public void testReplaceMultipleLabelsOnSingleNode() throws Exception {
+ // Successfully replace labels
+ dummyNodeLabelsManager.addToCluserNodeLabels(ImmutableSet.of("x", "y"));
+ String[] args =
+ { "-replaceLabelsOnNode", "node1,x,y",
+ "-directlyAccessNodeLabelStore" };
+ assertTrue(0 != rmAdminCLI.run(args));
+ }
private void testError(String[] args, String template,
ByteArrayOutputStream data, int resultCode) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index daefe8d..b30689b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -43,11 +43,12 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
-import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels;
import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEvent;
import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEventType;
import org.apache.hadoop.yarn.nodelabels.event.RemoveClusterNodeLabels;
+import org.apache.hadoop.yarn.nodelabels.event.StoreNewClusterNodeLabels;
import org.apache.hadoop.yarn.nodelabels.event.UpdateNodeToLabelsMappingsEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -287,14 +288,35 @@ public class CommonNodeLabelsManager extends AbstractService {
// check all labels being added existed
Set<String> knownLabels = labelCollections.keySet();
for (Entry<NodeId, Set<String>> entry : addedLabelsToNode.entrySet()) {
- if (!knownLabels.containsAll(entry.getValue())) {
+ NodeId nodeId = entry.getKey();
+ Set<String> labels = entry.getValue();
+
+ if (!knownLabels.containsAll(labels)) {
String msg =
"Not all labels being added contained by known "
+ "label collections, please check" + ", added labels=["
- + StringUtils.join(entry.getValue(), ",") + "]";
+ + StringUtils.join(labels, ",") + "]";
LOG.error(msg);
throw new IOException(msg);
}
+
+ // In YARN-2694, we temporarily disable user add more than 1 labels on a
+ // same host
+ if (!labels.isEmpty()) {
+ Set<String> newLabels = new HashSet<String>(getLabelsByNode(nodeId));
+ newLabels.addAll(labels);
+ // we don't allow number of labels on a node > 1 after added labels
+ if (newLabels.size() > 1) {
+ String msg =
+ String.format(
+ "%d labels specified on host=%s after add labels to node"
+ + ", please note that we do not support specifying multiple"
+ + " labels on a single host for now.",
+ newLabels.size(), nodeId.getHost());
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+ }
}
}
@@ -537,11 +559,24 @@ public class CommonNodeLabelsManager extends AbstractService {
// check all labels being added existed
Set<String> knownLabels = labelCollections.keySet();
for (Entry<NodeId, Set<String>> entry : replaceLabelsToNode.entrySet()) {
- if (!knownLabels.containsAll(entry.getValue())) {
+ NodeId nodeId = entry.getKey();
+ Set<String> labels = entry.getValue();
+
+ // As in YARN-2694, we disable user add more than 1 labels on a same host
+ if (labels.size() > 1) {
+ String msg = String.format("%d labels specified on host=%s"
+ + ", please note that we do not support specifying multiple"
+ + " labels on a single host for now.", labels.size(),
+ nodeId.getHost());
+ LOG.error(msg);
+ throw new IOException(msg);
+ }
+
+ if (!knownLabels.containsAll(labels)) {
String msg =
"Not all labels being replaced contained by known "
+ "label collections, please check" + ", new labels=["
- + StringUtils.join(entry.getValue(), ",") + "]";
+ + StringUtils.join(labels, ",") + "]";
LOG.error(msg);
throw new IOException(msg);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
index a56a595..2fb1676 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestCommonNodeLabelsManager.java
@@ -221,19 +221,17 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
ImmutableMap.of(toNodeId("n1"), CommonNodeLabelsManager.EMPTY_STRING_SET));
// add label on node
- mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
- toNodeId("n2"), toSet("p2")));
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
assertMapEquals(
mgr.getNodeLabels(),
ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"),
- toSet("p2", "p3"), toNodeId("n3"), toSet("p3")));
+ toSet("p3"), toNodeId("n3"), toSet("p3")));
assertMapEquals(mgr.lastNodeToLabels,
- ImmutableMap.of(toNodeId("n1"), toSet("p1"), toNodeId("n2"),
- toSet("p2", "p3")));
+ ImmutableMap.of(toNodeId("n1"), toSet("p1")));
// remove labels on node
mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
- toNodeId("n2"), toSet("p2", "p3"), toNodeId("n3"), toSet("p3")));
+ toNodeId("n2"), toSet("p3"), toNodeId("n3"), toSet("p3")));
Assert.assertEquals(0, mgr.getNodeLabels().size());
assertMapEquals(mgr.lastNodeToLabels, ImmutableMap.of(toNodeId("n1"),
CommonNodeLabelsManager.EMPTY_STRING_SET, toNodeId("n2"),
@@ -270,10 +268,10 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
@Test(timeout = 5000)
public void testTrimLabelsWhenModifyLabelsOnNodes() throws IOException {
mgr.addToCluserNodeLabels(toSet(" p1", "p2"));
- mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ", "p2")));
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1 ")));
assertMapEquals(
mgr.getNodeLabels(),
- ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+ ImmutableMap.of(toNodeId("n1"), toSet("p1")));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2")));
assertMapEquals(
mgr.getNodeLabels(),
@@ -281,4 +279,37 @@ public class TestCommonNodeLabelsManager extends NodeLabelTestBase {
mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("n1"), toSet(" p2 ")));
Assert.assertTrue(mgr.getNodeLabels().isEmpty());
}
+
+ @Test(timeout = 5000)
+ public void testNoMoreThanOneLabelExistedInOneHost() throws IOException {
+ boolean failed = false;
+ // As in YARN-2694, we temporarily disable no more than one label existed in
+ // one host
+ mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+ try {
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+ } catch (IOException e) {
+ failed = true;
+ }
+ Assert.assertTrue("Should failed when set > 1 labels on a host", failed);
+
+ try {
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1", "p2")));
+ } catch (IOException e) {
+ failed = true;
+ }
+ Assert.assertTrue("Should failed when add > 1 labels on a host", failed);
+
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+ // add a same label to a node, #labels in this node is still 1, shouldn't
+ // fail
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+ try {
+ mgr.addLabelsToNode(ImmutableMap.of(toNodeId("n1"), toSet("p2")));
+ } catch (IOException e) {
+ failed = true;
+ }
+ Assert.assertTrue("Should failed when #labels > 1 on a host after add",
+ failed);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index 2a4166c..c896bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -488,10 +488,11 @@ public class ApplicationMasterService extends AbstractService implements
RMApp app =
this.rmContext.getRMApps().get(applicationId);
- // set label expression for Resource Requests
+ // set label expression for Resource Requests if resourceName=ANY
ApplicationSubmissionContext asc = app.getApplicationSubmissionContext();
for (ResourceRequest req : ask) {
- if (null == req.getNodeLabelExpression()) {
+ if (null == req.getNodeLabelExpression()
+ && ResourceRequest.ANY.equals(req.getResourceName())) {
req.setNodeLabelExpression(asc.getNodeLabelExpression());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c41aae0..bdc9da1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -262,6 +262,25 @@ public class SchedulerUtils {
}
String labelExp = resReq.getNodeLabelExpression();
+ // we don't allow specify label expression other than resourceName=ANY now
+ if (!ResourceRequest.ANY.equals(resReq.getResourceName())
+ && labelExp != null && !labelExp.trim().isEmpty()) {
+ throw new InvalidResourceRequestException(
+ "Invailid resource request, queue=" + queueInfo.getQueueName()
+ + " specified node label expression in a "
+ + "resource request has resource name = "
+ + resReq.getResourceName());
+ }
+
+ // we don't allow specify label expression with more than one node labels now
+ if (labelExp != null && labelExp.contains("&&")) {
+ throw new InvalidResourceRequestException(
+ "Invailid resource request, queue=" + queueInfo.getQueueName()
+ + " specified more than one node label "
+ + "in a node label expression, node label expression = "
+ + labelExp);
+ }
+
if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
labelExp)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
index e2e3cc1..a3968f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockAM.java
@@ -165,12 +165,17 @@ public class MockAM {
int containers, String labelExpression) throws Exception {
List<ResourceRequest> reqs = new ArrayList<ResourceRequest>();
for (String host : hosts) {
- ResourceRequest hostReq = createResourceReq(host, memory, priority,
- containers, labelExpression);
- reqs.add(hostReq);
- ResourceRequest rackReq = createResourceReq("/default-rack", memory,
- priority, containers, labelExpression);
- reqs.add(rackReq);
+ // only add host/rack request when asked host isn't ANY
+ if (!host.equals(ResourceRequest.ANY)) {
+ ResourceRequest hostReq =
+ createResourceReq(host, memory, priority, containers,
+ labelExpression);
+ reqs.add(hostReq);
+ ResourceRequest rackReq =
+ createResourceReq("/default-rack", memory, priority, containers,
+ labelExpression);
+ reqs.add(rackReq);
+ }
}
ResourceRequest offRackReq = createResourceReq(ResourceRequest.ANY, memory,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index ed675f3..7c3d9ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -206,16 +206,14 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
/*
* Node->Labels:
- * host1 : red, blue
- * host2 : blue, yellow
+ * host1 : red
+ * host2 : blue
* host3 : yellow
* host4 :
*/
mgr.addToCluserNodeLabels(toSet("red", "blue", "yellow"));
- mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"),
- toSet("red", "blue")));
- mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"),
- toSet("blue", "yellow")));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host1"), toSet("red")));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host2"), toSet("blue")));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("host3"), toSet("yellow")));
// active two NM to n1, one large and one small
@@ -243,31 +241,29 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
- mgr.getQueueResource("Q2", q2Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ mgr.getQueueResource("Q2", q2Label, clusterResource));
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q4", q4Label, clusterResource));
Assert.assertEquals(clusterResource,
mgr.getQueueResource("Q5", q5Label, clusterResource));
- mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host1"), toSet("red"),
- toNodeId("host2"), toSet("blue", "yellow")));
- mgr.addLabelsToNode(ImmutableMap.of(toNodeId("host3"), toSet("red")));
+ mgr.removeLabelsFromNode(ImmutableMap.of(toNodeId("host2"), toSet("blue")));
/*
* Check resource after changes some labels
* Node->Labels:
- * host1 : blue (was: red, blue)
- * host2 : (was: blue, yellow)
- * host3 : red, yellow (was: yellow)
+ * host1 : red
+ * host2 : (was: blue)
+ * host3 : yellow
* host4 :
*/
// check resource
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 4),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q2", q2Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q3", q3Label, clusterResource));
@@ -279,9 +275,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
/*
* Check resource after deactive/active some nodes
* Node->Labels:
- * (deactived) host1 : blue
+ * (deactived) host1 : red
* host2 :
- * (deactived and then actived) host3 : red, yellow
+ * (deactived and then actived) host3 : yellow
* host4 :
*/
mgr.deactivateNode(NodeId.newInstance("host1", 1));
@@ -289,7 +285,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
mgr.activateNode(NodeId.newInstance("host3", 1), SMALL_RESOURCE);
// check resource
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q1", q1Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q2", q2Label, clusterResource));
@@ -326,9 +322,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q2", q2Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 2),
mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -339,7 +335,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
* Active NMs in nodes already have NM
* Node->Labels:
* host2 :
- * host3 : red, yellow (3 NMs)
+ * host3 : yellow (3 NMs)
* host4 : (2 NMs)
*/
mgr.activateNode(NodeId.newInstance("host3", 2), SMALL_RESOURCE);
@@ -349,9 +345,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 6),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q2", q2Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 6),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
mgr.getQueueResource("Q4", q4Label, clusterResource));
@@ -362,7 +358,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
* Deactive NMs in nodes already have NMs
* Node->Labels:
* host2 :
- * host3 : red, yellow (2 NMs)
+ * host3 : yellow (2 NMs)
* host4 : (0 NMs)
*/
mgr.deactivateNode(NodeId.newInstance("host3", 3));
@@ -372,9 +368,9 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
// check resource
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q1", q1Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q2", q2Label, clusterResource));
- Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 3),
+ Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q3", q3Label, clusterResource));
Assert.assertEquals(Resources.multiply(SMALL_RESOURCE, 1),
mgr.getQueueResource("Q4", q4Label, clusterResource));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index a7c7ad6..8abaeb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -214,11 +214,7 @@ public class TestSchedulerUtils {
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
-
- resReq.setNodeLabelExpression("x && y");
- SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
-
+
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
@@ -253,6 +249,8 @@ public class TestSchedulerUtils {
} catch (InvalidResourceRequestException e) {
}
+ // we don't allow specify more than two node labels in a single expression
+ // now
try {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
@@ -263,7 +261,7 @@ public class TestSchedulerUtils {
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
- resReq.setNodeLabelExpression("x && y && z");
+ resReq.setNodeLabelExpression("x && y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
fail("Should fail");
@@ -328,7 +326,7 @@ public class TestSchedulerUtils {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
- resReq.setNodeLabelExpression("x && y && z");
+ resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
scheduler);
@@ -337,7 +335,45 @@ public class TestSchedulerUtils {
scheduler);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
- fail("Should be valid when request labels is empty");
+ fail("Should be valid when queue can access any labels");
+ }
+
+ // we don't allow resource name other than ANY and specify label
+ try {
+ // set queue accessible node labesl to [x, y]
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), "rack", resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
+ }
+
+ // we don't allow resource name other than ANY and specify label even if
+ // queue has accessible label = *
+ try {
+ // set queue accessible node labesl to *
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays
+ .asList(CommonNodeLabelsManager.ANY));
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), "rack", resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
index ad834ac..a9f9420 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerAllocation.java
@@ -87,6 +87,7 @@ public class TestContainerAllocation {
@Test(timeout = 3000000)
public void testExcessReservationThanNodeManagerCapacity() throws Exception {
+ @SuppressWarnings("resource")
MockRM rm = new MockRM(conf);
rm.start();
@@ -393,6 +394,7 @@ public class TestContainerAllocation {
}
}
+ @SuppressWarnings("unchecked")
private <E> Set<E> toSet(E... elements) {
Set<E> set = Sets.newHashSet(elements);
return set;
@@ -449,7 +451,7 @@ public class TestContainerAllocation {
return conf;
}
- @Test(timeout = 300000)
+ @Test (timeout = 300000)
public void testContainerAllocationWithSingleUserLimits() throws Exception {
final RMNodeLabelsManager mgr = new MemoryRMNodeLabelsManager();
mgr.init(conf);
@@ -470,7 +472,7 @@ public class TestContainerAllocation {
rm1.getRMContext().setNodeLabelManager(mgr);
rm1.start();
MockNM nm1 = rm1.registerNode("h1:1234", 8000); // label = x
- MockNM nm2 = rm1.registerNode("h2:1234", 8000); // label = y
+ rm1.registerNode("h2:1234", 8000); // label = y
MockNM nm3 = rm1.registerNode("h3:1234", 8000); // label = <empty>
// launch an app to queue a1 (label = x), and check all container will
@@ -518,9 +520,9 @@ public class TestContainerAllocation {
*
* Node structure:
* h1 : x
- * h2 : x, y
+ * h2 : y
* h3 : y
- * h4 : y, z
+ * h4 : z
* h5 : NO
*
* Total resource:
@@ -540,9 +542,9 @@ public class TestContainerAllocation {
// set node -> label
mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));
mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0),
- toSet("x"), NodeId.newInstance("h2", 0), toSet("x", "y"),
+ toSet("x"), NodeId.newInstance("h2", 0), toSet("y"),
NodeId.newInstance("h3", 0), toSet("y"), NodeId.newInstance("h4", 0),
- toSet("y", "z"), NodeId.newInstance("h5", 0),
+ toSet("z"), NodeId.newInstance("h5", 0),
RMNodeLabelsManager.EMPTY_STRING_SET));
// inject node label manager
@@ -568,12 +570,10 @@ public class TestContainerAllocation {
RMApp app1 = rm1.submitApp(1024, "app", "user", null, "a1");
MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
- // request a container (label = x && y). can only allocate on nm2
- am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "x && y");
+ // request a container (label = y). can be allocated on nm2
+ am1.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "y");
containerId =
- ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
- Assert.assertFalse(rm1.waitForState(nm1, containerId,
- RMContainerState.ALLOCATED, 10 * 1000));
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L);
Assert.assertTrue(rm1.waitForState(nm2, containerId,
RMContainerState.ALLOCATED, 10 * 1000));
checkTaskContainersHost(am1.getApplicationAttemptId(), containerId, rm1,
@@ -609,12 +609,10 @@ public class TestContainerAllocation {
checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1,
"h3");
- // try to allocate container (request label = y && z) on nm3 (label = y) and
- // nm4 (label = y,z). Will sucessfully allocate on nm4 only.
- am3.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "y && z");
- containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3);
- Assert.assertFalse(rm1.waitForState(nm3, containerId,
- RMContainerState.ALLOCATED, 10 * 1000));
+ // try to allocate container (request label = z) on nm4 (label = y,z).
+ // Will successfully allocate on nm4 only.
+ am3.allocate("*", 1024, 1, new ArrayList<ContainerId>(), "z");
+ containerId = ContainerId.newContainerId(am3.getApplicationAttemptId(), 3L);
Assert.assertTrue(rm1.waitForState(nm4, containerId,
RMContainerState.ALLOCATED, 10 * 1000));
checkTaskContainersHost(am3.getApplicationAttemptId(), containerId, rm1,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/637e7f9e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
index df5aecb..5a0bdd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodeLabels.java
@@ -155,7 +155,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.path("replace-labels")
.queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+ .entity("{\"nodeLabels\": [\"a\"]}",
MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
LOG.info("posted node nodelabel");
@@ -168,8 +168,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
- jarr = json.getJSONArray("nodeLabels");
- assertEquals(2, jarr.length());
+ assertEquals("a", json.getString("nodeLabels"));
+
// Replace
response =
@@ -178,9 +178,10 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.path("replace-labels")
.queryParam("user.name", userName)
.accept(MediaType.APPLICATION_JSON)
- .entity("{\"nodeLabels\":\"a\"}", MediaType.APPLICATION_JSON)
+ .entity("{\"nodeLabels\":\"b\"}", MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
LOG.info("posted node nodelabel");
+
// Verify
response =
r.path("ws").path("v1").path("cluster")
@@ -189,13 +190,12 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.accept(MediaType.APPLICATION_JSON).get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
json = response.getEntity(JSONObject.class);
- assertEquals("a", json.getString("nodeLabels"));
+ assertEquals("b", json.getString("nodeLabels"));
// Replace labels using node-to-labels
NodeToLabelsInfo ntli = new NodeToLabelsInfo();
NodeLabelsInfo nli = new NodeLabelsInfo();
nli.getNodeLabels().add("a");
- nli.getNodeLabels().add("b");
ntli.getNodeToLabels().put("nid:0", nli);
response =
r.path("ws").path("v1").path("cluster")
@@ -214,9 +214,8 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
ntli = response.getEntity(NodeToLabelsInfo.class);
nli = ntli.getNodeToLabels().get("nid:0");
- assertEquals(2, nli.getNodeLabels().size());
+ assertEquals(1, nli.getNodeLabels().size());
assertTrue(nli.getNodeLabels().contains("a"));
- assertTrue(nli.getNodeLabels().contains("b"));
// Remove all
response =
@@ -267,7 +266,7 @@ public class TestRMWebServicesNodeLabels extends JerseyTest {
.path("replace-labels")
.queryParam("user.name", notUserName)
.accept(MediaType.APPLICATION_JSON)
- .entity("{\"nodeLabels\": [\"a\", \"b\"]}",
+ .entity("{\"nodeLabels\": [\"b\"]}",
MediaType.APPLICATION_JSON)
.post(ClientResponse.class);
// Verify
[24/50] [abbrv] hadoop git commit: YARN-3999. RM hangs on draing
events. Contributed by Jian He
Posted by sj...@apache.org.
YARN-3999. RM hangs on draing events. Contributed by Jian He
(cherry picked from commit 3ae716fa696b87e849dae40225dc59fb5ed114cb)
(cherry picked from commit 2ebdf5bfcee9ede80681a5266df225885d830883)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d59bf81e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d59bf81e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d59bf81e
Branch: refs/heads/branch-2.6
Commit: d59bf81e0866b1922090aeb0690a71c036e9a5c2
Parents: 6ed2486
Author: Xuan <xg...@apache.org>
Authored: Tue Aug 11 18:25:11 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 22:57:35 2015 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/util/JvmPauseMonitor.java | 12 ++++----
hadoop-yarn-project/CHANGES.txt | 2 ++
.../hadoop/yarn/conf/YarnConfiguration.java | 5 ++++
.../hadoop/yarn/event/AsyncDispatcher.java | 9 +++++-
.../src/main/resources/yarn-default.xml | 9 ++++++
.../resourcemanager/RMActiveServiceContext.java | 30 -------------------
.../server/resourcemanager/RMContextImpl.java | 23 +++++++--------
.../server/resourcemanager/ResourceManager.java | 31 ++++++++++----------
.../server/resourcemanager/TestAppManager.java | 5 ++--
.../resourcemanager/TestResourceManager.java | 1 +
.../rmapp/TestRMAppTransitions.java | 6 ++--
.../attempt/TestRMAppAttemptTransitions.java | 8 ++---
.../scheduler/capacity/TestUtils.java | 7 +++--
.../scheduler/fifo/TestFifoScheduler.java | 17 +++++------
14 files changed, 80 insertions(+), 85 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
index e8af45e..573b511 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/JvmPauseMonitor.java
@@ -83,11 +83,13 @@ public class JvmPauseMonitor {
public void stop() {
shouldRun = false;
- monitorThread.interrupt();
- try {
- monitorThread.join();
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
+ if (monitorThread != null) {
+ monitorThread.interrupt();
+ try {
+ monitorThread.join();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9fae242..ac114fa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -200,6 +200,8 @@ Release 2.6.1 - UNRELEASED
YARN-2918. RM should not fail on startup if queue's configured labels do
not exist in cluster-node-labels. (Wangda Tan via jianhe)
+ YARN-3999. RM hangs on draing events. (Jian He via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 39e12d4..84322c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1104,6 +1104,11 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_RESOURCEMANAGER_CONNECT_RETRY_INTERVAL_MS
= 30 * 1000;
+ public static final String DISPATCHER_DRAIN_EVENTS_TIMEOUT =
+ YARN_PREFIX + "dispatcher.drain-events.timeout";
+
+ public static final long DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT = 300000;
+
/**
* CLASSPATH for YARN applications. A comma-separated list of CLASSPATH
* entries
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 28be6ac..0a869fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.util.ShutdownHookManager;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
/**
@@ -136,8 +137,14 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
if (drainEventsOnStop) {
blockNewEvents = true;
LOG.info("AsyncDispatcher is draining to stop, igonring any new events.");
+ long endTime = System.currentTimeMillis() + getConfig()
+ .getLong(YarnConfiguration.DISPATCHER_DRAIN_EVENTS_TIMEOUT,
+ YarnConfiguration.DEFAULT_DISPATCHER_DRAIN_EVENTS_TIMEOUT);
+
synchronized (waitForDrained) {
- while (!drained && eventHandlingThread.isAlive()) {
+ while (!drained && eventHandlingThread != null
+ && eventHandlingThread.isAlive()
+ && System.currentTimeMillis() < endTime) {
waitForDrained.wait(1000);
LOG.info("Waiting for AsyncDispatcher to drain.");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e168600..2ffa8b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -77,6 +77,15 @@
</property>
<property>
+ <description>Timeout in milliseconds when YARN dispatcher tries to drain the
+ events. Typically, this happens when service is stopping. e.g. RM drains
+ the ATS events dispatcher when stopping.
+ </description>
+ <name>yarn.dispatcher.drain-events.timeout</name>
+ <value>300000</value>
+ </property>
+
+ <property>
<description>The expiry interval for application master reporting.</description>
<name>yarn.am.liveness-monitor.expiry-interval-ms</name>
<value>600000</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 03fc40e..049e71e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -92,8 +92,6 @@ public class RMActiveServiceContext {
private NodesListManager nodesListManager;
private ResourceTrackerService resourceTrackerService;
private ApplicationMasterService applicationMasterService;
- private RMApplicationHistoryWriter rmApplicationHistoryWriter;
- private SystemMetricsPublisher systemMetricsPublisher;
private RMNodeLabelsManager nodeLabelManager;
private long epoch;
private Clock systemClock = new SystemClock();
@@ -117,7 +115,6 @@ public class RMActiveServiceContext {
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter,
ResourceScheduler scheduler) {
this();
this.setContainerAllocationExpirer(containerAllocationExpirer);
@@ -128,7 +125,6 @@ public class RMActiveServiceContext {
this.setContainerTokenSecretManager(containerTokenSecretManager);
this.setNMTokenSecretManager(nmTokenSecretManager);
this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
- this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
this.setScheduler(scheduler);
RMStateStore nullStore = new NullRMStateStore();
@@ -370,32 +366,6 @@ public class RMActiveServiceContext {
@Private
@Unstable
- public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
- return rmApplicationHistoryWriter;
- }
-
- @Private
- @Unstable
- public void setSystemMetricsPublisher(
- SystemMetricsPublisher systemMetricsPublisher) {
- this.systemMetricsPublisher = systemMetricsPublisher;
- }
-
- @Private
- @Unstable
- public SystemMetricsPublisher getSystemMetricsPublisher() {
- return systemMetricsPublisher;
- }
-
- @Private
- @Unstable
- public void setRMApplicationHistoryWriter(
- RMApplicationHistoryWriter rmApplicationHistoryWriter) {
- this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
- }
-
- @Private
- @Unstable
public long getEpoch() {
return this.epoch;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 1d0d6c0..dc66726 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -68,6 +68,9 @@ public class RMContextImpl implements RMContext {
private Configuration yarnConfiguration;
+ private RMApplicationHistoryWriter rmApplicationHistoryWriter;
+ private SystemMetricsPublisher systemMetricsPublisher;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -87,7 +90,6 @@ public class RMContextImpl implements RMContext {
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter,
ResourceScheduler scheduler) {
this();
this.setDispatcher(rmDispatcher);
@@ -95,7 +97,7 @@ public class RMContextImpl implements RMContext {
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
delegationTokenRenewer, appTokenSecretManager,
containerTokenSecretManager, nmTokenSecretManager,
- clientToAMTokenSecretManager, rmApplicationHistoryWriter,
+ clientToAMTokenSecretManager,
scheduler));
ConfigurationProvider provider = new LocalConfigurationProvider();
@@ -112,8 +114,7 @@ public class RMContextImpl implements RMContext {
AMRMTokenSecretManager appTokenSecretManager,
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
- ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager) {
this(
rmDispatcher,
containerAllocationExpirer,
@@ -123,9 +124,7 @@ public class RMContextImpl implements RMContext {
appTokenSecretManager,
containerTokenSecretManager,
nmTokenSecretManager,
- clientToAMTokenSecretManager,
- rmApplicationHistoryWriter,
- null);
+ clientToAMTokenSecretManager, null);
}
@Override
@@ -350,25 +349,25 @@ public class RMContextImpl implements RMContext {
@Override
public RMApplicationHistoryWriter getRMApplicationHistoryWriter() {
- return activeServiceContext.getRMApplicationHistoryWriter();
+ return this.rmApplicationHistoryWriter;
}
@Override
public void setSystemMetricsPublisher(
SystemMetricsPublisher systemMetricsPublisher) {
- activeServiceContext.setSystemMetricsPublisher(systemMetricsPublisher);
+ this.systemMetricsPublisher = systemMetricsPublisher;
}
@Override
public SystemMetricsPublisher getSystemMetricsPublisher() {
- return activeServiceContext.getSystemMetricsPublisher();
+ return this.systemMetricsPublisher;
}
@Override
public void setRMApplicationHistoryWriter(
RMApplicationHistoryWriter rmApplicationHistoryWriter) {
- activeServiceContext
- .setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+ this.rmApplicationHistoryWriter = rmApplicationHistoryWriter;
+
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 711acf2..34d7e47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -251,7 +251,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
adminService = createAdminService();
addService(adminService);
rmContext.setRMAdminService(adminService);
-
+
rmContext.setYarnConfiguration(conf);
createAndInitActiveServices();
@@ -260,6 +260,15 @@ public class ResourceManager extends CompositeService implements Recoverable {
YarnConfiguration.RM_BIND_HOST,
WebAppUtils.getRMWebAppURLWithoutScheme(this.conf));
+ RMApplicationHistoryWriter rmApplicationHistoryWriter =
+ createRMApplicationHistoryWriter();
+ addService(rmApplicationHistoryWriter);
+ rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+
+ SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher();
+ addService(systemMetricsPublisher);
+ rmContext.setSystemMetricsPublisher(systemMetricsPublisher);
+
super.serviceInit(this.conf);
}
@@ -415,7 +424,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setActiveServiceContext(activeServiceContext);
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
-
rmSecretManagerService = createRMSecretManagerService();
addService(rmSecretManagerService);
@@ -472,15 +480,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setDelegationTokenRenewer(delegationTokenRenewer);
}
- RMApplicationHistoryWriter rmApplicationHistoryWriter =
- createRMApplicationHistoryWriter();
- addService(rmApplicationHistoryWriter);
- rmContext.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
-
- SystemMetricsPublisher systemMetricsPublisher = createSystemMetricsPublisher();
- addService(systemMetricsPublisher);
- rmContext.setSystemMetricsPublisher(systemMetricsPublisher);
-
// Register event handler for NodesListManager
nodesListManager = new NodesListManager(rmContext);
rmDispatcher.register(NodesListManagerEventType.class, nodesListManager);
@@ -596,8 +595,9 @@ public class ResourceManager extends CompositeService implements Recoverable {
@Override
protected void serviceStop() throws Exception {
- DefaultMetricsSystem.shutdown();
+ super.serviceStop();
+ DefaultMetricsSystem.shutdown();
if (rmContext != null) {
RMStateStore store = rmContext.getStateStore();
try {
@@ -607,7 +607,6 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
}
- super.serviceStop();
}
protected void createPolicyMonitors() {
@@ -1068,12 +1067,12 @@ public class ResourceManager extends CompositeService implements Recoverable {
}
LOG.info("Transitioning to standby state");
- if (rmContext.getHAServiceState() ==
- HAServiceProtocol.HAServiceState.ACTIVE) {
+ HAServiceState state = rmContext.getHAServiceState();
+ rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
+ if (state == HAServiceProtocol.HAServiceState.ACTIVE) {
stopActiveServices();
reinitialize(initialize);
}
- rmContext.setHAServiceState(HAServiceProtocol.HAServiceState.STANDBY);
LOG.info("Transitioned to standby state");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
index e146611..fe392b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
@@ -115,7 +115,7 @@ public class TestAppManager{
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext context = new RMContextImpl(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
- null, null, null, null, null, writer) {
+ null, null, null, null, null) {
@Override
public ConcurrentMap<ApplicationId, RMApp> getRMApps() {
return map;
@@ -123,7 +123,8 @@ public class TestAppManager{
};
((RMContextImpl)context).setStateStore(mock(RMStateStore.class));
metricsPublisher = mock(SystemMetricsPublisher.class);
- ((RMContextImpl)context).setSystemMetricsPublisher(metricsPublisher);
+ context.setSystemMetricsPublisher(metricsPublisher);
+ context.setRMApplicationHistoryWriter(writer);
return context;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
index 6735575..1ab195a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceManager.java
@@ -66,6 +66,7 @@ public class TestResourceManager {
@After
public void tearDown() throws Exception {
+ resourceManager.stop();
}
private org.apache.hadoop.yarn.server.resourcemanager.NodeManager
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 0a2f0d4..658f8a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -212,11 +212,11 @@ public class TestRMAppTransitions {
renewer, new AMRMTokenSecretManager(conf, this.rmContext),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(),
- writer);
+ new ClientToAMTokenSecretManagerInRM());
((RMContextImpl)realRMContext).setStateStore(store);
publisher = mock(SystemMetricsPublisher.class);
- ((RMContextImpl)realRMContext).setSystemMetricsPublisher(publisher);
+ realRMContext.setSystemMetricsPublisher(publisher);
+ realRMContext.setRMApplicationHistoryWriter(writer);
this.rmContext = spy(realRMContext);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 8f14660..8018f41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -258,14 +258,14 @@ public class TestRMAppAttemptTransitions {
null, amRMTokenManager,
new RMContainerTokenSecretManager(conf),
nmTokenManager,
- clientToAMTokenManager,
- writer);
+ clientToAMTokenManager);
store = mock(RMStateStore.class);
((RMContextImpl) rmContext).setStateStore(store);
publisher = mock(SystemMetricsPublisher.class);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(publisher);
-
+ rmContext.setSystemMetricsPublisher(publisher);
+ rmContext.setRMApplicationHistoryWriter(writer);
+
scheduler = mock(YarnScheduler.class);
masterService = mock(ApplicationMasterService.class);
applicationMasterLauncher = mock(ApplicationMasterLauncher.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
index 9e352a7..b38a497 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
@@ -98,7 +98,7 @@ public class TestUtils {
new AMRMTokenSecretManager(conf, null),
new RMContainerTokenSecretManager(conf),
new NMTokenSecretManagerInRM(conf),
- new ClientToAMTokenSecretManagerInRM(), writer);
+ new ClientToAMTokenSecretManagerInRM());
RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
when(
nlm.getQueueResource(any(String.class), any(Set.class),
@@ -112,8 +112,8 @@ public class TestUtils {
when(nlm.getResourceByLabel(any(String.class), any(Resource.class)))
.thenAnswer(new Answer<Resource>() {
- @Override
- public Resource answer(InvocationOnMock invocation) throws Throwable {
+ @Override public Resource answer(InvocationOnMock invocation)
+ throws Throwable {
Object[] args = invocation.getArguments();
return (Resource) args[1];
}
@@ -121,6 +121,7 @@ public class TestUtils {
rmContext.setNodeLabelManager(nlm);
rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
+ rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
return rmContext;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d59bf81e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 7f9a817..991f3ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -147,7 +147,7 @@ public class TestFifoScheduler {
FifoScheduler scheduler = new FifoScheduler();
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext rmContext = new RMContextImpl(dispatcher, null,
- null, null, null, null, null, null, null, writer, scheduler);
+ null, null, null, null, null, null, null, scheduler);
((RMContextImpl) rmContext).setSystemMetricsPublisher(
mock(SystemMetricsPublisher.class));
@@ -193,10 +193,10 @@ public class TestFifoScheduler {
FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
- scheduler);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(
- mock(SystemMetricsPublisher.class));
+ null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
+ rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
+ rmContext.setRMApplicationHistoryWriter(
+ mock(RMApplicationHistoryWriter.class));
((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
scheduler.setRMContext(rmContext);
@@ -272,10 +272,9 @@ public class TestFifoScheduler {
}
};
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
- scheduler);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(
- mock(SystemMetricsPublisher.class));
+ null, containerTokenSecretManager, nmTokenSecretManager, null, scheduler);
+ rmContext.setSystemMetricsPublisher(mock(SystemMetricsPublisher.class));
+ rmContext.setRMApplicationHistoryWriter(mock(RMApplicationHistoryWriter.class));
((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
scheduler.setRMContext(rmContext);
[26/50] [abbrv] hadoop git commit: HDFS-7552. Change FsVolumeList
toString() to fix TestDataNodeVolumeFailureToleration (Liang Xie via Colin P.
McCabe) (cherry picked from commit a4876c130f1627e59ef055e586640d1933fc49af)
Posted by sj...@apache.org.
HDFS-7552. Change FsVolumeList toString() to fix TestDataNodeVolumeFailureToleration (Liang Xie via Colin P. McCabe)
(cherry picked from commit a4876c130f1627e59ef055e586640d1933fc49af)
(cherry picked from commit 22f72c79462c424e47610470fd48e871887ac326)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/adc34053
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/adc34053
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/adc34053
Branch: refs/heads/branch-2.6
Commit: adc340532164c9f39962de0eeab837a8cd9f3460
Parents: 0adcfe6
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Fri Dec 19 11:13:04 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 9 14:59:31 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc34053/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2234da3..2eeb79f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -175,6 +175,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8863. The remaining space check in BlockPlacementPolicyDefault is
flawed. (Kihwal Lee via yliu)
+ HDFS-7552. Change FsVolumeList toString() to fix
+ TestDataNodeVolumeFailureToleration (Liang Xie via Colin P. McCabe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/adc34053/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index b17b90b..367824a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -216,7 +216,7 @@ class FsVolumeList {
@Override
public String toString() {
- return volumes.toString();
+ return Arrays.toString(volumes.get());
}
/**
[41/50] [abbrv] hadoop git commit: CHANGES.txt: Dropping YARN-3379 as
only the refactor part of it exists in branch-2.6.1.
Posted by sj...@apache.org.
CHANGES.txt: Dropping YARN-3379 as only the refactor part of it exists in branch-2.6.1.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4697d94c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4697d94c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4697d94c
Branch: refs/heads/branch-2.6
Commit: 4697d94c6ae706721409551bd019945a1d9e08f9
Parents: b4d876d
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Wed Sep 23 13:40:37 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Wed Sep 23 13:52:16 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ---
1 file changed, 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4697d94c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ecb3704..c0f630c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -211,9 +211,6 @@ Release 2.6.1 - 2015-09-16
YARN-3740. Fixed the typo in the configuration name:
APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
- YARN-3379. Fixed missing data in localityTable and ResourceRequests table
- in RM WebUI. (Xuan Gong via jianhe)
-
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
(Varun Vasudev via xgong)
[39/50] [abbrv] hadoop git commit: YARN-2890. MiniYarnCluster should
turn on timeline service if configured to do so. Contributed by Mit Desai.
Posted by sj...@apache.org.
YARN-2890. MiniYarnCluster should turn on timeline service if configured to do so. Contributed by Mit Desai.
(cherry picked from commit 265ed1fe804743601a8b62cabc1e4dc2ec8e502f)
(cherry picked from commit 55b794e7fa205df655c19bbfe1de99091fa9dc64)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dba2b60f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dba2b60f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dba2b60f
Branch: refs/heads/branch-2.6
Commit: dba2b60fdd83aa501e158f42abc75d5ce9ad3cee
Parents: 9c4a6e1
Author: Hitesh Shah <hi...@apache.org>
Authored: Wed Apr 8 14:13:10 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:15 2015 -0700
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 52 ++++++++-
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 6 +-
.../hadoop/yarn/server/TestMiniYarnCluster.java | 115 +++++++++++++++++++
6 files changed, 172 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index 43e3dbe..de260c9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index c2ef128..eab9026 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,6 +35,52 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+
+ /*
+ * Timeline service should not start if the config is set to false
+ * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
+ MiniMRYarnCluster cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
+ cluster = null;
+ try {
+ cluster = new MiniMRYarnCluster(
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+
+ @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -42,7 +88,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -96,7 +142,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -133,7 +179,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6ef7c62..6c72dd7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -220,6 +220,9 @@ Release 2.6.1 - 2015-09-09
YARN-3544. Got back AM logs link on the RM web UI for a completed app.
(Xuan Gong via zjshen)
+ YARN-2890. MiniYarnCluster should turn on timeline service if
+ configured to do so. (Mit Desai via hitesh)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 17c29ec..904ad58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -84,7 +84,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1, true);
+ numNodeManager, 1, 1);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index e83d601..cdaa9f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,7 +57,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -260,8 +259,9 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if (enableAHS) {
- addService(new ApplicationHistoryServerWrapper());
+ if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED) || enableAHS) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dba2b60f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
new file mode 100644
index 0000000..8a3c9e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnCluster.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestMiniYarnCluster {
+ @Test
+ public void testTimelineServiceStartInMiniCluster() throws Exception {
+ Configuration conf = new YarnConfiguration();
+ int numNodeManagers = 1;
+ int numLocalDirs = 1;
+ int numLogDirs = 1;
+ boolean enableAHS;
+
+ /*
+ * Timeline service should not start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS flag == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = false;
+ MiniYARNCluster cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //verify that the timeline service is not started.
+ Assert.assertNull("Timeline Service should not have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == true
+ * and enableAHS == false
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ enableAHS = false;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ /*
+ * Timeline service should start if TIMELINE_SERVICE_ENABLED == false
+ * and enableAHS == true
+ */
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
+ enableAHS = true;
+ cluster = null;
+ try {
+ cluster = new MiniYARNCluster(TestMiniYarnCluster.class.getSimpleName(),
+ numNodeManagers, numLocalDirs, numLogDirs, numLogDirs, enableAHS);
+ cluster.init(conf);
+ cluster.start();
+
+ //Timeline service may sometime take a while to get started
+ int wait = 0;
+ while(cluster.getApplicationHistoryServer() == null && wait < 20) {
+ Thread.sleep(500);
+ wait++;
+ }
+ //verify that the timeline service is started.
+ Assert.assertNotNull("Timeline Service should have been started",
+ cluster.getApplicationHistoryServer());
+ }
+ finally {
+ if(cluster != null) {
+ cluster.stop();
+ }
+ }
+ }
+}
[05/50] [abbrv] hadoop git commit: Add missing test file of YARN-3733
Posted by sj...@apache.org.
Add missing test file of YARN-3733
(cherry picked from commit 405bbcf68c32d8fd8a83e46e686eacd14e5a533c)
(cherry picked from commit 344b7509153cdd993218cd5104c7e5c07cd35d3c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2073fc0f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2073fc0f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2073fc0f
Branch: refs/heads/branch-2.6
Commit: 2073fc0f84c75434f023a1145d2f14289a3f5ef1
Parents: 85d9272
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jun 4 13:18:25 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 17:43:03 2015 -0700
----------------------------------------------------------------------
.../util/resource/TestResourceCalculator.java | 125 +++++++++++++++++++
1 file changed, 125 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2073fc0f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
new file mode 100644
index 0000000..6a0b62e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/resource/TestResourceCalculator.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.util.resource;
+
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestResourceCalculator {
+ private ResourceCalculator resourceCalculator;
+
+ @Parameterized.Parameters
+ public static Collection<ResourceCalculator[]> getParameters() {
+ return Arrays.asList(new ResourceCalculator[][] {
+ { new DefaultResourceCalculator() },
+ { new DominantResourceCalculator() } });
+ }
+
+ public TestResourceCalculator(ResourceCalculator rs) {
+ this.resourceCalculator = rs;
+ }
+
+ @Test(timeout = 10000)
+ public void testResourceCalculatorCompareMethod() {
+ Resource clusterResource = Resource.newInstance(0, 0);
+
+ // For lhs == rhs
+ Resource lhs = Resource.newInstance(0, 0);
+ Resource rhs = Resource.newInstance(0, 0);
+ assertResourcesOperations(clusterResource, lhs, rhs, false, true, false,
+ true, lhs, lhs);
+
+ // lhs > rhs
+ lhs = Resource.newInstance(1, 1);
+ rhs = Resource.newInstance(0, 0);
+ assertResourcesOperations(clusterResource, lhs, rhs, false, false, true,
+ true, lhs, rhs);
+
+ // For lhs < rhs
+ lhs = Resource.newInstance(0, 0);
+ rhs = Resource.newInstance(1, 1);
+ assertResourcesOperations(clusterResource, lhs, rhs, true, true, false,
+ false, rhs, lhs);
+
+ if (!(resourceCalculator instanceof DominantResourceCalculator)) {
+ return;
+ }
+
+ // verify for 2 dimensional resources i.e memory and cpu
+ // dominant resource types
+ lhs = Resource.newInstance(1, 0);
+ rhs = Resource.newInstance(0, 1);
+ assertResourcesOperations(clusterResource, lhs, rhs, false, true, false,
+ true, lhs, lhs);
+
+ lhs = Resource.newInstance(0, 1);
+ rhs = Resource.newInstance(1, 0);
+ assertResourcesOperations(clusterResource, lhs, rhs, false, true, false,
+ true, lhs, lhs);
+
+ lhs = Resource.newInstance(1, 1);
+ rhs = Resource.newInstance(1, 0);
+ assertResourcesOperations(clusterResource, lhs, rhs, false, false, true,
+ true, lhs, rhs);
+
+ lhs = Resource.newInstance(0, 1);
+ rhs = Resource.newInstance(1, 1);
+ assertResourcesOperations(clusterResource, lhs, rhs, true, true, false,
+ false, rhs, lhs);
+
+ }
+
+
+ private void assertResourcesOperations(Resource clusterResource,
+ Resource lhs, Resource rhs, boolean lessThan, boolean lessThanOrEqual,
+ boolean greaterThan, boolean greaterThanOrEqual, Resource max,
+ Resource min) {
+
+ Assert.assertEquals("Less Than operation is wrongly calculated.", lessThan,
+ Resources.lessThan(resourceCalculator, clusterResource, lhs, rhs));
+
+ Assert.assertEquals(
+ "Less Than Or Equal To operation is wrongly calculated.",
+ lessThanOrEqual, Resources.lessThanOrEqual(resourceCalculator,
+ clusterResource, lhs, rhs));
+
+ Assert.assertEquals("Greater Than operation is wrongly calculated.",
+ greaterThan,
+ Resources.greaterThan(resourceCalculator, clusterResource, lhs, rhs));
+
+ Assert.assertEquals(
+ "Greater Than Or Equal To operation is wrongly calculated.",
+ greaterThanOrEqual, Resources.greaterThanOrEqual(resourceCalculator,
+ clusterResource, lhs, rhs));
+
+ Assert.assertEquals("Max(value) Operation wrongly calculated.", max,
+ Resources.max(resourceCalculator, clusterResource, lhs, rhs));
+
+ Assert.assertEquals("Min(value) operation is wrongly calculated.", min,
+ Resources.min(resourceCalculator, clusterResource, lhs, rhs));
+ }
+
+}
\ No newline at end of file
[50/50] [abbrv] hadoop git commit: YARN-4087. Followup fixes after
YARN-2019 regarding RM behavior when state-store error occurs. Contributed by
Jian He
Posted by sj...@apache.org.
YARN-4087. Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs. Contributed by Jian He
(cherry picked from commit 9f7fcb54e798cf4fda1ea7972dd96491976e1857)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1828ba00
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1828ba00
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1828ba00
Branch: refs/heads/branch-2.6
Commit: 1828ba00be40721ae798e2288a75a5e7d187b643
Parents: d27f09c
Author: Xuan <xg...@apache.org>
Authored: Mon Sep 7 18:09:35 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 16:43:06 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 2 +-
.../src/main/resources/yarn-default.xml | 5 ++++-
.../resourcemanager/recovery/RMStateStore.java | 15 +++++++++------
4 files changed, 17 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 15d082b..fdf65bb 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -21,6 +21,9 @@ Release 2.6.2 - UNRELEASED
BUG FIXES
+ YARN-4087. Followup fixes after YARN-2019 regarding RM behavior when
+ state-store error occurs. (Jian He via xgong)
+
Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 471297c..d7f24ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -381,7 +381,7 @@ public class YarnConfiguration extends Configuration {
public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast";
- public static final boolean DEFAULT_YARN_FAIL_FAST = true;
+ public static final boolean DEFAULT_YARN_FAIL_FAST = false;
public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index fa2e5cc..9b56d99 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -286,9 +286,12 @@
<property>
<description>Should YARN fail fast if it encounters any errors.
+ This is a global config for all other components including RM,NM etc.
+ If no value is set for component-specific config (e.g yarn.resourcemanager.fail-fast),
+ this value will be the default.
</description>
<name>yarn.fail-fast</name>
- <value>true</value>
+ <value>false</value>
</property>
<property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1828ba00/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 37855f7..fa912ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
+import org.apache.hadoop.yarn.conf.HAUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -822,17 +823,19 @@ public abstract class RMStateStore extends AbstractService {
*/
protected void notifyStoreOperationFailed(Exception failureCause) {
LOG.error("State store operation failed ", failureCause);
- if (failureCause instanceof StoreFencedException) {
+ if (HAUtil.isHAEnabled(getConfig())) {
+ LOG.warn("State-store fenced ! Transitioning RM to standby");
Thread standByTransitionThread =
new Thread(new StandByTransitionThread());
standByTransitionThread.setName("StandByTransitionThread Handler");
standByTransitionThread.start();
+ } else if (YarnConfiguration.shouldRMFailFast(getConfig())) {
+ LOG.fatal("Fail RM now due to state-store error!");
+ rmDispatcher.getEventHandler().handle(
+ new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED,
+ failureCause));
} else {
- if (YarnConfiguration.shouldRMFailFast(getConfig())) {
- rmDispatcher.getEventHandler().handle(
- new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED,
- failureCause));
- }
+ LOG.warn("Skip the state-store error.");
}
}
[40/50] [abbrv] hadoop git commit: Set the release date for 2.6.1-RC1
Posted by sj...@apache.org.
Set the release date for 2.6.1-RC1
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b4d876d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b4d876d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b4d876d8
Branch: refs/heads/branch-2.6
Commit: b4d876d837b830405ccdb6af94742f99d49f9c04
Parents: dba2b60
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 16 12:04:44 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 16 12:04:57 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d876d8/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index bfe6a96..1511483 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop Change Log
-Release 2.6.1 - 2015-09-09
+Release 2.6.1 - 2015-09-16
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d876d8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8800a83..2c38a58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop HDFS Change Log
-Release 2.6.1 - 2015-09-09
+Release 2.6.1 - 2015-09-16
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d876d8/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 0fa1e45..d6a8f76 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop MapReduce Change Log
-Release 2.6.1 - 2015-09-09
+Release 2.6.1 - 2015-09-16
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b4d876d8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6c72dd7..ecb3704 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop YARN Change Log
-Release 2.6.1 - 2015-09-09
+Release 2.6.1 - 2015-09-16
INCOMPATIBLE CHANGES
[27/50] [abbrv] hadoop git commit: Preparing for release 2.6.1:
Updating site update page.
Posted by sj...@apache.org.
Preparing for release 2.6.1: Updating site update page.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfb012a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfb012a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfb012a2
Branch: refs/heads/branch-2.6
Commit: cfb012a25548fe2ea6cddbf9bf3ed93680e24449
Parents: adc3405
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 9 15:29:03 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 9 15:29:03 2015 -0700
----------------------------------------------------------------------
hadoop-project/src/site/apt/index.apt.vm | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfb012a2/hadoop-project/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/apt/index.apt.vm b/hadoop-project/src/site/apt/index.apt.vm
index 661d942..27bdf4f 100644
--- a/hadoop-project/src/site/apt/index.apt.vm
+++ b/hadoop-project/src/site/apt/index.apt.vm
@@ -19,7 +19,7 @@
Apache Hadoop ${project.version}
Apache Hadoop ${project.version} is a minor release in the 2.x.y release
- line, building upon the previous stable release 2.4.1.
+ line, building upon the previous stable release 2.6.0.
Here is a short overview of the major features and improvements.
[30/50] [abbrv] hadoop git commit: Preparing for release 2.6.1:
Posting release notes generated.
Posted by sj...@apache.org.
Preparing for release 2.6.1: Posting release notes generated.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c86d9bd1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c86d9bd1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c86d9bd1
Branch: refs/heads/branch-2.6
Commit: c86d9bd10bcb72305b49ca1b8fe44d327a33ff2a
Parents: 3462a00
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 9 15:31:14 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 9 15:31:14 2015 -0700
----------------------------------------------------------------------
.../src/main/docs/releasenotes.html | 29494 +----------------
1 file changed, 479 insertions(+), 29015 deletions(-)
----------------------------------------------------------------------
[46/50] [abbrv] hadoop git commit: MAPREDUCE-6454. Fixed MapReduce to
modify HADOOP_CLASSPATH to have distributed cache files so that child
processes running hadoop scripts can access these files. Contributed by
Junping Du.
Posted by sj...@apache.org.
MAPREDUCE-6454. Fixed MapReduce to modify HADOOP_CLASSPATH to have distributed cache files so that child processes running hadoop scripts can access these files. Contributed by Junping Du.
(cherry picked from commit 9da81a395584cb1dbfdee012222fd4de53522681)
(cherry picked from commit d4b86e6f403edc6cdd687f978e3abff8752faa34)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab4c2ce2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab4c2ce2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab4c2ce2
Branch: refs/heads/branch-2.6
Commit: ab4c2ce248f4595a7dab071c397baae37d63f932
Parents: b99a851
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Aug 20 17:35:24 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 16:11:11 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 4 ++
.../v2/app/job/impl/TaskAttemptImpl.java | 15 ++++-
.../apache/hadoop/mapreduce/v2/util/MRApps.java | 41 ++++++++++---
.../hadoop/mapreduce/v2/util/TestMRApps.java | 63 ++++++++++++++------
.../hadoop/yarn/api/ApplicationConstants.java | 5 ++
5 files changed, 98 insertions(+), 30 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab4c2ce2/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index cb0cca0..7270b8c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -12,6 +12,10 @@ Release 2.6.2 - UNRELEASED
BUG FIXES
+ MAPREDUCE-6454. Fixed MapReduce to modify HADOOP_CLASSPATH to have distributed
+ cache files so that child processes running hadoop scripts can access these
+ files. (Junping Du via vinodkv)
+
Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab4c2ce2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 5a8a658..53d400d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -172,6 +172,7 @@ public abstract class TaskAttemptImpl implements
private static AtomicBoolean initialClasspathFlag = new AtomicBoolean();
private static String initialClasspath = null;
private static String initialAppClasspath = null;
+ private static String initialHadoopClasspath = null;
private static Object commonContainerSpecLock = new Object();
private static ContainerLaunchContext commonContainerSpec = null;
private static final Object classpathLock = new Object();
@@ -620,6 +621,7 @@ public abstract class TaskAttemptImpl implements
MRApps.setClasspath(env, conf);
initialClasspath = env.get(Environment.CLASSPATH.name());
initialAppClasspath = env.get(Environment.APP_CLASSPATH.name());
+ initialHadoopClasspath = env.get(Environment.HADOOP_CLASSPATH.name());
initialClasspathFlag.set(true);
return initialClasspath;
}
@@ -753,14 +755,21 @@ public abstract class TaskAttemptImpl implements
}
MRApps.addToEnvironment(
- environment,
- Environment.CLASSPATH.name(),
+ environment,
+ Environment.CLASSPATH.name(),
getInitialClasspath(conf), conf);
+ if (initialHadoopClasspath != null) {
+ MRApps.addToEnvironment(
+ environment,
+ Environment.HADOOP_CLASSPATH.name(),
+ initialHadoopClasspath, conf);
+ }
+
if (initialAppClasspath != null) {
MRApps.addToEnvironment(
environment,
- Environment.APP_CLASSPATH.name(),
+ Environment.APP_CLASSPATH.name(),
initialAppClasspath, conf);
}
} catch (IOException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab4c2ce2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
index 07b9427..af62c1c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
@@ -231,7 +231,7 @@ public class MRApps extends Apps {
}
// TODO: Remove duplicates.
}
-
+
@SuppressWarnings("deprecation")
public static void setClasspath(Map<String, String> environment,
Configuration conf) throws IOException {
@@ -242,11 +242,30 @@ public class MRApps extends Apps {
conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, false)
? Environment.APP_CLASSPATH.name() : Environment.CLASSPATH.name();
+ String hadoopClasspathEnvVar = Environment.HADOOP_CLASSPATH.name();
+
MRApps.addToEnvironment(environment,
classpathEnvVar, crossPlatformifyMREnv(conf, Environment.PWD), conf);
+
+ MRApps.addToEnvironment(environment,
+ hadoopClasspathEnvVar, crossPlatformifyMREnv(conf, Environment.PWD),
+ conf);
+
if (!userClassesTakesPrecedence) {
MRApps.setMRFrameworkClasspath(environment, conf);
}
+
+ addClasspathToEnv(environment, classpathEnvVar, conf);
+ addClasspathToEnv(environment, hadoopClasspathEnvVar, conf);
+
+ if (userClassesTakesPrecedence) {
+ MRApps.setMRFrameworkClasspath(environment, conf);
+ }
+ }
+
+ @SuppressWarnings("deprecation")
+ public static void addClasspathToEnv(Map<String, String> environment,
+ String classpathEnvVar, Configuration conf) throws IOException {
MRApps.addToEnvironment(
environment,
classpathEnvVar,
@@ -254,15 +273,21 @@ public class MRApps extends Apps {
MRApps.addToEnvironment(
environment,
classpathEnvVar,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR, conf);
+ MRJobConfig.JOB_JAR + Path.SEPARATOR + "classes" + Path.SEPARATOR,
+ conf);
+
MRApps.addToEnvironment(
environment,
classpathEnvVar,
- MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*", conf);
+ MRJobConfig.JOB_JAR + Path.SEPARATOR + "lib" + Path.SEPARATOR + "*",
+ conf);
+
MRApps.addToEnvironment(
environment,
classpathEnvVar,
- crossPlatformifyMREnv(conf, Environment.PWD) + Path.SEPARATOR + "*", conf);
+ crossPlatformifyMREnv(conf, Environment.PWD) + Path.SEPARATOR + "*",
+ conf);
+
// a * in the classpath will only find a .jar, so we need to filter out
// all .jars and add everything else
addToClasspathIfNotJar(DistributedCache.getFileClassPaths(conf),
@@ -273,11 +298,8 @@ public class MRApps extends Apps {
DistributedCache.getCacheArchives(conf),
conf,
environment, classpathEnvVar);
- if (userClassesTakesPrecedence) {
- MRApps.setMRFrameworkClasspath(environment, conf);
- }
}
-
+
/**
* Add the paths to the classpath if they are not jars
* @param paths the paths to add to the classpath
@@ -443,6 +465,7 @@ public class MRApps extends Apps {
return startCommitFile;
}
+ @SuppressWarnings("deprecation")
public static void setupDistributedCache(
Configuration conf,
Map<String, LocalResource> localResources)
@@ -475,6 +498,7 @@ public class MRApps extends Apps {
* @param conf
* @throws java.io.IOException
*/
+ @SuppressWarnings("deprecation")
public static void setupDistributedCacheLocal(Configuration conf)
throws IOException {
@@ -542,6 +566,7 @@ public class MRApps extends Apps {
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
+ @SuppressWarnings("deprecation")
private static void parseDistributedCacheArtifacts(
Configuration conf,
Map<String, LocalResource> localResources,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab4c2ce2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
index 2e77f5e..87473ba 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
@@ -36,6 +36,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FilterFileSystem;
@@ -66,7 +68,10 @@ import org.junit.Test;
public class TestMRApps {
private static File testWorkDir = null;
-
+
+ private static final Log LOG =
+ LogFactory.getLog(TestMRApps.class);
+
@BeforeClass
public static void setupTestDirs() throws IOException {
testWorkDir = new File("target", TestMRApps.class.getCanonicalName());
@@ -74,14 +79,14 @@ public class TestMRApps {
testWorkDir.mkdirs();
testWorkDir = testWorkDir.getAbsoluteFile();
}
-
+
@AfterClass
public static void cleanupTestDirs() throws IOException {
if (testWorkDir != null) {
delete(testWorkDir);
}
}
-
+
private static void delete(File dir) throws IOException {
Configuration conf = new Configuration();
FileSystem fs = FileSystem.getLocal(conf);
@@ -193,9 +198,9 @@ public class TestMRApps {
Job job = Job.getInstance(conf);
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, job.getConfiguration());
- assertTrue(environment.get("CLASSPATH").startsWith(
- ApplicationConstants.Environment.PWD.$$()
- + ApplicationConstants.CLASS_PATH_SEPARATOR));
+ assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.
+ name()).startsWith(ApplicationConstants.Environment.PWD.$$()
+ + ApplicationConstants.CLASS_PATH_SEPARATOR));
String yarnAppClasspath = job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
StringUtils.join(",",
@@ -205,8 +210,9 @@ public class TestMRApps {
yarnAppClasspath.replaceAll(",\\s*",
ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
}
- assertTrue(environment.get("CLASSPATH").contains(yarnAppClasspath));
- String mrAppClasspath =
+ assertTrue(environment.get(ApplicationConstants.Environment.
+ CLASSPATH.name()).contains(yarnAppClasspath));
+ String mrAppClasspath =
job.getConfiguration().get(
MRJobConfig.MAPREDUCE_APPLICATION_CLASSPATH,
MRJobConfig.DEFAULT_MAPREDUCE_CROSS_PLATFORM_APPLICATION_CLASSPATH);
@@ -215,7 +221,8 @@ public class TestMRApps {
mrAppClasspath.replaceAll(",\\s*",
ApplicationConstants.CLASS_PATH_SEPARATOR).trim();
}
- assertTrue(environment.get("CLASSPATH").contains(mrAppClasspath));
+ assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.
+ name()).contains(mrAppClasspath));
}
@Test (timeout = 120000)
@@ -234,8 +241,12 @@ public class TestMRApps {
conf.set(MRJobConfig.CACHE_ARCHIVES, testTGZQualifiedPath + "#testTGZ");
Map<String, String> environment = new HashMap<String, String>();
MRApps.setClasspath(environment, conf);
- assertTrue(environment.get("CLASSPATH").startsWith(
+ assertTrue(environment.get(ApplicationConstants.Environment.CLASSPATH.name()).startsWith(
ApplicationConstants.Environment.PWD.$$() + ApplicationConstants.CLASS_PATH_SEPARATOR));
+ assertTrue(environment.get(ApplicationConstants.Environment.
+ HADOOP_CLASSPATH.name()).startsWith(
+ ApplicationConstants.Environment.PWD.$$() +
+ ApplicationConstants.CLASS_PATH_SEPARATOR));
String confClasspath = job.getConfiguration().get(
YarnConfiguration.YARN_APPLICATION_CLASSPATH,
StringUtils.join(",",
@@ -244,8 +255,19 @@ public class TestMRApps {
confClasspath = confClasspath.replaceAll(",\\s*", ApplicationConstants.CLASS_PATH_SEPARATOR)
.trim();
}
- assertTrue(environment.get("CLASSPATH").contains(confClasspath));
- assertTrue(environment.get("CLASSPATH").contains("testTGZ"));
+ LOG.info("CLASSPATH: "+ environment.get(
+ ApplicationConstants.Environment.CLASSPATH.name()));
+ LOG.info("confClasspath: " + confClasspath);
+ assertTrue(environment.get(
+ ApplicationConstants.Environment.CLASSPATH.name()).contains(
+ confClasspath));
+ LOG.info("HADOOP_CLASSPATH: " + environment.get(
+ ApplicationConstants.Environment.HADOOP_CLASSPATH.name()));
+ assertTrue(environment.get(
+ ApplicationConstants.Environment.CLASSPATH.name()).contains("testTGZ"));
+ assertTrue(environment.get(
+ ApplicationConstants.Environment.HADOOP_CLASSPATH.name()).
+ contains("testTGZ"));
}
@Test (timeout = 120000)
@@ -259,7 +281,7 @@ public class TestMRApps {
} catch (Exception e) {
fail("Got exception while setting classpath");
}
- String env_str = env.get("CLASSPATH");
+ String env_str = env.get(ApplicationConstants.Environment.CLASSPATH.name());
String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
Arrays.asList(ApplicationConstants.Environment.PWD.$$(), "job.jar/job.jar",
"job.jar/classes/", "job.jar/lib/*",
@@ -279,7 +301,7 @@ public class TestMRApps {
} catch (Exception e) {
fail("Got exception while setting classpath");
}
- String env_str = env.get("CLASSPATH");
+ String env_str = env.get(ApplicationConstants.Environment.CLASSPATH.name());
String expectedClasspath = StringUtils.join(ApplicationConstants.CLASS_PATH_SEPARATOR,
Arrays.asList("job.jar/job.jar", "job.jar/classes/", "job.jar/lib/*",
ApplicationConstants.Environment.PWD.$$() + "/*"));
@@ -296,8 +318,9 @@ public class TestMRApps {
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_CLASSLOADER, true);
Map<String, String> env = new HashMap<String, String>();
MRApps.setClasspath(env, conf);
- String cp = env.get("CLASSPATH");
- String appCp = env.get("APP_CLASSPATH");
+ String cp = env.get(ApplicationConstants.Environment.CLASSPATH.name());
+ String appCp = env.get(ApplicationConstants.Environment.
+ APP_CLASSPATH.name());
assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but job.jar is in the"
+ " classpath!", cp.contains("jar" + ApplicationConstants.CLASS_PATH_SEPARATOR + "job"));
assertFalse("MAPREDUCE_JOB_CLASSLOADER true, but PWD is in the classpath!",
@@ -338,7 +361,8 @@ public class TestMRApps {
Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
FRAMEWORK_CLASSPATH, stdClasspath));
assertEquals("Incorrect classpath with framework and no user precedence",
- expectedClasspath, env.get("CLASSPATH"));
+ expectedClasspath, env.get(ApplicationConstants.Environment.
+ CLASSPATH.name()));
env.clear();
conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_USER_CLASSPATH_FIRST, true);
@@ -347,7 +371,8 @@ public class TestMRApps {
Arrays.asList(ApplicationConstants.Environment.PWD.$$(),
stdClasspath, FRAMEWORK_CLASSPATH));
assertEquals("Incorrect classpath with framework and user precedence",
- expectedClasspath, env.get("CLASSPATH"));
+ expectedClasspath, env.get(ApplicationConstants.Environment.CLASSPATH.
+ name()));
}
@Test (timeout = 30000)
@@ -358,7 +383,7 @@ public class TestMRApps {
assertTrue("Empty Config did not produce an empty list of resources",
localResources.isEmpty());
}
-
+
@SuppressWarnings("deprecation")
public void testSetupDistributedCacheConflicts() throws Exception {
Configuration conf = new Configuration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab4c2ce2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
index d42cd67..b00c15d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/ApplicationConstants.java
@@ -160,6 +160,11 @@ public interface ApplicationConstants {
APP_CLASSPATH("APP_CLASSPATH"),
/**
+ * $HADOOP_CLASSPATH.
+ */
+ HADOOP_CLASSPATH("HADOOP_CLASSPATH"),
+
+ /**
* $LD_LIBRARY_PATH
*/
LD_LIBRARY_PATH("LD_LIBRARY_PATH"),
[10/50] [abbrv] hadoop git commit: YARN-3099. Capacity Scheduler
LeafQueue/ParentQueue should use ResourceUsage to track
used-resources-by-label. Contributed by Wangda Tan (cherry picked from commit
86358221fc85a7743052a0b4c1647353508bf308)
Posted by sj...@apache.org.
YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage to track used-resources-by-label. Contributed by Wangda Tan
(cherry picked from commit 86358221fc85a7743052a0b4c1647353508bf308)
(cherry picked from commit cabf97ae4f2dad53c7b9e3d10a67876b16d94074)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d9281fbb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d9281fbb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d9281fbb
Branch: refs/heads/branch-2.6
Commit: d9281fbbab4a2e0475789e724f21faa52a465f62
Parents: b0ad553
Author: Jian He <ji...@apache.org>
Authored: Fri Jan 30 15:15:20 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:20 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/capacity/AbstractCSQueue.java | 48 +++----
.../scheduler/capacity/LeafQueue.java | 128 +++++++------------
.../scheduler/capacity/ParentQueue.java | 25 ++--
.../TestWorkPreservingRMRestart.java | 2 +-
.../scheduler/capacity/TestCSQueueUtils.java | 24 ++--
6 files changed, 93 insertions(+), 137 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 975e4e1..bac13ff 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -180,6 +180,9 @@ Release 2.6.1 - UNRELEASED
YARN-2978. Fixed potential NPE while getting queue info. (Varun Saxena via
jianhe)
+ YARN-3099. Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage
+ to track used-resources-by-label.(Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fec3a56..3c1663f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -35,10 +35,11 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
public abstract class AbstractCSQueue implements CSQueue {
@@ -64,16 +65,17 @@ public abstract class AbstractCSQueue implements CSQueue {
Set<String> accessibleLabels;
RMNodeLabelsManager labelManager;
String defaultLabelExpression;
- Resource usedResources = Resources.createResource(0, 0);
Map<String, Float> absoluteCapacityByNodeLabels;
Map<String, Float> capacitiyByNodeLabels;
- Map<String, Resource> usedResourcesByNodeLabels = new HashMap<String, Resource>();
Map<String, Float> absoluteMaxCapacityByNodeLabels;
Map<String, Float> maxCapacityByNodeLabels;
Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
boolean reservationsContinueLooking;
+
+ // Track resource usage-by-label like used-resource/pending-resource, etc.
+ ResourceUsage queueUsage;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -120,6 +122,7 @@ public abstract class AbstractCSQueue implements CSQueue {
maxCapacityByNodeLabels =
cs.getConfiguration().getMaximumNodeLabelCapacities(getQueuePath(),
accessibleLabels, labelManager);
+ queueUsage = new ResourceUsage();
}
@Override
@@ -153,8 +156,8 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized Resource getUsedResources() {
- return usedResources;
+ public Resource getUsedResources() {
+ return queueUsage.getUsed();
}
public synchronized int getNumContainers() {
@@ -344,22 +347,13 @@ public abstract class AbstractCSQueue implements CSQueue {
synchronized void allocateResource(Resource clusterResource,
Resource resource, Set<String> nodeLabels) {
- Resources.addTo(usedResources, resource);
// Update usedResources by labels
if (nodeLabels == null || nodeLabels.isEmpty()) {
- if (!usedResourcesByNodeLabels.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- usedResourcesByNodeLabels.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.addTo(usedResourcesByNodeLabels.get(RMNodeLabelsManager.NO_LABEL),
- resource);
+ queueUsage.incUsed(resource);
} else {
for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
- Resources.addTo(usedResourcesByNodeLabels.get(label), resource);
+ queueUsage.incUsed(label, resource);
}
}
@@ -370,23 +364,12 @@ public abstract class AbstractCSQueue implements CSQueue {
protected synchronized void releaseResource(Resource clusterResource,
Resource resource, Set<String> nodeLabels) {
- // Update queue metrics
- Resources.subtractFrom(usedResources, resource);
-
// Update usedResources by labels
if (null == nodeLabels || nodeLabels.isEmpty()) {
- if (!usedResourcesByNodeLabels.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- usedResourcesByNodeLabels.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.subtractFrom(
- usedResourcesByNodeLabels.get(RMNodeLabelsManager.NO_LABEL), resource);
+ queueUsage.decUsed(resource);
} else {
for (String label : Sets.intersection(accessibleLabels, nodeLabels)) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
- Resources.subtractFrom(usedResourcesByNodeLabels.get(label), resource);
+ queueUsage.decUsed(label, resource);
}
}
@@ -452,6 +435,11 @@ public abstract class AbstractCSQueue implements CSQueue {
@Private
public Resource getUsedResourceByLabel(String nodeLabel) {
- return usedResourcesByNodeLabels.get(nodeLabel);
+ return queueUsage.getUsed(nodeLabel);
+ }
+
+ @VisibleForTesting
+ public ResourceUsage getResourceUsage() {
+ return queueUsage;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 78b5fb6..6fbc8e4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEven
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
@@ -117,10 +118,6 @@ public class LeafQueue extends AbstractCSQueue {
private volatile float absoluteMaxAvailCapacity;
- // sum of resources used by application masters for applications
- // running in this queue
- private final Resource usedAMResources = Resource.newInstance(0, 0);
-
public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
@@ -440,7 +437,7 @@ public class LeafQueue extends AbstractCSQueue {
return queueName + ": " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
- "usedResources=" + usedResources + ", " +
+ "usedResources=" + queueUsage.getUsed() + ", " +
"usedCapacity=" + getUsedCapacity() + ", " +
"absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
@@ -469,7 +466,7 @@ public class LeafQueue extends AbstractCSQueue {
ArrayList<UserInfo> usersToReturn = new ArrayList<UserInfo>();
for (Map.Entry<String, User> entry: users.entrySet()) {
usersToReturn.add(new UserInfo(entry.getKey(), Resources.clone(
- entry.getValue().consumed), entry.getValue().getActiveApplications(),
+ entry.getValue().getUsed()), entry.getValue().getActiveApplications(),
entry.getValue().getPendingApplications()));
}
return usersToReturn;
@@ -638,7 +635,7 @@ public class LeafQueue extends AbstractCSQueue {
// Check am resource limit
Resource amIfStarted =
- Resources.add(application.getAMResource(), usedAMResources);
+ Resources.add(application.getAMResource(), queueUsage.getAMUsed());
if (LOG.isDebugEnabled()) {
LOG.debug("application AMResource " + application.getAMResource() +
@@ -683,9 +680,8 @@ public class LeafQueue extends AbstractCSQueue {
}
user.activateApplication();
activeApplications.add(application);
- Resources.addTo(usedAMResources, application.getAMResource());
- Resources.addTo(user.getConsumedAMResources(),
- application.getAMResource());
+ queueUsage.incAMUsed(application.getAMResource());
+ user.getResourceUsage().incAMUsed(application.getAMResource());
i.remove();
LOG.info("Application " + application.getApplicationId() +
" from user: " + application.getUser() +
@@ -736,9 +732,8 @@ public class LeafQueue extends AbstractCSQueue {
if (!wasActive) {
pendingApplications.remove(application);
} else {
- Resources.subtractFrom(usedAMResources, application.getAMResource());
- Resources.subtractFrom(user.getConsumedAMResources(),
- application.getAMResource());
+ queueUsage.decAMUsed(application.getAMResource());
+ user.getResourceUsage().decAMUsed(application.getAMResource());
}
applicationAttemptMap.remove(application.getApplicationAttemptId());
@@ -987,8 +982,8 @@ public class LeafQueue extends AbstractCSQueue {
*/
Resource headroom =
Resources.min(resourceCalculator, clusterResource,
- Resources.subtract(userLimit, user.getTotalConsumedResources()),
- Resources.subtract(queueMaxCap, usedResources)
+ Resources.subtract(userLimit, user.getUsed()),
+ Resources.subtract(queueMaxCap, queueUsage.getUsed())
);
return headroom;
}
@@ -1008,12 +1003,8 @@ public class LeafQueue extends AbstractCSQueue {
boolean canAssign = true;
for (String label : labelCanAccess) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
-
Resource potentialTotalCapacity =
- Resources.add(usedResourcesByNodeLabels.get(label), required);
+ Resources.add(queueUsage.getUsed(label), required);
float potentialNewCapacity =
Resources.divide(resourceCalculator, clusterResource,
@@ -1036,14 +1027,14 @@ public class LeafQueue extends AbstractCSQueue {
LOG.debug("try to use reserved: "
+ getQueueName()
+ " usedResources: "
- + usedResources
+ + queueUsage.getUsed()
+ " clusterResources: "
+ clusterResource
+ " reservedResources: "
+ application.getCurrentReservation()
+ " currentCapacity "
+ Resources.divide(resourceCalculator, clusterResource,
- usedResources, clusterResource) + " required " + required
+ queueUsage.getUsed(), clusterResource) + " required " + required
+ " potentialNewWithoutReservedCapacity: "
+ potentialNewWithoutReservedCapacity + " ( " + " max-capacity: "
+ absoluteMaxCapacity + ")");
@@ -1063,11 +1054,11 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
LOG.debug(getQueueName()
+ "Check assign to queue, label=" + label
- + " usedResources: " + usedResourcesByNodeLabels.get(label)
+ + " usedResources: " + queueUsage.getUsed(label)
+ " clusterResources: " + clusterResource
+ " currentCapacity "
+ Resources.divide(resourceCalculator, clusterResource,
- usedResourcesByNodeLabels.get(label),
+ queueUsage.getUsed(label),
labelManager.getResourceByLabel(label, clusterResource))
+ " potentialNewCapacity: " + potentialNewCapacity + " ( "
+ " max-capacity: " + absoluteMaxCapacity + ")");
@@ -1118,7 +1109,7 @@ public class LeafQueue extends AbstractCSQueue {
LOG.debug("Headroom calculation for user " + user + ": " +
" userLimit=" + userLimit +
" queueMaxCap=" + queueMaxCap +
- " consumed=" + queueUser.getTotalConsumedResources() +
+ " consumed=" + queueUser.getUsed() +
" headroom=" + headroom);
}
@@ -1173,8 +1164,8 @@ public class LeafQueue extends AbstractCSQueue {
Resource currentCapacity =
Resources.lessThan(resourceCalculator, clusterResource,
- usedResources, queueCapacity) ?
- queueCapacity : Resources.add(usedResources, required);
+ queueUsage.getUsed(), queueCapacity) ?
+ queueCapacity : Resources.add(queueUsage.getUsed(), required);
// Never allow a single user to take more than the
// queue's configured capacity * user-limit-factor.
@@ -1209,10 +1200,10 @@ public class LeafQueue extends AbstractCSQueue {
" userLimit=" + userLimit +
" userLimitFactor=" + userLimitFactor +
" required: " + required +
- " consumed: " + user.getTotalConsumedResources() +
+ " consumed: " + user.getUsed() +
" limit: " + limit +
" queueCapacity: " + queueCapacity +
- " qconsumed: " + usedResources +
+ " qconsumed: " + queueUsage.getUsed() +
" currentCapacity: " + currentCapacity +
" activeUsers: " + activeUsers +
" clusterCapacity: " + clusterResource
@@ -1237,7 +1228,7 @@ public class LeafQueue extends AbstractCSQueue {
// overhead of the AM, but it's a > check, not a >= check, so...
if (Resources
.greaterThan(resourceCalculator, clusterResource,
- user.getConsumedResourceByLabel(label),
+ user.getUsed(label),
limit)) {
// if enabled, check to see if could we potentially use this node instead
// of a reserved node if the application has reserved containers
@@ -1245,13 +1236,13 @@ public class LeafQueue extends AbstractCSQueue {
if (Resources.lessThanOrEqual(
resourceCalculator,
clusterResource,
- Resources.subtract(user.getTotalConsumedResources(),
+ Resources.subtract(user.getUsed(),
application.getCurrentReservation()), limit)) {
if (LOG.isDebugEnabled()) {
LOG.debug("User " + userName + " in queue " + getQueueName()
+ " will exceed limit based on reservations - " + " consumed: "
- + user.getTotalConsumedResources() + " reserved: "
+ + user.getUsed() + " reserved: "
+ application.getCurrentReservation() + " limit: " + limit);
}
return true;
@@ -1260,7 +1251,7 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
LOG.debug("User " + userName + " in queue " + getQueueName()
+ " will exceed limit - " + " consumed: "
- + user.getTotalConsumedResources() + " limit: " + limit);
+ + user.getUsed() + " limit: " + limit);
}
return false;
}
@@ -1682,7 +1673,7 @@ public class LeafQueue extends AbstractCSQueue {
" queue=" + this.toString() +
" usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
+ " used=" + queueUsage.getUsed() +
" cluster=" + clusterResource);
return request.getCapability();
@@ -1783,9 +1774,9 @@ public class LeafQueue extends AbstractCSQueue {
if (LOG.isDebugEnabled()) {
LOG.info(getQueueName() +
" user=" + userName +
- " used=" + usedResources + " numContainers=" + numContainers +
+ " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
" headroom = " + application.getHeadroom() +
- " user-resources=" + user.getTotalConsumedResources()
+ " user-resources=" + user.getUsed()
);
}
}
@@ -1801,8 +1792,8 @@ public class LeafQueue extends AbstractCSQueue {
metrics.setAvailableResourcesToUser(userName, application.getHeadroom());
LOG.info(getQueueName() +
- " used=" + usedResources + " numContainers=" + numContainers +
- " user=" + userName + " user-resources=" + user.getTotalConsumedResources());
+ " used=" + queueUsage.getUsed() + " numContainers=" + numContainers +
+ " user=" + userName + " user-resources=" + user.getUsed());
}
private void updateAbsoluteCapacityResource(Resource clusterResource) {
@@ -1844,22 +1835,20 @@ public class LeafQueue extends AbstractCSQueue {
@VisibleForTesting
public static class User {
- Resource consumed = Resources.createResource(0, 0);
- Resource consumedAMResources = Resources.createResource(0, 0);
- Map<String, Resource> consumedByLabel = new HashMap<String, Resource>();
+ ResourceUsage userResourceUsage = new ResourceUsage();
int pendingApplications = 0;
int activeApplications = 0;
- public Resource getTotalConsumedResources() {
- return consumed;
+ public ResourceUsage getResourceUsage() {
+ return userResourceUsage;
}
- public Resource getConsumedResourceByLabel(String label) {
- Resource r = consumedByLabel.get(label);
- if (null != r) {
- return r;
- }
- return Resources.none();
+ public Resource getUsed() {
+ return userResourceUsage.getUsed();
+ }
+
+ public Resource getUsed(String label) {
+ return userResourceUsage.getUsed(label);
}
public int getPendingApplications() {
@@ -1871,7 +1860,7 @@ public class LeafQueue extends AbstractCSQueue {
}
public Resource getConsumedAMResources() {
- return consumedAMResources;
+ return userResourceUsage.getAMUsed();
}
public int getTotalApplications() {
@@ -1896,47 +1885,26 @@ public class LeafQueue extends AbstractCSQueue {
}
}
- public synchronized void assignContainer(Resource resource,
+ public void assignContainer(Resource resource,
Set<String> nodeLabels) {
- Resources.addTo(consumed, resource);
-
if (nodeLabels == null || nodeLabels.isEmpty()) {
- if (!consumedByLabel.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- consumedByLabel.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.addTo(consumedByLabel.get(RMNodeLabelsManager.NO_LABEL),
- resource);
+ userResourceUsage.incUsed(resource);
} else {
for (String label : nodeLabels) {
- if (!consumedByLabel.containsKey(label)) {
- consumedByLabel.put(label, Resources.createResource(0));
- }
- Resources.addTo(consumedByLabel.get(label), resource);
+ userResourceUsage.incUsed(label, resource);
}
}
}
- public synchronized void releaseContainer(Resource resource, Set<String> nodeLabels) {
- Resources.subtractFrom(consumed, resource);
-
- // Update usedResources by labels
+ public void releaseContainer(Resource resource, Set<String> nodeLabels) {
if (nodeLabels == null || nodeLabels.isEmpty()) {
- if (!consumedByLabel.containsKey(RMNodeLabelsManager.NO_LABEL)) {
- consumedByLabel.put(RMNodeLabelsManager.NO_LABEL,
- Resources.createResource(0));
- }
- Resources.subtractFrom(
- consumedByLabel.get(RMNodeLabelsManager.NO_LABEL), resource);
+ userResourceUsage.decUsed(resource);
} else {
for (String label : nodeLabels) {
- if (!consumedByLabel.containsKey(label)) {
- consumedByLabel.put(label, Resources.createResource(0));
- }
- Resources.subtractFrom(consumedByLabel.get(label), resource);
+ userResourceUsage.decUsed(label, resource);
}
}
- }
+ }
}
@Override
@@ -1995,7 +1963,7 @@ public class LeafQueue extends AbstractCSQueue {
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
+ " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
- + usedResources + " cluster=" + clusterResource);
+ + queueUsage.getUsed() + " cluster=" + clusterResource);
// Inform the parent queue
getParent().attachContainer(clusterResource, application, rmContainer);
}
@@ -2013,7 +1981,7 @@ public class LeafQueue extends AbstractCSQueue {
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
+ " absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + " used="
- + usedResources + " cluster=" + clusterResource);
+ + queueUsage.getUsed() + " cluster=" + clusterResource);
// Inform the parent queue
getParent().detachContainer(clusterResource, application, rmContainer);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index f820cca..de92c9c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -256,7 +256,7 @@ public class ParentQueue extends AbstractCSQueue {
"numChildQueue= " + childQueues.size() + ", " +
"capacity=" + capacity + ", " +
"absoluteCapacity=" + absoluteCapacity + ", " +
- "usedResources=" + usedResources +
+ "usedResources=" + queueUsage.getUsed() +
"usedCapacity=" + getUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
"numContainers=" + getNumContainers();
@@ -463,7 +463,7 @@ public class ParentQueue extends AbstractCSQueue {
" queue=" + getQueueName() +
" usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
+ " used=" + queueUsage.getUsed() +
" cluster=" + clusterResource);
} else {
@@ -506,19 +506,16 @@ public class ParentQueue extends AbstractCSQueue {
boolean canAssign = true;
for (String label : labelCanAccess) {
- if (!usedResourcesByNodeLabels.containsKey(label)) {
- usedResourcesByNodeLabels.put(label, Resources.createResource(0));
- }
float currentAbsoluteLabelUsedCapacity =
Resources.divide(resourceCalculator, clusterResource,
- usedResourcesByNodeLabels.get(label),
+ queueUsage.getUsed(label),
labelManager.getResourceByLabel(label, clusterResource));
// if any of the label doesn't beyond limit, we can allocate on this node
if (currentAbsoluteLabelUsedCapacity >=
getAbsoluteMaximumCapacityByNodeLabel(label)) {
if (LOG.isDebugEnabled()) {
- LOG.debug(getQueueName() + " used=" + usedResources
- + " current-capacity (" + usedResourcesByNodeLabels.get(label) + ") "
+ LOG.debug(getQueueName() + " used=" + queueUsage.getUsed()
+ + " current-capacity (" + queueUsage.getUsed(label) + ") "
+ " >= max-capacity ("
+ labelManager.getResourceByLabel(label, clusterResource) + ")");
}
@@ -540,16 +537,16 @@ public class ParentQueue extends AbstractCSQueue {
.getReservedMB(), getMetrics().getReservedVirtualCores());
float capacityWithoutReservedCapacity = Resources.divide(
resourceCalculator, clusterResource,
- Resources.subtract(usedResources, reservedResources),
+ Resources.subtract(queueUsage.getUsed(), reservedResources),
clusterResource);
if (capacityWithoutReservedCapacity <= absoluteMaxCapacity) {
if (LOG.isDebugEnabled()) {
LOG.debug("parent: try to use reserved: " + getQueueName()
- + " usedResources: " + usedResources.getMemory()
+ + " usedResources: " + queueUsage.getUsed().getMemory()
+ " clusterResources: " + clusterResource.getMemory()
+ " reservedResources: " + reservedResources.getMemory()
- + " currentCapacity " + ((float) usedResources.getMemory())
+ + " currentCapacity " + ((float) queueUsage.getUsed().getMemory())
/ clusterResource.getMemory()
+ " potentialNewWithoutReservedCapacity: "
+ capacityWithoutReservedCapacity + " ( " + " max-capacity: "
@@ -645,7 +642,7 @@ public class ParentQueue extends AbstractCSQueue {
" queue=" + getQueueName() +
" usedCapacity=" + getUsedCapacity() +
" absoluteUsedCapacity=" + getAbsoluteUsedCapacity() +
- " used=" + usedResources +
+ " used=" + queueUsage.getUsed() +
" cluster=" + clusterResource);
}
@@ -735,7 +732,7 @@ public class ParentQueue extends AbstractCSQueue {
.getResource(), node.getLabels());
LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
- + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+ + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
+ clusterResource);
// Inform the parent
if (parent != null) {
@@ -755,7 +752,7 @@ public class ParentQueue extends AbstractCSQueue {
node.getLabels());
LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
- + getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
+ + getAbsoluteUsedCapacity() + " used=" + queueUsage.getUsed() + " cluster="
+ clusterResource);
// Inform the parent
if (parent != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
index 354a5ab..5c5c2df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
@@ -298,7 +298,7 @@ public class TestWorkPreservingRMRestart {
1e-8);
// assert user consumed resources.
assertEquals(usedResource, leafQueue.getUser(app.getUser())
- .getTotalConsumedResources());
+ .getUsed());
}
private void checkFifoQueue(SchedulerApplication schedulerApp,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d9281fbb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
index a62889b..d643c9d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
@@ -202,33 +202,33 @@ public class TestCSQueueUtils {
LOG.info("t2 l2q2 " + result);
//some usage, but below the base capacity
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
- Resources.addTo(l1q2.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.4f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//usage gt base on parent sibling
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.3f));
- Resources.addTo(l1q2.getUsedResources(), Resources.multiply(clusterResource, 0.3f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+ l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//same as last, but with usage also on direct parent
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
- Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.1f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//add to direct sibling, below the threshold of effect at present
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l2q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
@@ -236,9 +236,9 @@ public class TestCSQueueUtils {
//add to direct sibling, now above the threshold of effect
//(it's cumulative with prior tests)
- Resources.addTo(root.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l1q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
- Resources.addTo(l2q1.getUsedResources(), Resources.multiply(clusterResource, 0.2f));
+ root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.1f, result, 0.000001f);
[14/50] [abbrv] hadoop git commit: HADOOP-11934. Use of
JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop. Contributed
by Larry McCay.
Posted by sj...@apache.org.
HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes infinite loop. Contributed by Larry McCay.
(cherry picked from commit 860b8373c3a851386b8cd2d4265dd35e5aabc941)
(cherry picked from commit 33648268ce0f79bf51facafa3d151612e3d00ddb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c76f5b31
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c76f5b31
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c76f5b31
Branch: refs/heads/branch-2.6
Commit: c76f5b3188d7a78307d142c91198c8d106a7c431
Parents: 2336264
Author: cnauroth <cn...@apache.org>
Authored: Thu May 28 15:01:42 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Sep 7 11:09:20 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../alias/AbstractJavaKeyStoreProvider.java | 353 +++++++++++++++++++
.../security/alias/JavaKeyStoreProvider.java | 265 ++------------
.../alias/LocalJavaKeyStoreProvider.java | 192 ++++++++++
...oop.security.alias.CredentialProviderFactory | 1 +
.../alias/TestCredentialProviderFactory.java | 25 +-
6 files changed, 610 insertions(+), 229 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c76f5b31/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c4735a0..7f868be 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -75,6 +75,9 @@ Release 2.6.1 - UNRELEASED
HADOOP-11932. MetricsSinkAdapter may hang when being stopped.
(Brahma Reddy Battula via jianhe)
+ HADOOP-11934. Use of JavaKeyStoreProvider in LdapGroupsMapping causes
+ infinite loop. (Larry McCay via cnauroth)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c76f5b31/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
new file mode 100644
index 0000000..e18f7a7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/AbstractJavaKeyStoreProvider.java
@@ -0,0 +1,353 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.ProviderUtils;
+
+import com.google.common.base.Charsets;
+
+import javax.crypto.spec.SecretKeySpec;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URL;
+import java.security.KeyStore;
+import java.security.KeyStoreException;
+import java.security.NoSuchAlgorithmException;
+import java.security.UnrecoverableKeyException;
+import java.security.cert.CertificateException;
+import java.util.ArrayList;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+/**
+ * Abstract class for implementing credential providers that are based on
+ * Java Keystores as the underlying credential store.
+ *
+ * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
+ * environment variable with a default of 'none'.
+ *
+ * It is expected that for access to credential protected resource to copy the
+ * creds from the original provider into the job's Credentials object, which is
+ * accessed via the UserProvider. Therefore, these providers won't be directly
+ * used by MapReduce tasks.
+ */
+@InterfaceAudience.Private
+public abstract class AbstractJavaKeyStoreProvider extends CredentialProvider {
+ public static final String CREDENTIAL_PASSWORD_NAME =
+ "HADOOP_CREDSTORE_PASSWORD";
+ public static final String KEYSTORE_PASSWORD_FILE_KEY =
+ "hadoop.security.credstore.java-keystore-provider.password-file";
+ public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
+
+ private Path path;
+ private final URI uri;
+ private final KeyStore keyStore;
+ private char[] password = null;
+ private boolean changed = false;
+ private Lock readLock;
+ private Lock writeLock;
+
+ protected AbstractJavaKeyStoreProvider(URI uri, Configuration conf)
+ throws IOException {
+ this.uri = uri;
+ initFileSystem(uri, conf);
+ // Get the password from the user's environment
+ if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
+ password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
+ }
+ // if not in ENV get check for file
+ if (password == null) {
+ String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
+ if (pwFile != null) {
+ ClassLoader cl = Thread.currentThread().getContextClassLoader();
+ URL pwdFile = cl.getResource(pwFile);
+ if (pwdFile != null) {
+ InputStream is = pwdFile.openStream();
+ try {
+ password = IOUtils.toString(is).trim().toCharArray();
+ } finally {
+ is.close();
+ }
+ }
+ }
+ }
+ if (password == null) {
+ password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
+ }
+ try {
+ keyStore = KeyStore.getInstance("jceks");
+ if (keystoreExists()) {
+ stashOriginalFilePermissions();
+ InputStream in = getInputStreamForFile();
+ try {
+ keyStore.load(in, password);
+ } finally {
+ in.close();
+ }
+ } else {
+ createPermissions("700");
+ // required to create an empty keystore. *sigh*
+ keyStore.load(null, password);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't create keystore", e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("Can't load keystore " + getPathAsString(), e);
+ } catch (CertificateException e) {
+ throw new IOException("Can't load keystore " + getPathAsString(), e);
+ }
+ ReadWriteLock lock = new ReentrantReadWriteLock(true);
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+ }
+
+ public Path getPath() {
+ return path;
+ }
+
+ public void setPath(Path p) {
+ this.path = p;
+ }
+
+ public char[] getPassword() {
+ return password;
+ }
+
+ public void setPassword(char[] pass) {
+ this.password = pass;
+ }
+
+ public boolean isChanged() {
+ return changed;
+ }
+
+ public void setChanged(boolean chg) {
+ this.changed = chg;
+ }
+
+ public Lock getReadLock() {
+ return readLock;
+ }
+
+ public void setReadLock(Lock rl) {
+ this.readLock = rl;
+ }
+
+ public Lock getWriteLock() {
+ return writeLock;
+ }
+
+ public void setWriteLock(Lock wl) {
+ this.writeLock = wl;
+ }
+
+ public URI getUri() {
+ return uri;
+ }
+
+ public KeyStore getKeyStore() {
+ return keyStore;
+ }
+
+ public Map<String, CredentialEntry> getCache() {
+ return cache;
+ }
+
+ private final Map<String, CredentialEntry> cache =
+ new HashMap<String, CredentialEntry>();
+
+ protected final String getPathAsString() {
+ return getPath().toString();
+ }
+
+ protected abstract String getSchemeName();
+
+ protected abstract OutputStream getOutputStreamForKeystore()
+ throws IOException;
+
+ protected abstract boolean keystoreExists() throws IOException;
+
+ protected abstract InputStream getInputStreamForFile() throws IOException;
+
+ protected abstract void createPermissions(String perms) throws IOException;
+
+ protected abstract void stashOriginalFilePermissions() throws IOException;
+
+ protected void initFileSystem(URI keystoreUri, Configuration conf)
+ throws IOException {
+ path = ProviderUtils.unnestUri(keystoreUri);
+ }
+
+ @Override
+ public CredentialEntry getCredentialEntry(String alias)
+ throws IOException {
+ readLock.lock();
+ try {
+ SecretKeySpec key = null;
+ try {
+ if (cache.containsKey(alias)) {
+ return cache.get(alias);
+ }
+ if (!keyStore.containsAlias(alias)) {
+ return null;
+ }
+ key = (SecretKeySpec) keyStore.getKey(alias, password);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't get credential " + alias + " from "
+ + getPathAsString(), e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("Can't get algorithm for credential " + alias
+ + " from " + getPathAsString(), e);
+ } catch (UnrecoverableKeyException e) {
+ throw new IOException("Can't recover credential " + alias + " from "
+ + getPathAsString(), e);
+ }
+ return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ public static char[] bytesToChars(byte[] bytes) throws IOException {
+ String pass;
+ pass = new String(bytes, Charsets.UTF_8);
+ return pass.toCharArray();
+ }
+
+ @Override
+ public List<String> getAliases() throws IOException {
+ readLock.lock();
+ try {
+ ArrayList<String> list = new ArrayList<String>();
+ String alias = null;
+ try {
+ Enumeration<String> e = keyStore.aliases();
+ while (e.hasMoreElements()) {
+ alias = e.nextElement();
+ list.add(alias);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't get alias " + alias + " from "
+ + getPathAsString(), e);
+ }
+ return list;
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public CredentialEntry createCredentialEntry(String alias, char[] credential)
+ throws IOException {
+ writeLock.lock();
+ try {
+ if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
+ throw new IOException("Credential " + alias + " already exists in "
+ + this);
+ }
+ return innerSetCredential(alias, credential);
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem looking up credential " + alias + " in "
+ + this, e);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public void deleteCredentialEntry(String name) throws IOException {
+ writeLock.lock();
+ try {
+ try {
+ if (keyStore.containsAlias(name)) {
+ keyStore.deleteEntry(name);
+ } else {
+ throw new IOException("Credential " + name + " does not exist in "
+ + this);
+ }
+ } catch (KeyStoreException e) {
+ throw new IOException("Problem removing " + name + " from " + this, e);
+ }
+ cache.remove(name);
+ changed = true;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ CredentialEntry innerSetCredential(String alias, char[] material)
+ throws IOException {
+ writeLock.lock();
+ try {
+ keyStore.setKeyEntry(alias,
+ new SecretKeySpec(new String(material).getBytes("UTF-8"), "AES"),
+ password, null);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't store credential " + alias + " in " + this,
+ e);
+ } finally {
+ writeLock.unlock();
+ }
+ changed = true;
+ return new CredentialEntry(alias, material);
+ }
+
+ @Override
+ public void flush() throws IOException {
+ writeLock.lock();
+ try {
+ if (!changed) {
+ return;
+ }
+ // write out the keystore
+ OutputStream out = getOutputStreamForKeystore();
+ try {
+ keyStore.store(out, password);
+ } catch (KeyStoreException e) {
+ throw new IOException("Can't store keystore " + this, e);
+ } catch (NoSuchAlgorithmException e) {
+ throw new IOException("No such algorithm storing keystore " + this, e);
+ } catch (CertificateException e) {
+ throw new IOException("Certificate exception storing keystore " + this,
+ e);
+ } finally {
+ out.close();
+ }
+ changed = false;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ return uri.toString();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c76f5b31/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
index 5dc2abf..ff73a23 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/JavaKeyStoreProvider.java
@@ -18,266 +18,75 @@
package org.apache.hadoop.security.alias;
-import org.apache.commons.io.IOUtils;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.security.ProviderUtils;
-
-import javax.crypto.spec.SecretKeySpec;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URI;
-import java.net.URL;
-import java.security.KeyStore;
-import java.security.KeyStoreException;
-import java.security.NoSuchAlgorithmException;
-import java.security.UnrecoverableKeyException;
-import java.security.cert.CertificateException;
-import java.util.ArrayList;
-import java.util.Enumeration;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.concurrent.locks.Lock;
-import java.util.concurrent.locks.ReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
/**
- * CredentialProvider based on Java's KeyStore file format. The file may be
+ * CredentialProvider based on Java's KeyStore file format. The file may be
* stored in any Hadoop FileSystem using the following name mangling:
- * jceks://hdfs@nn1.example.com/my/creds.jceks -> hdfs://nn1.example.com/my/creds.jceks
- * jceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
- *
- * The password for the keystore is taken from the HADOOP_CREDSTORE_PASSWORD
- * environment variable with a default of 'none'.
- *
- * It is expected that for access to credential protected resource to copy the
- * creds from the original provider into the job's Credentials object, which is
- * accessed via the UserProvider. Therefore, this provider won't be directly
- * used by MapReduce tasks.
+ * jceks://hdfs@nn1.example.com/my/creds.jceks ->
+ * hdfs://nn1.example.com/my/creds.jceks jceks://file/home/larry/creds.jceks ->
+ * file:///home/larry/creds.jceks
*/
@InterfaceAudience.Private
-public class JavaKeyStoreProvider extends CredentialProvider {
+public class JavaKeyStoreProvider extends AbstractJavaKeyStoreProvider {
public static final String SCHEME_NAME = "jceks";
- public static final String CREDENTIAL_PASSWORD_NAME =
- "HADOOP_CREDSTORE_PASSWORD";
- public static final String KEYSTORE_PASSWORD_FILE_KEY =
- "hadoop.security.credstore.java-keystore-provider.password-file";
- public static final String KEYSTORE_PASSWORD_DEFAULT = "none";
- private final URI uri;
- private final Path path;
- private final FileSystem fs;
- private final FsPermission permissions;
- private final KeyStore keyStore;
- private char[] password = null;
- private boolean changed = false;
- private Lock readLock;
- private Lock writeLock;
+ private FileSystem fs;
+ private FsPermission permissions;
- private final Map<String, CredentialEntry> cache = new HashMap<String, CredentialEntry>();
-
- private JavaKeyStoreProvider(URI uri, Configuration conf) throws IOException {
- this.uri = uri;
- path = ProviderUtils.unnestUri(uri);
- fs = path.getFileSystem(conf);
- // Get the password from the user's environment
- if (System.getenv().containsKey(CREDENTIAL_PASSWORD_NAME)) {
- password = System.getenv(CREDENTIAL_PASSWORD_NAME).toCharArray();
- }
- // if not in ENV get check for file
- if (password == null) {
- String pwFile = conf.get(KEYSTORE_PASSWORD_FILE_KEY);
- if (pwFile != null) {
- ClassLoader cl = Thread.currentThread().getContextClassLoader();
- URL pwdFile = cl.getResource(pwFile);
- if (pwdFile != null) {
- InputStream is = pwdFile.openStream();
- try {
- password = IOUtils.toString(is).trim().toCharArray();
- } finally {
- is.close();
- }
- }
- }
- }
- if (password == null) {
- password = KEYSTORE_PASSWORD_DEFAULT.toCharArray();
- }
- try {
- keyStore = KeyStore.getInstance(SCHEME_NAME);
- if (fs.exists(path)) {
- // save off permissions in case we need to
- // rewrite the keystore in flush()
- FileStatus s = fs.getFileStatus(path);
- permissions = s.getPermission();
-
- keyStore.load(fs.open(path), password);
- } else {
- permissions = new FsPermission("700");
- // required to create an empty keystore. *sigh*
- keyStore.load(null, password);
- }
- } catch (KeyStoreException e) {
- throw new IOException("Can't create keystore", e);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException("Can't load keystore " + path, e);
- } catch (CertificateException e) {
- throw new IOException("Can't load keystore " + path, e);
- }
- ReadWriteLock lock = new ReentrantReadWriteLock(true);
- readLock = lock.readLock();
- writeLock = lock.writeLock();
+ private JavaKeyStoreProvider(URI uri, Configuration conf)
+ throws IOException {
+ super(uri, conf);
}
@Override
- public CredentialEntry getCredentialEntry(String alias) throws IOException {
- readLock.lock();
- try {
- SecretKeySpec key = null;
- try {
- if (cache.containsKey(alias)) {
- return cache.get(alias);
- }
- if (!keyStore.containsAlias(alias)) {
- return null;
- }
- key = (SecretKeySpec) keyStore.getKey(alias, password);
- } catch (KeyStoreException e) {
- throw new IOException("Can't get credential " + alias + " from " +
- path, e);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException("Can't get algorithm for credential " + alias + " from " +
- path, e);
- } catch (UnrecoverableKeyException e) {
- throw new IOException("Can't recover credential " + alias + " from " + path, e);
- }
- return new CredentialEntry(alias, bytesToChars(key.getEncoded()));
- }
- finally {
- readLock.unlock();
- }
- }
-
- public static char[] bytesToChars(byte[] bytes) {
- String pass = new String(bytes);
- return pass.toCharArray();
+ protected String getSchemeName() {
+ return SCHEME_NAME;
}
@Override
- public List<String> getAliases() throws IOException {
- readLock.lock();
- try {
- ArrayList<String> list = new ArrayList<String>();
- String alias = null;
- try {
- Enumeration<String> e = keyStore.aliases();
- while (e.hasMoreElements()) {
- alias = e.nextElement();
- list.add(alias);
- }
- } catch (KeyStoreException e) {
- throw new IOException("Can't get alias " + alias + " from " + path, e);
- }
- return list;
- }
- finally {
- readLock.unlock();
- }
+ protected OutputStream getOutputStreamForKeystore() throws IOException {
+ FSDataOutputStream out = FileSystem.create(fs, getPath(), permissions);
+ return out;
}
@Override
- public CredentialEntry createCredentialEntry(String alias, char[] credential)
- throws IOException {
- writeLock.lock();
- try {
- if (keyStore.containsAlias(alias) || cache.containsKey(alias)) {
- throw new IOException("Credential " + alias + " already exists in " + this);
- }
- return innerSetCredential(alias, credential);
- } catch (KeyStoreException e) {
- throw new IOException("Problem looking up credential " + alias + " in " + this,
- e);
- } finally {
- writeLock.unlock();
- }
+ protected boolean keystoreExists() throws IOException {
+ return fs.exists(getPath());
}
@Override
- public void deleteCredentialEntry(String name) throws IOException {
- writeLock.lock();
- try {
- try {
- if (keyStore.containsAlias(name)) {
- keyStore.deleteEntry(name);
- }
- else {
- throw new IOException("Credential " + name + " does not exist in " + this);
- }
- } catch (KeyStoreException e) {
- throw new IOException("Problem removing " + name + " from " +
- this, e);
- }
- cache.remove(name);
- changed = true;
- }
- finally {
- writeLock.unlock();
- }
+ protected InputStream getInputStreamForFile() throws IOException {
+ return fs.open(getPath());
}
- CredentialEntry innerSetCredential(String alias, char[] material)
- throws IOException {
- writeLock.lock();
- try {
- keyStore.setKeyEntry(alias, new SecretKeySpec(
- new String(material).getBytes("UTF-8"), "AES"),
- password, null);
- } catch (KeyStoreException e) {
- throw new IOException("Can't store credential " + alias + " in " + this,
- e);
- } finally {
- writeLock.unlock();
- }
- changed = true;
- return new CredentialEntry(alias, material);
+ @Override
+ protected void createPermissions(String perms) {
+ permissions = new FsPermission(perms);
}
@Override
- public void flush() throws IOException {
- writeLock.lock();
- try {
- if (!changed) {
- return;
- }
- // write out the keystore
- FSDataOutputStream out = FileSystem.create(fs, path, permissions);
- try {
- keyStore.store(out, password);
- } catch (KeyStoreException e) {
- throw new IOException("Can't store keystore " + this, e);
- } catch (NoSuchAlgorithmException e) {
- throw new IOException("No such algorithm storing keystore " + this, e);
- } catch (CertificateException e) {
- throw new IOException("Certificate exception storing keystore " + this,
- e);
- }
- out.close();
- changed = false;
- }
- finally {
- writeLock.unlock();
- }
+ protected void stashOriginalFilePermissions() throws IOException {
+ // save off permissions in case we need to
+ // rewrite the keystore in flush()
+ FileStatus s = fs.getFileStatus(getPath());
+ permissions = s.getPermission();
}
- @Override
- public String toString() {
- return uri.toString();
+ protected void initFileSystem(URI uri, Configuration conf)
+ throws IOException {
+ super.initFileSystem(uri, conf);
+ fs = getPath().getFileSystem(conf);
}
/**
@@ -286,7 +95,7 @@ public class JavaKeyStoreProvider extends CredentialProvider {
public static class Factory extends CredentialProviderFactory {
@Override
public CredentialProvider createProvider(URI providerName,
- Configuration conf) throws IOException {
+ Configuration conf) throws IOException {
if (SCHEME_NAME.equals(providerName.getScheme())) {
return new JavaKeyStoreProvider(providerName, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c76f5b31/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
new file mode 100644
index 0000000..3840979
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/LocalJavaKeyStoreProvider.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.security.alias;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileUtil;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.util.Set;
+import java.util.StringTokenizer;
+import java.util.EnumSet;
+
+/**
+ * CredentialProvider based on Java's KeyStore file format. The file may be
+ * stored only on the local filesystem using the following name mangling:
+ * localjceks://file/home/larry/creds.jceks -> file:///home/larry/creds.jceks
+ */
+@InterfaceAudience.Private
+public final class LocalJavaKeyStoreProvider extends
+ AbstractJavaKeyStoreProvider {
+ public static final String SCHEME_NAME = "localjceks";
+ private File file;
+ private Set<PosixFilePermission> permissions;
+
+ private LocalJavaKeyStoreProvider(URI uri, Configuration conf)
+ throws IOException {
+ super(uri, conf);
+ }
+
+ @Override
+ protected String getSchemeName() {
+ return SCHEME_NAME;
+ }
+
+ @Override
+ protected OutputStream getOutputStreamForKeystore() throws IOException {
+ FileOutputStream out = new FileOutputStream(file);
+ return out;
+ }
+
+ @Override
+ protected boolean keystoreExists() throws IOException {
+ return file.exists();
+ }
+
+ @Override
+ protected InputStream getInputStreamForFile() throws IOException {
+ FileInputStream is = new FileInputStream(file);
+ return is;
+ }
+
+ @Override
+ protected void createPermissions(String perms) throws IOException {
+ int mode = 700;
+ try {
+ mode = Integer.parseInt(perms, 8);
+ } catch (NumberFormatException nfe) {
+ throw new IOException("Invalid permissions mode provided while "
+ + "trying to createPermissions", nfe);
+ }
+ permissions = modeToPosixFilePermission(mode);
+ }
+
+ @Override
+ protected void stashOriginalFilePermissions() throws IOException {
+ // save off permissions in case we need to
+ // rewrite the keystore in flush()
+ if (!Shell.WINDOWS) {
+ Path path = Paths.get(file.getCanonicalPath());
+ permissions = Files.getPosixFilePermissions(path);
+ } else {
+ // On Windows, the JDK does not support the POSIX file permission APIs.
+ // Instead, we can do a winutils call and translate.
+ String[] cmd = Shell.getGetPermissionCommand();
+ String[] args = new String[cmd.length + 1];
+ System.arraycopy(cmd, 0, args, 0, cmd.length);
+ args[cmd.length] = file.getCanonicalPath();
+ String out = Shell.execCommand(args);
+ StringTokenizer t = new StringTokenizer(out, Shell.TOKEN_SEPARATOR_REGEX);
+ // The winutils output consists of 10 characters because of the leading
+ // directory indicator, i.e. "drwx------". The JDK parsing method expects
+ // a 9-character string, so remove the leading character.
+ String permString = t.nextToken().substring(1);
+ permissions = PosixFilePermissions.fromString(permString);
+ }
+ }
+
+ @Override
+ protected void initFileSystem(URI uri, Configuration conf)
+ throws IOException {
+ super.initFileSystem(uri, conf);
+ try {
+ file = new File(new URI(getPath().toString()));
+ } catch (URISyntaxException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void flush() throws IOException {
+ super.flush();
+ if (!Shell.WINDOWS) {
+ Files.setPosixFilePermissions(Paths.get(file.getCanonicalPath()),
+ permissions);
+ } else {
+ // FsPermission expects a 10-character string because of the leading
+ // directory indicator, i.e. "drwx------". The JDK toString method returns
+ // a 9-character string, so prepend a leading character.
+ FsPermission fsPermission = FsPermission.valueOf(
+ "-" + PosixFilePermissions.toString(permissions));
+ FileUtil.setPermission(file, fsPermission);
+ }
+ }
+
+ /**
+ * The factory to create JksProviders, which is used by the ServiceLoader.
+ */
+ public static class Factory extends CredentialProviderFactory {
+ @Override
+ public CredentialProvider createProvider(URI providerName,
+ Configuration conf) throws IOException {
+ if (SCHEME_NAME.equals(providerName.getScheme())) {
+ return new LocalJavaKeyStoreProvider(providerName, conf);
+ }
+ return null;
+ }
+ }
+
+ private static Set<PosixFilePermission> modeToPosixFilePermission(
+ int mode) {
+ Set<PosixFilePermission> perms = EnumSet.noneOf(PosixFilePermission.class);
+ if ((mode & 0001) != 0) {
+ perms.add(PosixFilePermission.OTHERS_EXECUTE);
+ }
+ if ((mode & 0002) != 0) {
+ perms.add(PosixFilePermission.OTHERS_WRITE);
+ }
+ if ((mode & 0004) != 0) {
+ perms.add(PosixFilePermission.OTHERS_READ);
+ }
+ if ((mode & 0010) != 0) {
+ perms.add(PosixFilePermission.GROUP_EXECUTE);
+ }
+ if ((mode & 0020) != 0) {
+ perms.add(PosixFilePermission.GROUP_WRITE);
+ }
+ if ((mode & 0040) != 0) {
+ perms.add(PosixFilePermission.GROUP_READ);
+ }
+ if ((mode & 0100) != 0) {
+ perms.add(PosixFilePermission.OWNER_EXECUTE);
+ }
+ if ((mode & 0200) != 0) {
+ perms.add(PosixFilePermission.OWNER_WRITE);
+ }
+ if ((mode & 0400) != 0) {
+ perms.add(PosixFilePermission.OWNER_READ);
+ }
+ return perms;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c76f5b31/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
index bb7366e..f673cf4 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
+++ b/hadoop-common-project/hadoop-common/src/main/resources/META-INF/services/org.apache.hadoop.security.alias.CredentialProviderFactory
@@ -14,4 +14,5 @@
# limitations under the License.
org.apache.hadoop.security.alias.JavaKeyStoreProvider$Factory
+org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider$Factory
org.apache.hadoop.security.alias.UserProvider$Factory
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c76f5b31/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
index 1eb1e8b..16cb0be 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredentialProviderFactory.java
@@ -207,7 +207,29 @@ public class TestCredentialProviderFactory {
checkPermissionRetention(conf, ourUrl, path);
}
- public void checkPermissionRetention(Configuration conf, String ourUrl,
+ @Test
+ public void testLocalJksProvider() throws Exception {
+ Configuration conf = new Configuration();
+ final Path jksPath = new Path(tmpDir.toString(), "test.jks");
+ final String ourUrl =
+ LocalJavaKeyStoreProvider.SCHEME_NAME + "://file" + jksPath.toUri();
+
+ File file = new File(tmpDir, "test.jks");
+ file.delete();
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+ checkSpecificProvider(conf, ourUrl);
+ Path path = ProviderUtils.unnestUri(new URI(ourUrl));
+ FileSystem fs = path.getFileSystem(conf);
+ FileStatus s = fs.getFileStatus(path);
+ assertTrue("Unexpected permissions: " + s.getPermission().toString(), s.getPermission().toString().equals("rwx------"));
+ assertTrue(file + " should exist", file.isFile());
+
+ // check permission retention after explicit change
+ fs.setPermission(path, new FsPermission("777"));
+ checkPermissionRetention(conf, ourUrl, path);
+ }
+
+ public void checkPermissionRetention(Configuration conf, String ourUrl,
Path path) throws Exception {
CredentialProvider provider = CredentialProviderFactory.getProviders(conf).get(0);
// let's add a new credential and flush and check that permissions are still set to 777
@@ -233,3 +255,4 @@ public class TestCredentialProviderFactory {
"keystore.", s.getPermission().toString().equals("rwxrwxrwx"));
}
}
+
[02/50] [abbrv] hadoop git commit: YARN-2637. Fixed
max-am-resource-percent calculation in CapacityScheduler when activating
applications. Contributed by Craig Welch (cherry picked from commit
c53420f58364b11fbda1dace7679d45534533382)
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index b4c4c10..3918bf7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -143,13 +143,14 @@ public class TestFifoScheduler {
@Test(timeout=5000)
public void testAppAttemptMetrics() throws Exception {
AsyncDispatcher dispatcher = new InlineDispatcher();
+
+ FifoScheduler scheduler = new FifoScheduler();
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
RMContext rmContext = new RMContextImpl(dispatcher, null,
- null, null, null, null, null, null, null, writer);
+ null, null, null, null, null, null, null, writer, scheduler);
((RMContextImpl) rmContext).setSystemMetricsPublisher(
mock(SystemMetricsPublisher.class));
- FifoScheduler scheduler = new FifoScheduler();
Configuration conf = new Configuration();
scheduler.setRMContext(rmContext);
scheduler.init(conf);
@@ -189,12 +190,14 @@ public class TestFifoScheduler {
new NMTokenSecretManagerInRM(conf);
nmTokenSecretManager.rollMasterKey();
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
+
+ FifoScheduler scheduler = new FifoScheduler();
RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, containerTokenSecretManager, nmTokenSecretManager, null, writer);
+ null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
+ scheduler);
((RMContextImpl) rmContext).setSystemMetricsPublisher(
mock(SystemMetricsPublisher.class));
- FifoScheduler scheduler = new FifoScheduler();
scheduler.setRMContext(rmContext);
scheduler.init(conf);
scheduler.start();
@@ -260,17 +263,19 @@ public class TestFifoScheduler {
new NMTokenSecretManagerInRM(conf);
nmTokenSecretManager.rollMasterKey();
RMApplicationHistoryWriter writer = mock(RMApplicationHistoryWriter.class);
- RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
- null, containerTokenSecretManager, nmTokenSecretManager, null, writer);
- ((RMContextImpl) rmContext).setSystemMetricsPublisher(
- mock(SystemMetricsPublisher.class));
-
+
FifoScheduler scheduler = new FifoScheduler(){
@SuppressWarnings("unused")
public Map<NodeId, FiCaSchedulerNode> getNodes(){
return nodes;
}
};
+ RMContext rmContext = new RMContextImpl(dispatcher, null, null, null, null,
+ null, containerTokenSecretManager, nmTokenSecretManager, null, writer,
+ scheduler);
+ ((RMContextImpl) rmContext).setSystemMetricsPublisher(
+ mock(SystemMetricsPublisher.class));
+
scheduler.setRMContext(rmContext);
scheduler.init(conf);
scheduler.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index 87bacc6..d418dab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -82,8 +82,6 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
int numContainers;
int maxApplications;
int maxApplicationsPerUser;
- int maxActiveApplications;
- int maxActiveApplicationsPerUser;
int userLimit;
float userLimitFactor;
}
@@ -303,10 +301,6 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
WebServicesTestUtils.getXmlInt(qElem, "maxApplications");
lqi.maxApplicationsPerUser =
WebServicesTestUtils.getXmlInt(qElem, "maxApplicationsPerUser");
- lqi.maxActiveApplications =
- WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplications");
- lqi.maxActiveApplicationsPerUser =
- WebServicesTestUtils.getXmlInt(qElem, "maxActiveApplicationsPerUser");
lqi.userLimit = WebServicesTestUtils.getXmlInt(qElem, "userLimit");
lqi.userLimitFactor =
WebServicesTestUtils.getXmlFloat(qElem, "userLimitFactor");
@@ -386,8 +380,6 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
lqi.numContainers = info.getInt("numContainers");
lqi.maxApplications = info.getInt("maxApplications");
lqi.maxApplicationsPerUser = info.getInt("maxApplicationsPerUser");
- lqi.maxActiveApplications = info.getInt("maxActiveApplications");
- lqi.maxActiveApplicationsPerUser = info.getInt("maxActiveApplicationsPerUser");
lqi.userLimit = info.getInt("userLimit");
lqi.userLimitFactor = (float) info.getDouble("userLimitFactor");
verifyLeafQueueGeneric(q, lqi);
@@ -449,10 +441,6 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
(float)expectedMaxAppsPerUser,
(float)info.maxApplicationsPerUser, info.userLimitFactor);
- assertTrue("maxActiveApplications doesn't match",
- info.maxActiveApplications > 0);
- assertTrue("maxActiveApplicationsPerUser doesn't match",
- info.maxActiveApplicationsPerUser > 0);
assertEquals("userLimit doesn't match", csConf.getUserLimit(q),
info.userLimit);
assertEquals("userLimitFactor doesn't match",
[21/50] [abbrv] hadoop git commit: YARN-2301. Improved yarn container
command. Contributed by Naganarasimha G R (cherry picked from commit
258623ff8bb1a1057ae3501d4f20982d5a59ea34)
Posted by sj...@apache.org.
YARN-2301. Improved yarn container command. Contributed by Naganarasimha G R
(cherry picked from commit 258623ff8bb1a1057ae3501d4f20982d5a59ea34)
(cherry picked from commit 1d1e7682c9cad6a2f819b390ca3368dfa29c7097)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/92742b44
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/92742b44
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/92742b44
Branch: refs/heads/branch-2.6
Commit: 92742b4402906264b519f08c4c9355ed2f46a8cc
Parents: c3928b3
Author: Jian He <ji...@apache.org>
Authored: Thu Dec 4 12:51:15 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 22:57:28 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 8 +++-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 41 +++++++++++++-------
.../yarn/server/resourcemanager/RMContext.java | 3 ++
.../server/resourcemanager/RMContextImpl.java | 11 ++++++
.../server/resourcemanager/ResourceManager.java | 2 +
.../rmcontainer/RMContainerImpl.java | 9 ++++-
.../resourcemanager/TestClientRMService.java | 1 +
.../rmcontainer/TestRMContainerImpl.java | 6 ++-
9 files changed, 63 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1ad0199..f2b8c07 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -22,6 +22,8 @@ Release 2.6.1 - UNRELEASED
YARN-3098. Created common QueueCapacities class in Capacity Scheduler to
track capacities-by-labels of queues. (Wangda Tan via jianhe)
+ YARN-2301. Improved yarn container command. (Naganarasimha G R via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index a847cd5..83d212d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
@@ -536,8 +537,11 @@ public class ApplicationCLI extends YarnCLI {
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
"Finish Time", "State", "Host", "LOG-URL");
for (ContainerReport containerReport : appsReport) {
- writer.printf(CONTAINER_PATTERN, containerReport.getContainerId(),
- containerReport.getCreationTime(), containerReport.getFinishTime(),
+ writer.printf(
+ CONTAINER_PATTERN,
+ containerReport.getContainerId(),
+ Times.format(containerReport.getCreationTime()),
+ Times.format(containerReport.getFinishTime()),
containerReport.getContainerState(), containerReport
.getAssignedNode(), containerReport.getLogUrl());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 9d9a86a..b4df551 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -32,19 +32,17 @@ import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
import java.io.PrintWriter;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
-import java.util.Map;
import java.util.Set;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
-import org.apache.hadoop.yarn.api.protocolrecords.GetClusterNodeLabelsResponse;
-import org.apache.hadoop.yarn.api.protocolrecords.GetNodesToLabelsResponse;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -65,9 +63,8 @@ import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
-import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.util.Records;
-import org.jboss.netty.logging.CommonsLoggerFactory;
+import org.apache.hadoop.yarn.util.Times;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -257,25 +254,30 @@ public class TestYarnCLI {
applicationId, 1);
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
ContainerId containerId1 = ContainerId.newContainerId(attemptId, 2);
+ ContainerId containerId2 = ContainerId.newContainerId(attemptId, 3);
+ long time1=1234,time2=5678;
ContainerReport container = ContainerReport.newInstance(containerId, null,
- NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
- NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
"diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ ContainerReport container2 = ContainerReport.newInstance(containerId2, null,
+ NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0,
+ "diagnosticInfo", "", 0, ContainerState.RUNNING);
List<ContainerReport> reports = new ArrayList<ContainerReport>();
reports.add(container);
reports.add(container1);
+ reports.add(container2);
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
int result = cli.run(new String[] { "container", "-list",
attemptId.toString() });
assertEquals(0, result);
verify(client).getContainers(attemptId);
- Log.info(sysOutStream.toString());
ByteArrayOutputStream baos = new ByteArrayOutputStream();
PrintWriter pw = new PrintWriter(baos);
- pw.println("Total number of containers :2");
+ pw.println("Total number of containers :3");
pw.print(" Container-Id");
pw.print("\t Start Time");
pw.print("\t Finish Time");
@@ -283,19 +285,30 @@ public class TestYarnCLI {
pw.print("\t Host");
pw.println("\t LOG-URL");
pw.print(" container_1234_0005_01_000001");
- pw.print("\t 1234");
- pw.print("\t 5678");
+ pw.print("\t"+Times.format(time1));
+ pw.print("\t"+Times.format(time2));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000002");
- pw.print("\t 1234");
- pw.print("\t 5678");
+ pw.print("\t"+Times.format(time1));
+ pw.print("\t"+Times.format(time2));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
pw.println("\t logURL");
+ pw.print(" container_1234_0005_01_000003");
+ pw.print("\t"+Times.format(time1));
+ pw.print("\t N/A");
+ pw.print("\t RUNNING");
+ pw.print("\t host:1234");
+ pw.println("\t ");
pw.close();
String appReportStr = baos.toString("UTF-8");
+ Log.info("ExpectedOutput");
+ Log.info("["+appReportStr+"]");
+ Log.info("OutputFrom command");
+ String actualOutput = sysOutStream.toString();
+ Log.info("["+actualOutput+"]");
Assert.assertEquals(appReportStr, sysOutStream.toString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
index 56984e6..ecf6166 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.NodeId;
@@ -121,4 +122,6 @@ public interface RMContext {
ReservationSystem getReservationSystem();
boolean isSchedulerReadyForAllocatingContainers();
+
+ Configuration getYarnConfiguration();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 32216e5..1d0d6c0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -22,6 +22,7 @@ import java.nio.ByteBuffer;
import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
@@ -65,6 +66,8 @@ public class RMContextImpl implements RMContext {
private RMActiveServiceContext activeServiceContext;
+ private Configuration yarnConfiguration;
+
/**
* Default constructor. To be used in conjunction with setter methods for
* individual fields.
@@ -427,4 +430,12 @@ public class RMContextImpl implements RMContext {
this.activeServiceContext = activeServiceContext;
}
+ @Override
+ public Configuration getYarnConfiguration() {
+ return this.yarnConfiguration;
+ }
+
+ public void setYarnConfiguration(Configuration yarnConfiguration) {
+ this.yarnConfiguration=yarnConfiguration;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 353851c..711acf2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -252,6 +252,8 @@ public class ResourceManager extends CompositeService implements Recoverable {
addService(adminService);
rmContext.setRMAdminService(adminService);
+ rmContext.setYarnConfiguration(conf);
+
createAndInitActiveServices();
webAppAddress = WebAppUtils.getWebAppBindURL(this.conf,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 479734a..e37d8fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -285,8 +285,13 @@ public class RMContainerImpl implements RMContainer {
public String getLogURL() {
try {
readLock.lock();
- return WebAppUtils.getRunningLogURL("//" + container.getNodeHttpAddress(),
- ConverterUtils.toString(containerId), user);
+ StringBuilder logURL = new StringBuilder();
+ logURL.append(WebAppUtils.getHttpSchemePrefix(rmContext
+ .getYarnConfiguration()));
+ logURL.append(WebAppUtils.getRunningLogURL(
+ container.getNodeHttpAddress(), ConverterUtils.toString(containerId),
+ user));
+ return logURL.toString();
} finally {
readLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index 15aca42..a344e9a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -443,6 +443,7 @@ public class TestClientRMService {
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
yarnScheduler);
when(rmContext.getRMApps()).thenReturn(apps);
+ when(rmContext.getYarnConfiguration()).thenReturn(new Configuration());
RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null,
mock(ApplicationACLsManager.class), new Configuration());
when(rmContext.getDispatcher().getEventHandler()).thenReturn(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/92742b44/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 76cdcae..21aba3b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -107,6 +107,7 @@ public class TestRMContainerImpl {
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getRMApps()).thenReturn(rmApps);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@@ -130,7 +131,7 @@ public class TestRMContainerImpl {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
- assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify RELEASED and associated actions.
@@ -196,6 +197,7 @@ public class TestRMContainerImpl {
when(rmContext.getContainerAllocationExpirer()).thenReturn(expirer);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
RMContainer rmContainer = new RMContainerImpl(container, appAttemptId,
nodeId, "user", rmContext);
@@ -220,7 +222,7 @@ public class TestRMContainerImpl {
RMContainerEventType.LAUNCHED));
drainDispatcher.await();
assertEquals(RMContainerState.RUNNING, rmContainer.getState());
- assertEquals("//host:3465/node/containerlogs/container_1_0001_01_000001/user",
+ assertEquals("http://host:3465/node/containerlogs/container_1_0001_01_000001/user",
rmContainer.getLogURL());
// In RUNNING state. Verify EXPIRE and associated actions.
[20/50] [abbrv] hadoop git commit: HDFS-7570. SecondaryNameNode need
twice memory when calling reloadFromImageFile. Contributed by zhaoyunjiong.
Posted by sj...@apache.org.
HDFS-7570. SecondaryNameNode need twice memory when calling reloadFromImageFile. Contributed by zhaoyunjiong.
(cherry picked from commit 85aec75ce53445e1abf840076d2e10f1e3c6d69b)
(cherry picked from commit 27eecaef10feb49c4160ad289fd301bba6608973)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3928b30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3928b30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3928b30
Branch: refs/heads/branch-2.6
Commit: c3928b3068ab117f92d22ef7e4b99ea36a546e49
Parents: 3353707
Author: cnauroth <cn...@apache.org>
Authored: Tue Jan 13 16:10:28 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 17:21:58 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/blockmanagement/BlockManager.java | 5 +++++
.../apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java | 6 +++++-
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 1 +
4 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3928b30/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eaaea5d..9bc7974 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -169,6 +169,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8846. Add a unit test for INotify functionality across a layout
version upgrade (Zhe Zhang via Colin P. McCabe)
+ HDFS-7470. SecondaryNameNode need twice memory when calling
+ reloadFromImageFile. (zhaoyunjiong via cnauroth)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3928b30/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 846bc92..2f9d29b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -3811,4 +3811,9 @@ public class BlockManager {
stopReplicationInitializer();
blocksMap.close();
}
+
+ public void clear() {
+ clearQueues();
+ blocksMap.clear();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3928b30/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
index a675635..fafdc46 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlocksMap.java
@@ -85,9 +85,13 @@ class BlocksMap {
void close() {
+ clear();
+ blocks = null;
+ }
+
+ void clear() {
if (blocks != null) {
blocks.clear();
- blocks = null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3928b30/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 362b147..bd4f555 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -651,6 +651,7 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
snapshotManager.clearSnapshottableDirs();
cacheManager.clear();
setImageLoaded(false);
+ blockManager.clear();
}
@VisibleForTesting
[34/50] [abbrv] hadoop git commit: YARN-3740. Fixed the typo in the
configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan
Gong.
Posted by sj...@apache.org.
YARN-3740. Fixed the typo in the configuration name: APPLICATION_HISTORY_PREFIX_MAX_APPS. Contributed by Xuan Gong.
(cherry picked from commit eb6bf91eeacf97afb4cefe590f75ba94f3187d2b)
(cherry picked from commit 68cddb894a95a1ee63291af162849a23cb4eeb5f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ab820e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ab820e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ab820e6
Branch: refs/heads/branch-2.6
Commit: 3ab820e696ef9493bf8c7e2c302e4cb8bf3bf72d
Parents: 85ec6eb
Author: Zhijie Shen <zj...@apache.org>
Authored: Fri May 29 18:16:48 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Fri Sep 11 11:53:51 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java | 4 ++--
.../ApplicationHistoryManagerOnTimelineStore.java | 4 ++--
.../TestApplicationHistoryClientService.java | 2 +-
4 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cab1f0c..6641bb2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -211,6 +211,9 @@ Release 2.6.1 - 2015-09-09
YARN-3171. Sort by Application id, AppAttempt and ContainerID doesn't work
in ATS / RM web ui. (Naganarasimha G R via xgong)
+ YARN-3740. Fixed the typo in the configuration name:
+ APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 84322c3..881ecb5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1304,9 +1304,9 @@ public class YarnConfiguration extends Configuration {
* REST API or application history protocol and shown in timeline
* server web ui.
*/
- public static final String APPLICATION_HISTORY_PREFIX_MAX_APPS =
+ public static final String APPLICATION_HISTORY_MAX_APPS =
APPLICATION_HISTORY_PREFIX + "max-applications";
- public static final long DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS = 10000;
+ public static final long DEFAULT_APPLICATION_HISTORY_MAX_APPS = 10000;
/** Timeline service store class */
public static final String TIMELINE_SERVICE_STORE =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 4a407c3..b8f98c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -89,8 +89,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
serverHttpAddress = WebAppUtils.getHttpSchemePrefix(conf) +
WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
maxLoadedApplications =
- conf.getLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS,
- YarnConfiguration.DEFAULT_APPLICATION_HISTORY_PREFIX_MAX_APPS);
+ conf.getLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS,
+ YarnConfiguration.DEFAULT_APPLICATION_HISTORY_MAX_APPS);
super.serviceInit(conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ab820e6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
index feab8c1..ae49a2d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryClientService.java
@@ -172,7 +172,7 @@ public class TestApplicationHistoryClientService {
// Create a historyManager, and set the max_apps can be loaded
// as 1.
Configuration conf = new YarnConfiguration();
- conf.setLong(YarnConfiguration.APPLICATION_HISTORY_PREFIX_MAX_APPS, 1);
+ conf.setLong(YarnConfiguration.APPLICATION_HISTORY_MAX_APPS, 1);
ApplicationHistoryManagerOnTimelineStore historyManager2 =
new ApplicationHistoryManagerOnTimelineStore(dataManager,
new ApplicationACLsManager(conf));
[38/50] [abbrv] hadoop git commit: Revert "YARN-2890. MiniYARNCluster
should start the timeline server based on the configuration. Contributed by
Mit Desai."
Posted by sj...@apache.org.
Revert "YARN-2890. MiniYARNCluster should start the timeline server based on the configuration. Contributed by Mit Desai."
This reverts commit 8a47d1aa559c380869861c070af9d1e7a8732607.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c4a6e12
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c4a6e12
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c4a6e12
Branch: refs/heads/branch-2.6
Commit: 9c4a6e127098bf0c337903d01d7ec902a7cc6084
Parents: d57c3f0
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Sep 15 11:50:47 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:15 2015 -0700
----------------------------------------------------------------------
.../jobhistory/TestJobHistoryEventHandler.java | 2 +-
.../mapred/TestMRTimelineEventHandling.java | 58 +-------------------
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 -
.../distributedshell/TestDistributedShell.java | 2 +-
.../hadoop/yarn/client/ProtocolHATestBase.java | 6 +-
.../hadoop/yarn/server/MiniYARNCluster.java | 24 +++++++-
7 files changed, 34 insertions(+), 67 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
index de260c9..43e3dbe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
@@ -453,7 +453,7 @@ public class TestJobHistoryEventHandler {
long currentTime = System.currentTimeMillis();
try {
yarnCluster = new MiniYARNCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, 1, 1, 1, true);
yarnCluster.init(conf);
yarnCluster.start();
jheh.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
index 4ffbad7..c2ef128 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMRTimelineEventHandling.java
@@ -35,58 +35,6 @@ import org.junit.Test;
public class TestMRTimelineEventHandling {
@Test
- public void testTimelineServiceStartInMiniCluster() throws Exception {
- Configuration conf = new YarnConfiguration();
-
- /*
- * Timeline service should not start if the config is set to false
- * Regardless to the value of MAPREDUCE_JOB_EMIT_TIMELINE_DATA
- */
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, true);
- MiniMRYarnCluster cluster = null;
- try {
- cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
- cluster.init(conf);
- cluster.start();
-
- //verify that the timeline service is not started.
- Assert.assertNull("Timeline Service should not have been started",
- cluster.getApplicationHistoryServer());
-
- //Run a MR job and verify it succeeds
- Path inDir = new Path("input");
- Path outDir = new Path("output");
- RunningJob job =
- UtilsForTests.runJobSucceed(new JobConf(conf), inDir, outDir);
- Assert.assertEquals(JobStatus.SUCCEEDED,
- job.getJobStatus().getState().getValue());
- }
- finally {
- if(cluster != null) {
- cluster.stop();
- }
- }
- conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false);
- conf.setBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA, false);
- cluster = null;
- try {
- cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
- cluster.init(conf);
- cluster.start();
- Assert.assertNull("Timeline Service should not have been started",
- cluster.getApplicationHistoryServer());
- }
- finally {
- if(cluster != null) {
- cluster.stop();
- }
- }
- }
-
- @Test
public void testMRTimelineEventHandling() throws Exception {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
@@ -94,7 +42,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -148,7 +96,7 @@ public class TestMRTimelineEventHandling {
MiniMRYarnCluster cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
@@ -185,7 +133,7 @@ public class TestMRTimelineEventHandling {
cluster = null;
try {
cluster = new MiniMRYarnCluster(
- TestJobHistoryEventHandler.class.getSimpleName(), 1);
+ TestJobHistoryEventHandler.class.getSimpleName(), 1, true);
cluster.init(conf);
cluster.start();
TimelineStore ts = cluster.getApplicationHistoryServer()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
index 45b736e..47b38a1 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/MiniMRYarnCluster.java
@@ -72,7 +72,11 @@ public class MiniMRYarnCluster extends MiniYARNCluster {
}
public MiniMRYarnCluster(String testName, int noOfNMs) {
- super(testName, 1, noOfNMs, 4, 4);
+ this(testName, noOfNMs, false);
+ }
+
+ public MiniMRYarnCluster(String testName, int noOfNMs, boolean enableAHS) {
+ super(testName, 1, noOfNMs, 4, 4, enableAHS);
historyServerWrapper = new JobHistoryServerWrapper();
addService(historyServerWrapper);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 17d888d..6ef7c62 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -49,9 +49,6 @@ Release 2.6.1 - 2015-09-09
YARN-2905. AggregatedLogsBlock page can infinitely loop if the aggregated
log file is corrupted (Varun Saxena via jlowe)
- YARN-2890. MiniYARNCluster should start the timeline server based on the
- configuration. (Mit Desai via zjshen)
-
YARN-2894. Fixed a bug regarding application view acl when RM fails over.
(Rohith Sharmaks via jianhe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
index 904ad58..17c29ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/test/java/org/apache/hadoop/yarn/applications/distributedshell/TestDistributedShell.java
@@ -84,7 +84,7 @@ public class TestDistributedShell {
if (yarnCluster == null) {
yarnCluster =
new MiniYARNCluster(TestDistributedShell.class.getSimpleName(), 1,
- numNodeManager, 1, 1);
+ numNodeManager, 1, 1, true);
yarnCluster.init(conf);
yarnCluster.start();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index ea9e361..1cc43b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -294,7 +294,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
conf.setBoolean(YarnConfiguration.AUTO_FAILOVER_ENABLED, false);
cluster =
new MiniYARNClusterForHATesting(TestRMFailover.class.getName(), 2,
- numOfNMs, 1, 1, overrideClientRMService, overrideRTS,
+ numOfNMs, 1, 1, false, overrideClientRMService, overrideRTS,
overrideApplicationMasterService);
cluster.resetStartFailoverFlag(false);
cluster.init(conf);
@@ -325,10 +325,10 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public MiniYARNClusterForHATesting(String testName,
int numResourceManagers, int numNodeManagers, int numLocalDirs,
- int numLogDirs, boolean overrideClientRMService,
+ int numLogDirs, boolean enableAHS, boolean overrideClientRMService,
boolean overrideRTS, boolean overrideApplicationMasterService) {
super(testName, numResourceManagers, numNodeManagers, numLocalDirs,
- numLogDirs);
+ numLogDirs, enableAHS);
this.overrideClientRMService = overrideClientRMService;
this.overrideRTS = overrideRTS;
this.overrideApplicationMasterService = overrideApplicationMasterService;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c4a6e12/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
index 2eb5034..e83d601 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
@@ -57,6 +57,7 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerResp
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryStore;
import org.apache.hadoop.yarn.server.applicationhistoryservice.MemoryApplicationHistoryStore;
+import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.NodeHealthCheckerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
@@ -119,6 +120,7 @@ public class MiniYARNCluster extends CompositeService {
private int numLocalDirs;
// Number of nm-log-dirs per nodemanager
private int numLogDirs;
+ private boolean enableAHS;
/**
* @param testName name of the test
@@ -126,13 +128,15 @@ public class MiniYARNCluster extends CompositeService {
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
+ * @param enableAHS enable ApplicationHistoryServer or not
*/
public MiniYARNCluster(
String testName, int numResourceManagers, int numNodeManagers,
- int numLocalDirs, int numLogDirs) {
+ int numLocalDirs, int numLogDirs, boolean enableAHS) {
super(testName.replace("$", ""));
this.numLocalDirs = numLocalDirs;
this.numLogDirs = numLogDirs;
+ this.enableAHS = enableAHS;
String testSubDir = testName.replace("$", "");
File targetWorkDir = new File("target", testSubDir);
try {
@@ -184,6 +188,20 @@ public class MiniYARNCluster extends CompositeService {
/**
* @param testName name of the test
+ * @param numResourceManagers the number of resource managers in the cluster
+ * @param numNodeManagers the number of node managers in the cluster
+ * @param numLocalDirs the number of nm-local-dirs per nodemanager
+ * @param numLogDirs the number of nm-log-dirs per nodemanager
+ */
+ public MiniYARNCluster(
+ String testName, int numResourceManagers, int numNodeManagers,
+ int numLocalDirs, int numLogDirs) {
+ this(testName, numResourceManagers, numNodeManagers, numLocalDirs,
+ numLogDirs, false);
+ }
+
+ /**
+ * @param testName name of the test
* @param numNodeManagers the number of node managers in the cluster
* @param numLocalDirs the number of nm-local-dirs per nodemanager
* @param numLogDirs the number of nm-log-dirs per nodemanager
@@ -242,8 +260,8 @@ public class MiniYARNCluster extends CompositeService {
addService(new NodeManagerWrapper(index));
}
- if(conf.getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, false)) {
- addService(new ApplicationHistoryServerWrapper());
+ if (enableAHS) {
+ addService(new ApplicationHistoryServerWrapper());
}
super.serviceInit(
[15/50] [abbrv] hadoop git commit: HDFS-7980. Incremental BlockReport
will dramatically slow down namenode startup. Contributed by Walter Su
Posted by sj...@apache.org.
HDFS-7980. Incremental BlockReport will dramatically slow down namenode startup. Contributed by Walter Su
(cherry picked from commit 4e1f2eb3955a97a70cf127dc97ae49201a90f5e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ed162bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ed162bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ed162bc
Branch: refs/heads/branch-2.6
Commit: 8ed162bcbf89c5b05f2092731f60ee0a746cf4ca
Parents: c76f5b3
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Thu May 7 11:36:35 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Sep 7 11:46:33 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 8 +-
.../blockmanagement/TestBlockManager.java | 111 +++++++++++++++++++
3 files changed, 118 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 759a20f..d2f07c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -153,6 +153,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8270. create() always retried with hardcoded timeout when file already
exists with open lease (J.Andreina via vinayakumarb)
+ HDFS-7980. Incremental BlockReport will dramatically slow down namenode
+ startup. (Walter Su via szetszwo)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index bd9d2f7..846bc92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -1820,7 +1820,7 @@ public class BlockManager {
return !node.hasStaleStorages();
}
- if (storageInfo.numBlocks() == 0) {
+ if (storageInfo.getBlockReportCount() == 0) {
// The first block report can be processed a lot more efficiently than
// ordinary block reports. This shortens restart times.
processFirstBlockReport(storageInfo, newReport);
@@ -2043,7 +2043,7 @@ public class BlockManager {
final BlockListAsLongs report) throws IOException {
if (report == null) return;
assert (namesystem.hasWriteLock());
- assert (storageInfo.numBlocks() == 0);
+ assert (storageInfo.getBlockReportCount() == 0);
BlockReportIterator itBR = report.getBlockReportIterator();
while(itBR.hasNext()) {
@@ -2463,14 +2463,14 @@ public class BlockManager {
}
// just add it
- storageInfo.addBlock(storedBlock);
+ boolean result = storageInfo.addBlock(storedBlock);
// Now check for completion of blocks and safe block count
int numCurrentReplica = countLiveNodes(storedBlock);
if (storedBlock.getBlockUCState() == BlockUCState.COMMITTED
&& numCurrentReplica >= minReplication) {
completeBlock(storedBlock.getBlockCollection(), storedBlock, false);
- } else if (storedBlock.isComplete()) {
+ } else if (storedBlock.isComplete() && result == true) {
// check whether safe replication is reached for the block
// only complete blocks are counted towards that.
// In the case that the block just became complete above, completeBlock()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ed162bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 7eec52d..eeec73c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -47,10 +47,15 @@ import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor.BlockTargetPair;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
+import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaBeingWritten;
+import org.apache.hadoop.hdfs.server.datanode.ReplicaInfo;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
+import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.net.NetworkTopology;
import org.junit.Assert;
@@ -653,6 +658,112 @@ public class TestBlockManager {
new BlockListAsLongs(null, null), null, false);
assertEquals(1, ds.getBlockReportCount());
}
+
+ /**
+ * test when NN starts and in same mode, it receives an incremental blockReport
+ * firstly. Then receives first full block report.
+ */
+ @Test
+ public void testSafeModeIBRBeforeFirstFullBR() throws Exception {
+ // pretend to be in safemode
+ doReturn(true).when(fsn).isInStartupSafeMode();
+
+ DatanodeDescriptor node = nodes.get(0);
+ DatanodeStorageInfo ds = node.getStorageInfos()[0];
+ node.isAlive = true;
+ DatanodeRegistration nodeReg = new DatanodeRegistration(node, null, null, "");
+
+ // register new node
+ bm.getDatanodeManager().registerDatanode(nodeReg);
+ bm.getDatanodeManager().addDatanode(node);
+ assertEquals(node, bm.getDatanodeManager().getDatanode(node));
+ assertEquals(0, ds.getBlockReportCount());
+ // Build a incremental report
+ List<ReceivedDeletedBlockInfo> rdbiList =
+ new ArrayList<ReceivedDeletedBlockInfo>();
+ // Build a full report
+ List<Block> blocks = new ArrayList<Block>();
+
+ // blk_42 is finalized.
+ long receivedBlockId = 42; // arbitrary
+ BlockInfo receivedBlock = addBlockToBM(receivedBlockId);
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ blocks.add(new FinalizedReplica(receivedBlock, null, null));
+
+ // blk_43 is under construction.
+ long receivingBlockId = 43;
+ BlockInfo receivingBlock = addUcBlockToBM(receivingBlockId);
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
+ blocks.add(new ReplicaBeingWritten(receivingBlock, null, null, null));
+
+ // blk_44 has 2 records in IBR. It's finalized. So full BR has 1 record.
+ long receivingReceivedBlockId = 44;
+ BlockInfo receivingReceivedBlock = addBlockToBM(receivingReceivedBlockId);
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVING_BLOCK, null));
+ rdbiList.add(new ReceivedDeletedBlockInfo(new Block(receivingReceivedBlock),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ blocks.add(new FinalizedReplica(receivingReceivedBlock, null, null));
+
+ // blk_45 is not in full BR, because it's deleted.
+ long ReceivedDeletedBlockId = 45;
+ rdbiList.add(new ReceivedDeletedBlockInfo(
+ new Block(ReceivedDeletedBlockId),
+ ReceivedDeletedBlockInfo.BlockStatus.RECEIVED_BLOCK, null));
+ rdbiList.add(new ReceivedDeletedBlockInfo(
+ new Block(ReceivedDeletedBlockId),
+ ReceivedDeletedBlockInfo.BlockStatus.DELETED_BLOCK, null));
+
+ // blk_46 exists in DN for a long time, so it's in full BR, but not in IBR.
+ long existedBlockId = 46;
+ BlockInfo existedBlock = addBlockToBM(existedBlockId);
+ blocks.add(new FinalizedReplica(existedBlock, null, null));
+
+ // process IBR and full BR
+ StorageReceivedDeletedBlocks srdb =
+ new StorageReceivedDeletedBlocks(new DatanodeStorage(ds.getStorageID()),
+ rdbiList.toArray(new ReceivedDeletedBlockInfo[rdbiList.size()]));
+ bm.processIncrementalBlockReport(node, srdb);
+ // Make sure it's the first full report
+ assertEquals(0, ds.getBlockReportCount());
+ bm.processReport(node, new DatanodeStorage(ds.getStorageID()),
+ new BlockListAsLongs(blocks,
+ (List<ReplicaInfo>) new ArrayList<ReplicaInfo>()), null, false);
+ assertEquals(1, ds.getBlockReportCount());
+
+ // verify the storage info is correct
+ assertTrue(bm.getStoredBlock(new Block(receivedBlockId)).findStorageInfo
+ (ds) >= 0);
+ assertTrue(((BlockInfoUnderConstruction) bm.
+ getStoredBlock(new Block(receivingBlockId))).getNumExpectedLocations() > 0);
+ assertTrue(bm.getStoredBlock(new Block(receivingReceivedBlockId))
+ .findStorageInfo(ds) >= 0);
+ assertNull(bm.getStoredBlock(new Block(ReceivedDeletedBlockId)));
+ assertTrue(bm.getStoredBlock(new Block(existedBlock)).findStorageInfo
+ (ds) >= 0);
+ }
+
+ private BlockInfo addBlockToBM(long blkId) {
+ Block block = new Block(blkId);
+ BlockInfo blockInfo =
+ new BlockInfo(block, (short) 3);
+ BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ return blockInfo;
+ }
+
+ private BlockInfo addUcBlockToBM(long blkId) {
+ Block block = new Block(blkId);
+ BlockInfoUnderConstruction blockInfo =
+ new BlockInfoUnderConstruction(block, (short) 3);
+ BlockCollection bc = Mockito.mock(BlockCollection.class);
+ Mockito.doReturn((short) 3).when(bc).getBlockReplication();
+ bm.blocksMap.addBlockCollection(blockInfo, bc);
+ return blockInfo;
+ }
/**
* Tests that a namenode doesn't choose a datanode with full disks to
[23/50] [abbrv] hadoop git commit: YARN-4047. ClientRMService
getApplications has high scheduler lock contention. Contributed by Jason Lowe
(cherry picked from commit 7a445fcfabcf9c6aae219051f65d3f6cb8feb87c)
Posted by sj...@apache.org.
YARN-4047. ClientRMService getApplications has high scheduler lock contention. Contributed by Jason Lowe
(cherry picked from commit 7a445fcfabcf9c6aae219051f65d3f6cb8feb87c)
(cherry picked from commit 703fa1b141a98449746bd6fb3b144e74d964d1f5)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2b526ba7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2b526ba7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2b526ba7
Branch: refs/heads/branch-2.6
Commit: 2b526ba75780ac61ba6d0bcb36044c6f7d9b6633
Parents: d59bf81
Author: Jian He <ji...@apache.org>
Authored: Thu Aug 13 16:02:57 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 22:57:35 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/ClientRMService.java | 11 +++++++----
2 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b526ba7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ac114fa..7ff9d72 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -202,6 +202,9 @@ Release 2.6.1 - UNRELEASED
YARN-3999. RM hangs on draing events. (Jian He via xgong)
+ YARN-4047. ClientRMService getApplications has high scheduler lock contention.
+ (Jason Lowe via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2b526ba7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index abd08f1..6661d04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -738,13 +738,9 @@ public class ClientRMService extends AbstractService implements
RMApp application = appsIter.next();
// Check if current application falls under the specified scope
- boolean allowAccess = checkAccess(callerUGI, application.getUser(),
- ApplicationAccessType.VIEW_APP, application);
if (scope == ApplicationsRequestScope.OWN &&
!callerUGI.getUserName().equals(application.getUser())) {
continue;
- } else if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) {
- continue;
}
if (applicationTypes != null && !applicationTypes.isEmpty()) {
@@ -793,6 +789,13 @@ public class ClientRMService extends AbstractService implements
}
}
+ // checkAccess can grab the scheduler lock so call it last
+ boolean allowAccess = checkAccess(callerUGI, application.getUser(),
+ ApplicationAccessType.VIEW_APP, application);
+ if (scope == ApplicationsRequestScope.VIEWABLE && !allowAccess) {
+ continue;
+ }
+
reports.add(application.createAndGetApplicationReport(
callerUGI.getUserName(), allowAccess));
}
[49/50] [abbrv] hadoop git commit: YARN-2019. Retrospect on decision
of making RM crashed if any exception throw in ZKRMStateStore. Contributed by
Jian He.
Posted by sj...@apache.org.
YARN-2019. Retrospect on decision of making RM crashed if any exception throw in ZKRMStateStore. Contributed by Jian He.
(cherry picked from commit db57d91ac91e895bcb9a23fa50af0b2fbcb1db5a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d27f09c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d27f09c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d27f09c9
Branch: refs/heads/branch-2.6
Commit: d27f09c9369d9f2604eb31deaff50f4bd5aa98ac
Parents: c09bb46
Author: Xuan <xg...@apache.org>
Authored: Mon Sep 7 17:34:33 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 16:30:49 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/conf/YarnConfiguration.java | 11 +++++++++++
.../src/main/resources/yarn-default.xml | 16 ++++++++++++++++
.../resourcemanager/recovery/RMStateStore.java | 9 +++++++--
4 files changed, 37 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d27f09c9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6d016b7..15d082b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -6,6 +6,9 @@ Release 2.6.2 - UNRELEASED
NEW FEATURES
+ YARN-2019. Retrospect on decision of making RM crashed if any exception throw
+ in ZKRMStateStore. (Jian He via junping_du)
+
IMPROVEMENTS
YARN-4092. Fixed UI redirection to print useful messages when both RMs are
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d27f09c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 881ecb5..471297c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -380,6 +380,11 @@ public class YarnConfiguration extends Configuration {
public static final String RECOVERY_ENABLED = RM_PREFIX + "recovery.enabled";
public static final boolean DEFAULT_RM_RECOVERY_ENABLED = false;
+ public static final String YARN_FAIL_FAST = YARN_PREFIX + "fail-fast";
+ public static final boolean DEFAULT_YARN_FAIL_FAST = true;
+
+ public static final String RM_FAIL_FAST = RM_PREFIX + "fail-fast";
+
@Private
public static final String RM_WORK_PRESERVING_RECOVERY_ENABLED = RM_PREFIX
+ "work-preserving-recovery.enabled";
@@ -1568,6 +1573,12 @@ public class YarnConfiguration extends Configuration {
YARN_HTTP_POLICY_DEFAULT));
}
+ public static boolean shouldRMFailFast(Configuration conf) {
+ return conf.getBoolean(YarnConfiguration.RM_FAIL_FAST,
+ conf.getBoolean(YarnConfiguration.YARN_FAIL_FAST,
+ YarnConfiguration.DEFAULT_YARN_FAIL_FAST));
+ }
+
@Private
public static String getClusterId(Configuration conf) {
String clusterId = conf.get(YarnConfiguration.RM_CLUSTER_ID);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d27f09c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 2ffa8b8..fa2e5cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -276,6 +276,22 @@
</property>
<property>
+ <description>Should RM fail fast if it encounters any errors. By defalt, it
+ points to ${yarn.fail-fast}. Errors include:
+ 1) exceptions when state-store write/read operations fails.
+ </description>
+ <name>yarn.resourcemanager.fail-fast</name>
+ <value>${yarn.fail-fast}</value>
+ </property>
+
+ <property>
+ <description>Should YARN fail fast if it encounters any errors.
+ </description>
+ <name>yarn.fail-fast</name>
+ <value>true</value>
+ </property>
+
+ <property>
<description>Enable RM work preserving recovery. This configuration is private
to YARN for experimenting the feature.
</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d27f09c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index 983cc81..37855f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
@@ -820,14 +821,18 @@ public abstract class RMStateStore extends AbstractService {
* @param failureCause the exception due to which the operation failed
*/
protected void notifyStoreOperationFailed(Exception failureCause) {
+ LOG.error("State store operation failed ", failureCause);
if (failureCause instanceof StoreFencedException) {
Thread standByTransitionThread =
new Thread(new StandByTransitionThread());
standByTransitionThread.setName("StandByTransitionThread Handler");
standByTransitionThread.start();
} else {
- rmDispatcher.getEventHandler().handle(
- new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED, failureCause));
+ if (YarnConfiguration.shouldRMFailFast(getConfig())) {
+ rmDispatcher.getEventHandler().handle(
+ new RMFatalEvent(RMFatalEventType.STATE_STORE_OP_FAILED,
+ failureCause));
+ }
}
}
[06/50] [abbrv] hadoop git commit: YARN-2920. Changed
CapacityScheduler to kill containers on nodes where node labels are changed.
Contributed by Wangda Tan (cherry picked from commit
fdf042dfffa4d2474e3cac86cfb8fe9ee4648beb)
Posted by sj...@apache.org.
YARN-2920. Changed CapacityScheduler to kill containers on nodes where node labels are changed. Contributed by Wangda Tan
(cherry picked from commit fdf042dfffa4d2474e3cac86cfb8fe9ee4648beb)
(cherry picked from commit 411836b74c6c02c0b5aebbbce29c209d93db1de2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88f022da
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88f022da
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88f022da
Branch: refs/heads/branch-2.6
Commit: 88f022da245c5d34997494a822124d07f8a5f72f
Parents: 2073fc0
Author: Jian He <ji...@apache.org>
Authored: Mon Dec 22 16:50:15 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:18 2015 -0700
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 3 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 3 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/ResourceManager.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 23 +++
.../resourcemanager/rmnode/RMNodeImpl.java | 6 +-
.../scheduler/SchedulerNode.java | 24 ++-
.../scheduler/capacity/AbstractCSQueue.java | 5 +
.../scheduler/capacity/CSQueue.java | 8 +
.../scheduler/capacity/CapacityScheduler.java | 88 +++++++--
.../scheduler/capacity/LeafQueue.java | 26 +--
.../scheduler/capacity/ParentQueue.java | 24 ++-
.../common/fica/FiCaSchedulerNode.java | 11 +-
.../event/NodeLabelsUpdateSchedulerEvent.java | 37 ++++
.../scheduler/event/SchedulerEventType.java | 1 +
.../yarn/server/resourcemanager/MockNodes.java | 3 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 193 +++++++++++++++++++
17 files changed, 414 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index fdddcf4..ee6eb7b 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -162,7 +163,7 @@ public class NodeInfo {
@Override
public Set<String> getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index 3b185ae..b64be1b 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
.UpdatedContainerInfo;
@@ -150,6 +151,6 @@ public class RMNodeWrapper implements RMNode {
@Override
public Set<String> getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8244d61..fa1e120 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -171,6 +171,9 @@ Release 2.6.1 - UNRELEASED
YARN-3733. Fix DominantRC#compare() does not work as expected if
cluster resource is empty. (Rohith Sharmaks via wangda)
+ YARN-2920. Changed CapacityScheduler to kill containers on nodes where
+ node labels are changed. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index ea762c0..353851c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -430,6 +430,7 @@ public class ResourceManager extends CompositeService implements Recoverable {
rmContext.setAMFinishingMonitor(amFinishingMonitor);
RMNodeLabelsManager nlm = createNodeLabelManager();
+ nlm.setRMContext(rmContext);
addService(nlm);
rmContext.setNodeLabelManager(nlm);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index ba1727c..d9828e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -35,7 +35,10 @@ import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.collect.ImmutableSet;
@@ -57,6 +60,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
new ConcurrentHashMap<String, Queue>();
protected AccessControlList adminAcl;
+ private RMContext rmContext = null;
+
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
@@ -331,6 +336,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
return map;
}
+ @SuppressWarnings("unchecked")
private void updateResourceMappings(Map<String, Host> before,
Map<String, Host> after) {
// Get NMs in before only
@@ -341,6 +347,10 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
for (Entry<String, Host> entry : after.entrySet()) {
allNMs.addAll(entry.getValue().nms.keySet());
}
+
+ // Map used to notify RM
+ Map<NodeId, Set<String>> newNodeToLabelsMap =
+ new HashMap<NodeId, Set<String>>();
// traverse all nms
for (NodeId nodeId : allNMs) {
@@ -379,6 +389,9 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
Node newNM;
if ((newNM = getNMInNodeSet(nodeId, after, true)) != null) {
Set<String> newLabels = getLabelsByNode(nodeId, after);
+
+ newNodeToLabelsMap.put(nodeId, ImmutableSet.copyOf(newLabels));
+
// no label in the past
if (newLabels.isEmpty()) {
// update labels
@@ -405,6 +418,12 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
}
}
}
+
+ // Notify RM
+ if (rmContext != null && rmContext.getDispatcher() != null) {
+ rmContext.getDispatcher().getEventHandler().handle(
+ new NodeLabelsUpdateSchedulerEvent(newNodeToLabelsMap));
+ }
}
public Resource getResourceByLabel(String label, Resource clusterResource) {
@@ -452,4 +471,8 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
}
return false;
}
+
+ public void setRMContext(RMContext rmContext) {
+ this.rmContext = rmContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b92f399..b57b7cc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
@@ -860,9 +861,10 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
@Override
public Set<String> getNodeLabels() {
- if (context.getNodeLabelManager() == null) {
+ RMNodeLabelsManager nlm = context.getNodeLabelManager();
+ if (nlm == null || nlm.getLabelsOnNode(nodeId) == null) {
return CommonNodeLabelsManager.EMPTY_STRING_SET;
}
- return context.getNodeLabelManager().getLabelsOnNode(nodeId);
+ return nlm.getLabelsOnNode(nodeId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index f4d8731..d1922ee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -33,11 +34,14 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.collect.ImmutableSet;
+
/**
* Represents a YARN Cluster Node from the viewpoint of the scheduler.
@@ -61,8 +65,11 @@ public abstract class SchedulerNode {
private final RMNode rmNode;
private final String nodeName;
-
- public SchedulerNode(RMNode node, boolean usePortForNodeName) {
+
+ private volatile Set<String> labels = null;
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> labels) {
this.rmNode = node;
this.availableResource = Resources.clone(node.getTotalCapability());
this.totalResourceCapability = Resources.clone(node.getTotalCapability());
@@ -71,6 +78,11 @@ public abstract class SchedulerNode {
} else {
nodeName = rmNode.getHostName();
}
+ this.labels = ImmutableSet.copyOf(labels);
+ }
+
+ public SchedulerNode(RMNode node, boolean usePortForNodeName) {
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
}
public RMNode getRMNode() {
@@ -275,4 +287,12 @@ public abstract class SchedulerNode {
}
allocateContainer(rmContainer);
}
+
+ public Set<String> getLabels() {
+ return labels;
+ }
+
+ public void updateLabels(Set<String> labels) {
+ this.labels = labels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index fc0fbb4..1f6696d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -447,4 +447,9 @@ public abstract class AbstractCSQueue implements CSQueue {
public Map<QueueACL, AccessControlList> getACLs() {
return acls;
}
+
+ @Private
+ public Resource getUsedResourceByLabel(String nodeLabel) {
+ return usedResourcesByNodeLabels.get(nodeLabel);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 6438d6c..07a7e0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -144,6 +144,14 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
public Resource getUsedResources();
/**
+ * Get the currently utilized resources which allocated at nodes with label
+ * specified
+ *
+ * @return used resources by the queue and it's children
+ */
+ public Resource getUsedResourceByLabel(String nodeLabel);
+
+ /**
* Get the current run-state of the queue
* @return current run-state
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 042c83c..c49c498 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -25,15 +25,15 @@ import java.util.Collection;
import java.util.Comparator;
import java.util.EnumSet;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import java.util.HashSet;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -48,12 +48,15 @@ import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
+import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -79,12 +82,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.PreemptableResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration.QueueMapping.MappingType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -93,6 +99,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptR
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ContainerExpiredSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
@@ -106,11 +113,6 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.hadoop.yarn.api.records.ReservationId;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerDynamicEditException;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
-
@LimitedPrivate("yarn")
@Evolving
@SuppressWarnings("unchecked")
@@ -966,6 +968,51 @@ public class CapacityScheduler extends
updateNodeResource(nm, resourceOption);
root.updateClusterResource(clusterResource);
}
+
+ /**
+ * Process node labels update on a node.
+ *
+ * TODO: Currently capacity scheduler will kill containers on a node when
+ * labels on the node changed. It is a simply solution to ensure guaranteed
+ * capacity on labels of queues. When YARN-2498 completed, we can let
+ * preemption policy to decide if such containers need to be killed or just
+ * keep them running.
+ */
+ private synchronized void updateLabelsOnNode(NodeId nodeId,
+ Set<String> newLabels) {
+ FiCaSchedulerNode node = nodes.get(nodeId);
+ if (null == node) {
+ return;
+ }
+
+ // labels is same, we don't need do update
+ if (node.getLabels().size() == newLabels.size()
+ && node.getLabels().containsAll(newLabels)) {
+ return;
+ }
+
+ // Kill running containers since label is changed
+ for (RMContainer rmContainer : node.getRunningContainers()) {
+ ContainerId containerId = rmContainer.getContainerId();
+ completedContainer(rmContainer,
+ ContainerStatus.newInstance(containerId,
+ ContainerState.COMPLETE,
+ String.format(
+ "Container=%s killed since labels on the node=%s changed",
+ containerId.toString(), nodeId.toString()),
+ ContainerExitStatus.KILLED_BY_RESOURCEMANAGER),
+ RMContainerEventType.KILL);
+ }
+
+ // Unreserve container on this node
+ RMContainer reservedContainer = node.getReservedContainer();
+ if (null != reservedContainer) {
+ dropContainerReservation(reservedContainer);
+ }
+
+ // Update node labels after we've done this
+ node.updateLabels(newLabels);
+ }
private synchronized void allocateContainersToNode(FiCaSchedulerNode node) {
if (rmContext.isWorkPreservingRecoveryEnabled()
@@ -1049,6 +1096,19 @@ public class CapacityScheduler extends
nodeResourceUpdatedEvent.getResourceOption());
}
break;
+ case NODE_LABELS_UPDATE:
+ {
+ NodeLabelsUpdateSchedulerEvent labelUpdateEvent =
+ (NodeLabelsUpdateSchedulerEvent) event;
+
+ for (Entry<NodeId, Set<String>> entry : labelUpdateEvent
+ .getUpdatedNodeToLabels().entrySet()) {
+ NodeId id = entry.getKey();
+ Set<String> labels = entry.getValue();
+ updateLabelsOnNode(id, labels);
+ }
+ }
+ break;
case NODE_UPDATE:
{
NodeUpdateSchedulerEvent nodeUpdatedEvent = (NodeUpdateSchedulerEvent)event;
@@ -1117,14 +1177,8 @@ public class CapacityScheduler extends
}
private synchronized void addNode(RMNode nodeManager) {
- // update this node to node label manager
- if (labelManager != null) {
- labelManager.activateNode(nodeManager.getNodeID(),
- nodeManager.getTotalCapability());
- }
-
this.nodes.put(nodeManager.getNodeID(), new FiCaSchedulerNode(nodeManager,
- usePortForNodeName));
+ usePortForNodeName, nodeManager.getNodeLabels()));
Resources.addTo(clusterResource, nodeManager.getTotalCapability());
root.updateClusterResource(clusterResource);
int numNodes = numNodeManagers.incrementAndGet();
@@ -1135,6 +1189,12 @@ public class CapacityScheduler extends
if (scheduleAsynchronously && numNodes == 1) {
asyncSchedulerThread.beginSchedule();
}
+
+ // update this node to node label manager
+ if (labelManager != null) {
+ labelManager.activateNode(nodeManager.getNodeID(),
+ nodeManager.getTotalCapability());
+ }
}
private synchronized void removeNode(RMNode nodeInfo) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 60b5a59..509b0f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -809,7 +809,7 @@ public class LeafQueue extends AbstractCSQueue {
// if our queue cannot access this node, just return
if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels,
- labelManager.getLabelsOnNode(node.getNodeID()))) {
+ node.getLabels())) {
return NULL_ASSIGNMENT;
}
@@ -878,7 +878,7 @@ public class LeafQueue extends AbstractCSQueue {
// Check queue max-capacity limit
if (!canAssignToThisQueue(clusterResource, required,
- labelManager.getLabelsOnNode(node.getNodeID()), application, true)) {
+ node.getLabels(), application, true)) {
return NULL_ASSIGNMENT;
}
@@ -911,7 +911,7 @@ public class LeafQueue extends AbstractCSQueue {
// Book-keeping
// Note: Update headroom to account for current allocation too...
allocateResource(clusterResource, application, assigned,
- labelManager.getLabelsOnNode(node.getNodeID()));
+ node.getLabels());
// Don't reset scheduling opportunities for non-local assignments
// otherwise the app will be delayed for each non-local assignment.
@@ -1561,7 +1561,7 @@ public class LeafQueue extends AbstractCSQueue {
// check if the resource request can access the label
if (!SchedulerUtils.checkNodeLabelExpression(
- labelManager.getLabelsOnNode(node.getNodeID()),
+ node.getLabels(),
request.getNodeLabelExpression())) {
// this is a reserved container, but we cannot allocate it now according
// to label not match. This can be caused by node label changed
@@ -1752,8 +1752,7 @@ public class LeafQueue extends AbstractCSQueue {
// Book-keeping
if (removed) {
releaseResource(clusterResource, application,
- container.getResource(),
- labelManager.getLabelsOnNode(node.getNodeID()));
+ container.getResource(), node.getLabels());
LOG.info("completedContainer" +
" container=" + container +
" queue=" + this +
@@ -1950,9 +1949,10 @@ public class LeafQueue extends AbstractCSQueue {
}
// Careful! Locking order is important!
synchronized (this) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, attempt, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
}
getParent().recoverContainer(clusterResource, attempt, rmContainer);
}
@@ -1989,9 +1989,10 @@ public class LeafQueue extends AbstractCSQueue {
public void attachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
allocateResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveIn=" + this + " usedCapacity=" + getUsedCapacity()
@@ -2006,9 +2007,10 @@ public class LeafQueue extends AbstractCSQueue {
public void detachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
releaseResource(clusterResource, application, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer.getContainer()
- .getNodeId()));
+ .getResource(), node.getLabels());
LOG.info("movedContainer" + " container=" + rmContainer.getContainer()
+ " resource=" + rmContainer.getContainer().getResource()
+ " queueMoveOut=" + this + " usedCapacity=" + getUsedCapacity()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 6ffaf4c..fd598f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -73,6 +73,7 @@ public class ParentQueue extends AbstractCSQueue {
private final boolean rootQueue;
final Comparator<CSQueue> queueComparator;
volatile int numApplications;
+ private final CapacitySchedulerContext scheduler;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -80,7 +81,7 @@ public class ParentQueue extends AbstractCSQueue {
public ParentQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
super(cs, queueName, parent, old);
-
+ this.scheduler = cs;
this.queueComparator = cs.getQueueComparator();
this.rootQueue = (parent == null);
@@ -420,10 +421,10 @@ public class ParentQueue extends AbstractCSQueue {
Resource clusterResource, FiCaSchedulerNode node, boolean needToUnreserve) {
CSAssignment assignment =
new CSAssignment(Resources.createResource(0, 0), NodeType.NODE_LOCAL);
+ Set<String> nodeLabels = node.getLabels();
// if our queue cannot access this node, just return
- if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels,
- labelManager.getLabelsOnNode(node.getNodeID()))) {
+ if (!SchedulerUtils.checkQueueAccessToNode(accessibleLabels, nodeLabels)) {
return assignment;
}
@@ -434,7 +435,6 @@ public class ParentQueue extends AbstractCSQueue {
}
boolean localNeedToUnreserve = false;
- Set<String> nodeLabels = labelManager.getLabelsOnNode(node.getNodeID());
// Are we over maximum-capacity for this queue?
if (!canAssignToThisQueue(clusterResource, nodeLabels)) {
@@ -641,7 +641,7 @@ public class ParentQueue extends AbstractCSQueue {
// Book keeping
synchronized (this) {
super.releaseResource(clusterResource, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(node.getNodeID()));
+ .getResource(), node.getLabels());
LOG.info("completedContainer" +
" queue=" + getQueueName() +
@@ -703,9 +703,10 @@ public class ParentQueue extends AbstractCSQueue {
}
// Careful! Locking order is important!
synchronized (this) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
super.allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
}
if (parent != null) {
parent.recoverContainer(clusterResource, attempt, rmContainer);
@@ -730,9 +731,10 @@ public class ParentQueue extends AbstractCSQueue {
public void attachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
super.allocateResource(clusterResource, rmContainer.getContainer()
- .getResource(), labelManager.getLabelsOnNode(rmContainer
- .getContainer().getNodeId()));
+ .getResource(), node.getLabels());
LOG.info("movedContainer" + " queueMoveIn=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
@@ -748,9 +750,11 @@ public class ParentQueue extends AbstractCSQueue {
public void detachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer rmContainer) {
if (application != null) {
+ FiCaSchedulerNode node =
+ scheduler.getNode(rmContainer.getContainer().getNodeId());
super.releaseResource(clusterResource,
rmContainer.getContainer().getResource(),
- labelManager.getLabelsOnNode(rmContainer.getContainer().getNodeId()));
+ node.getLabels());
LOG.info("movedContainer" + " queueMoveOut=" + getQueueName()
+ " usedCapacity=" + getUsedCapacity() + " absoluteUsedCapacity="
+ getAbsoluteUsedCapacity() + " used=" + usedResources + " cluster="
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
index 5227aac..fe6db47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerNode.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica;
+import java.util.Set;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
-import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
@@ -32,9 +34,14 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
public class FiCaSchedulerNode extends SchedulerNode {
private static final Log LOG = LogFactory.getLog(FiCaSchedulerNode.class);
+
+ public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName,
+ Set<String> nodeLabels) {
+ super(node, usePortForNodeName, nodeLabels);
+ }
public FiCaSchedulerNode(RMNode node, boolean usePortForNodeName) {
- super(node, usePortForNodeName);
+ this(node, usePortForNodeName, CommonNodeLabelsManager.EMPTY_STRING_SET);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
new file mode 100644
index 0000000..7723e25
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/NodeLabelsUpdateSchedulerEvent.java
@@ -0,0 +1,37 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
+
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+public class NodeLabelsUpdateSchedulerEvent extends SchedulerEvent {
+ private Map<NodeId, Set<String>> nodeToLabels;
+
+ public NodeLabelsUpdateSchedulerEvent(Map<NodeId, Set<String>> nodeToLabels) {
+ super(SchedulerEventType.NODE_LABELS_UPDATE);
+ this.nodeToLabels = nodeToLabels;
+ }
+
+ public Map<NodeId, Set<String>> getUpdatedNodeToLabels() {
+ return nodeToLabels;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 062f831..13aecb3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -25,6 +25,7 @@ public enum SchedulerEventType {
NODE_REMOVED,
NODE_UPDATE,
NODE_RESOURCE_UPDATE,
+ NODE_LABELS_UPDATE,
// Source: RMApp
APP_ADDED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 228f200..278c151 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
@@ -206,7 +207,7 @@ public class MockNodes {
@Override
public Set<String> getNodeLabels() {
- return null;
+ return RMNodeLabelsManager.EMPTY_STRING_SET;
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88f022da/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
new file mode 100644
index 0000000..261fa01
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.ArrayList;
+import java.util.Set;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import com.google.common.collect.ImmutableMap;
+import com.google.common.collect.ImmutableSet;
+import com.google.common.collect.Sets;
+
+public class TestCapacitySchedulerNodeLabelUpdate {
+ private final int GB = 1024;
+
+ private YarnConfiguration conf;
+
+ RMNodeLabelsManager mgr;
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new YarnConfiguration();
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ mgr = new MemoryRMNodeLabelsManager();
+ mgr.init(conf);
+ }
+
+ private Configuration getConfigurationWithQueueLabels(Configuration config) {
+ CapacitySchedulerConfiguration conf =
+ new CapacitySchedulerConfiguration(config);
+
+ // Define top-level queues
+ conf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] {"a"});
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "y", 100);
+ conf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "z", 100);
+
+ final String A = CapacitySchedulerConfiguration.ROOT + ".a";
+ conf.setCapacity(A, 100);
+ conf.setAccessibleNodeLabels(A, ImmutableSet.of("x", "y", "z"));
+ conf.setCapacityByLabel(A, "x", 100);
+ conf.setCapacityByLabel(A, "y", 100);
+ conf.setCapacityByLabel(A, "z", 100);
+
+ return conf;
+ }
+
+ private Set<String> toSet(String... elements) {
+ Set<String> set = Sets.newHashSet(elements);
+ return set;
+ }
+
+ private void checkUsedResource(MockRM rm, String queueName, int memory) {
+ checkUsedResource(rm, queueName, memory, RMNodeLabelsManager.NO_LABEL);
+ }
+
+ private void checkUsedResource(MockRM rm, String queueName, int memory,
+ String label) {
+ CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
+ CSQueue queue = scheduler.getQueue(queueName);
+ Assert.assertEquals(memory, queue.getUsedResourceByLabel(label).getMemory());
+ }
+
+ @Test (timeout = 30000)
+ public void testNodeUpdate() throws Exception {
+ // set node -> label
+ mgr.addToCluserNodeLabels(ImmutableSet.of("x", "y", "z"));
+
+ // set mapping:
+ // h1 -> x
+ // h2 -> y
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
+ mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
+
+ // inject node label manager
+ MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {
+ @Override
+ public RMNodeLabelsManager createNodeLabelManager() {
+ return mgr;
+ }
+ };
+
+ rm.getRMContext().setNodeLabelManager(mgr);
+ rm.start();
+ MockNM nm1 = rm.registerNode("h1:1234", 8000);
+ MockNM nm2 = rm.registerNode("h2:1234", 8000);
+ MockNM nm3 = rm.registerNode("h3:1234", 8000);
+
+ ContainerId containerId;
+
+ // launch an app to queue a1 (label = x), and check all container will
+ // be allocated in h1
+ RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
+
+ // request a container.
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
+ containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.ALLOCATED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=1G, ""=1G
+ checkUsedResource(rm, "a", 1024, "x");
+ checkUsedResource(rm, "a", 1024);
+
+ // change h1's label to z, container should be killed
+ mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h1", 0),
+ toSet("z")));
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.KILLED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=0G, ""=1G ("" not changed)
+ checkUsedResource(rm, "a", 0, "x");
+ checkUsedResource(rm, "a", 1024);
+
+ // request a container with label = y
+ am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "y");
+ containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
+ Assert.assertTrue(rm.waitForState(nm2, containerId,
+ RMContainerState.ALLOCATED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used y=1G, ""=1G
+ checkUsedResource(rm, "a", 1024, "y");
+ checkUsedResource(rm, "a", 1024);
+
+ // change h2's label to no label, container should be killed
+ mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h2", 0),
+ CommonNodeLabelsManager.EMPTY_STRING_SET));
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.KILLED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=0G, y=0G, ""=1G ("" not changed)
+ checkUsedResource(rm, "a", 0, "x");
+ checkUsedResource(rm, "a", 0, "y");
+ checkUsedResource(rm, "a", 1024);
+
+ containerId =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
+
+ // change h3's label to z, AM container should be killed
+ mgr.replaceLabelsOnNode(ImmutableMap.of(NodeId.newInstance("h3", 0),
+ toSet("z")));
+ Assert.assertTrue(rm.waitForState(nm1, containerId,
+ RMContainerState.KILLED, 10 * 1000));
+
+ // check used resource:
+ // queue-a used x=0G, y=0G, ""=1G ("" not changed)
+ checkUsedResource(rm, "a", 0, "x");
+ checkUsedResource(rm, "a", 0, "y");
+ checkUsedResource(rm, "a", 0);
+
+ rm.close();
+ }
+}
[19/50] [abbrv] hadoop git commit: HDFS-8384. Allow NN to startup if
there are files having a lease but are not under construction. Contributed by
Jing Zhao.
Posted by sj...@apache.org.
HDFS-8384. Allow NN to startup if there are files having a lease but are not under construction. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/33537078
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/33537078
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/33537078
Branch: refs/heads/branch-2.6
Commit: 33537078a80822e64f25d62478732b01704b40c1
Parents: a976acc
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Tue Sep 8 16:45:38 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 16:45:38 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/FSNamesystem.java | 5 ++++-
.../hadoop/hdfs/server/namenode/LeaseManager.java | 14 +++++++++++---
3 files changed, 18 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33537078/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 287ffcb..eaaea5d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -27,6 +27,9 @@ Release 2.6.1 - UNRELEASED
HDFS-8046. Allow better control of getContentSummary (kihwal)
+ HDFS-8384. Allow NN to startup if there are files having a lease but are not
+ under construction. (jing9)
+
OPTIMIZATIONS
HDFS-8480. Fix performance and timeout issues in HDFS-7929 by using
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33537078/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 19edbb5..362b147 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -4693,7 +4693,10 @@ public class FSNamesystem implements Namesystem, FSClusterStats,
assert hasWriteLock();
FileUnderConstructionFeature uc = pendingFile.getFileUnderConstructionFeature();
- Preconditions.checkArgument(uc != null);
+ if (uc == null) {
+ throw new IOException("Cannot finalize file " + src
+ + " because it is not under construction");
+ }
leaseManager.removeLease(uc.getClientName(), src);
pendingFile.recordModification(latestSnapshot);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/33537078/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
index e13a5c6..55ce0bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/LeaseManager.java
@@ -116,7 +116,11 @@ public class LeaseManager {
final INodeFile cons;
try {
cons = this.fsnamesystem.getFSDirectory().getINode(path).asFile();
- Preconditions.checkState(cons.isUnderConstruction());
+ if (!cons.isUnderConstruction()) {
+ LOG.warn("The file " + cons.getFullPathName()
+ + " is not under construction but has lease.");
+ continue;
+ }
} catch (UnresolvedLinkException e) {
throw new AssertionError("Lease files should reside on this FS");
}
@@ -444,8 +448,12 @@ public class LeaseManager {
// verify that path exists in namespace
try {
INodeFile node = INodeFile.valueOf(fsnamesystem.dir.getINode(p), p);
- Preconditions.checkState(node.isUnderConstruction());
- inodes.put(p, node);
+ if (node.isUnderConstruction()) {
+ inodes.put(p, node);
+ } else {
+ LOG.warn("Ignore the lease of file " + p
+ + " for checkpoint since the file is not under construction");
+ }
} catch (IOException ioe) {
LOG.error(ioe);
}
[08/50] [abbrv] hadoop git commit: YARN-2978. Fixed potential NPE
while getting queue info. Contributed by Varun Saxena (cherry picked from
commit dd57c2047bfd21910acc38c98153eedf1db75169)
Posted by sj...@apache.org.
YARN-2978. Fixed potential NPE while getting queue info. Contributed by Varun Saxena
(cherry picked from commit dd57c2047bfd21910acc38c98153eedf1db75169)
(cherry picked from commit c61e8a7bfa7236e354f859a889083fab3d7ca9eb)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/419e18cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/419e18cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/419e18cb
Branch: refs/heads/branch-2.6
Commit: 419e18cb3743f9a146abb0c335883c2ac96583e6
Parents: 88f022d
Author: Jian He <ji...@apache.org>
Authored: Tue Jan 6 14:15:57 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:19 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../scheduler/capacity/AbstractCSQueue.java | 24 +++++++++++---------
.../scheduler/capacity/LeafQueue.java | 10 ++++----
.../scheduler/capacity/ParentQueue.java | 4 +---
4 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fa1e120..be9e2d5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -174,6 +174,9 @@ Release 2.6.1 - UNRELEASED
YARN-2920. Changed CapacityScheduler to kill containers on nodes where
node labels are changed. (Wangda Tan via jianhe)
+ YARN-2978. Fixed potential NPE while getting queue info. (Varun Saxena via
+ jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 1f6696d..fec3a56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -65,7 +65,6 @@ public abstract class AbstractCSQueue implements CSQueue {
RMNodeLabelsManager labelManager;
String defaultLabelExpression;
Resource usedResources = Resources.createResource(0, 0);
- QueueInfo queueInfo;
Map<String, Float> absoluteCapacityByNodeLabels;
Map<String, Float> capacitiyByNodeLabels;
Map<String, Resource> usedResourcesByNodeLabels = new HashMap<String, Resource>();
@@ -87,7 +86,6 @@ public abstract class AbstractCSQueue implements CSQueue {
this.parent = parent;
this.queueName = queueName;
this.resourceCalculator = cs.getResourceCalculator();
- this.queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
// must be called after parent and queueName is set
this.metrics = old != null ? old.getMetrics() :
@@ -99,9 +97,7 @@ public abstract class AbstractCSQueue implements CSQueue {
this.accessibleLabels = cs.getConfiguration().getAccessibleNodeLabels(getQueuePath());
this.defaultLabelExpression = cs.getConfiguration()
.getDefaultNodeLabelExpression(getQueuePath());
-
- this.queueInfo.setQueueName(queueName);
-
+
// inherit from parent if labels not set
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
@@ -280,12 +276,6 @@ public abstract class AbstractCSQueue implements CSQueue {
this.capacitiyByNodeLabels = new HashMap<String, Float>(nodeLabelCapacities);
this.maxCapacityByNodeLabels =
new HashMap<String, Float>(maximumNodeLabelCapacities);
-
- this.queueInfo.setAccessibleNodeLabels(this.accessibleLabels);
- this.queueInfo.setCapacity(this.capacity);
- this.queueInfo.setMaximumCapacity(this.maximumCapacity);
- this.queueInfo.setQueueState(this.state);
- this.queueInfo.setDefaultNodeLabelExpression(this.defaultLabelExpression);
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -330,6 +320,18 @@ public abstract class AbstractCSQueue implements CSQueue {
this.reservationsContinueLooking = reservationContinueLooking;
}
+ protected QueueInfo getQueueInfo() {
+ QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
+ queueInfo.setQueueName(queueName);
+ queueInfo.setAccessibleNodeLabels(accessibleLabels);
+ queueInfo.setCapacity(capacity);
+ queueInfo.setMaximumCapacity(maximumCapacity);
+ queueInfo.setQueueState(state);
+ queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
+ queueInfo.setCurrentCapacity(getUsedCapacity());
+ return queueInfo;
+ }
+
@Private
public Resource getMaximumAllocation() {
return maximumAllocation;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 509b0f2..78b5fb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -160,8 +160,6 @@ public class LeafQueue extends AbstractCSQueue {
float maxAMResourcePerQueuePercent = cs.getConfiguration()
.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
- this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
-
QueueState state = cs.getConfiguration().getState(getQueuePath());
Map<QueueACL, AccessControlList> acls =
@@ -237,14 +235,14 @@ public class LeafQueue extends AbstractCSQueue {
this.defaultLabelExpression)) {
throw new IOException("Invalid default label expression of "
+ " queue="
- + queueInfo.getQueueName()
+ + getQueueName()
+ " doesn't have permission to access all labels "
+ "in default label expression. labelExpression of resource request="
+ (this.defaultLabelExpression == null ? ""
: this.defaultLabelExpression)
+ ". Queue labels="
- + (queueInfo.getAccessibleNodeLabels() == null ? "" : StringUtils.join(queueInfo
- .getAccessibleNodeLabels().iterator(), ',')));
+ + (getAccessibleNodeLabels() == null ? "" : StringUtils.join(
+ getAccessibleNodeLabels().iterator(), ',')));
}
this.nodeLocalityDelay = nodeLocalityDelay;
@@ -412,7 +410,7 @@ public class LeafQueue extends AbstractCSQueue {
@Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
- queueInfo.setCurrentCapacity(usedCapacity);
+ QueueInfo queueInfo = getQueueInfo();
return queueInfo;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/419e18cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index fd598f2..f820cca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -110,8 +110,6 @@ public class ParentQueue extends AbstractCSQueue {
Map<QueueACL, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
- this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
-
setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
defaultLabelExpression, capacitiyByNodeLabels, maxCapacityByNodeLabels,
@@ -206,7 +204,7 @@ public class ParentQueue extends AbstractCSQueue {
@Override
public synchronized QueueInfo getQueueInfo(
boolean includeChildQueues, boolean recursive) {
- queueInfo.setCurrentCapacity(usedCapacity);
+ QueueInfo queueInfo = getQueueInfo();
List<QueueInfo> childQueuesInfo = new ArrayList<QueueInfo>();
if (includeChildQueues) {
[17/50] [abbrv] hadoop git commit: HDFS-8480. Fix performance and
timeout issues in HDFS-7929 by using hard-links to preserve old edit logs,
instead of copying them. (Zhe Zhang via Colin P. McCabe)
Posted by sj...@apache.org.
HDFS-8480. Fix performance and timeout issues in HDFS-7929 by using hard-links to preserve old edit logs, instead of copying them. (Zhe Zhang via Colin P. McCabe)
(cherry picked from commit 7b424f938c3c306795d574792b086d84e4f06425)
(cherry picked from commit cbd11681ce8a51d187d91748b67a708681e599de)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e86d1f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e86d1f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e86d1f2
Branch: refs/heads/branch-2.6
Commit: 2e86d1f25469af34e92a1eebd1f84b13a0261ee4
Parents: 4363145
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 22 14:37:10 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 16:12:45 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 4 ++
.../hdfs/server/namenode/NNUpgradeUtil.java | 18 +------
.../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 56 +++++++++++++++-----
3 files changed, 49 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e86d1f2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1f6ce36..622fb16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -29,6 +29,10 @@ Release 2.6.1 - UNRELEASED
OPTIMIZATIONS
+ HDFS-8480. Fix performance and timeout issues in HDFS-7929 by using
+ hard-links to preserve old edit logs, instead of copying them. (Zhe Zhang
+ via Colin P. McCabe)
+
BUG FIXES
HDFS-7425. NameNode block deletion logging uses incorrect appender.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e86d1f2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
index c01b11d..a4d9580 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NNUpgradeUtil.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.File;
import java.io.FilenameFilter;
import java.io.IOException;
+import java.nio.file.Files;
import java.util.List;
import org.apache.commons.logging.Log;
@@ -130,23 +131,8 @@ public abstract class NNUpgradeUtil {
for (String s : fileNameList) {
File prevFile = new File(tmpDir, s);
- Preconditions.checkState(prevFile.canRead(),
- "Edits log file " + s + " is not readable.");
File newFile = new File(curDir, prevFile.getName());
- Preconditions.checkState(newFile.createNewFile(),
- "Cannot create new edits log file in " + curDir);
- EditLogFileInputStream in = new EditLogFileInputStream(prevFile);
- EditLogFileOutputStream out =
- new EditLogFileOutputStream(conf, newFile, 512*1024);
- FSEditLogOp logOp = in.nextValidOp();
- while (logOp != null) {
- out.write(logOp);
- logOp = in.nextOp();
- }
- out.setReadyToFlush();
- out.flushAndSync(true);
- out.close();
- in.close();
+ Files.createLink(newFile.toPath(), prevFile.toPath());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e86d1f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
index 49762ff..e5777c7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
@@ -32,13 +32,17 @@ import java.io.FilenameFilter;
import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
+import java.nio.file.Files;
import java.util.List;
import java.util.regex.Pattern;
+import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
@@ -46,7 +50,11 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
+import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
+import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
import org.apache.hadoop.hdfs.server.namenode.NNStorage;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
@@ -54,6 +62,8 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
+import static org.apache.hadoop.hdfs.inotify.Event.CreateEvent;
+
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -466,29 +476,49 @@ public class TestDFSUpgrade {
log("Normal NameNode upgrade", 1);
File[] created =
UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
- List<String> beforeUpgrade = new LinkedList<String>();
for (final File createdDir : created) {
String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE);
- Collections.addAll(beforeUpgrade, fileNameList);
+ for (String fileName : fileNameList) {
+ String tmpFileName = fileName + ".tmp";
+ File existingFile = new File(createdDir, fileName);
+ File tmpFile = new File(createdDir, tmpFileName);
+ Files.move(existingFile.toPath(), tmpFile.toPath());
+ File newFile = new File(createdDir, fileName);
+ Preconditions.checkState(newFile.createNewFile(),
+ "Cannot create new edits log file in " + createdDir);
+ EditLogFileInputStream in = new EditLogFileInputStream(tmpFile,
+ HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID,
+ false);
+ EditLogFileOutputStream out = new EditLogFileOutputStream(conf, newFile,
+ (int)tmpFile.length());
+ out.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION + 1);
+ FSEditLogOp logOp = in.readOp();
+ while (logOp != null) {
+ out.write(logOp);
+ logOp = in.readOp();
+ }
+ out.setReadyToFlush();
+ out.flushAndSync(true);
+ out.close();
+ Files.delete(tmpFile.toPath());
+ }
}
cluster = createCluster();
- List<String> afterUpgrade = new LinkedList<String>();
- for (final File createdDir : created) {
- String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE);
- Collections.addAll(afterUpgrade, fileNameList);
- }
-
- for (String s : beforeUpgrade) {
- assertTrue(afterUpgrade.contains(s));
- }
-
+ DFSInotifyEventInputStream ieis =
+ cluster.getFileSystem().getInotifyEventStream(0);
+ EventBatch batch = ieis.poll();
+ Event[] events = batch.getEvents();
+ assertTrue("Should be able to get transactions before the upgrade.",
+ events.length > 0);
+ assertEquals(events[0].getEventType(), Event.EventType.CREATE);
+ assertEquals(((CreateEvent) events[0]).getPath(), "/TestUpgrade");
cluster.shutdown();
UpgradeUtilities.createEmptyDirs(nameNodeDirs);
}
- private static enum EditLogsFilter implements FilenameFilter {
+ private enum EditLogsFilter implements FilenameFilter {
INSTANCE;
@Override
[16/50] [abbrv] hadoop git commit: HDFS-7446. HDFS inotify should
have the ability to determine what txid it has read up to (cmccabe) (cherry
picked from commit 75a326aaff8c92349701d9b3473c3070b8c2be44)
Posted by sj...@apache.org.
HDFS-7446. HDFS inotify should have the ability to determine what txid it has read up to (cmccabe)
(cherry picked from commit 75a326aaff8c92349701d9b3473c3070b8c2be44)
(cherry picked from commit 06552a15d5172a2b0ad3d61aa7f9a849857385aa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43631451
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43631451
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43631451
Branch: refs/heads/branch-2.6
Commit: 4363145128f91b2fb1f1c0254ee5e8621a1ac383
Parents: 8ed162b
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Nov 25 17:44:34 2014 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 15:32:17 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 65 ++--
.../apache/hadoop/hdfs/inotify/EventBatch.java | 41 +++
.../hadoop/hdfs/inotify/EventBatchList.java | 63 ++++
.../apache/hadoop/hdfs/inotify/EventsList.java | 63 ----
.../hadoop/hdfs/protocol/ClientProtocol.java | 8 +-
.../ClientNamenodeProtocolTranslatorPB.java | 4 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 341 ++++++++++---------
.../namenode/InotifyFSEditLogOpTranslator.java | 74 ++--
.../hdfs/server/namenode/NameNodeRpcServer.java | 45 +--
.../hadoop-hdfs/src/main/proto/inotify.proto | 10 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 209 +++++++-----
12 files changed, 526 insertions(+), 400 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d2f07c2..1f6ce36 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -156,6 +156,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7980. Incremental BlockReport will dramatically slow down namenode
startup. (Walter Su via szetszwo)
+ HDFS-7446. HDFS inotify should have the ability to determine what txid it
+ has read up to (cmccabe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
index 73c5f55..83b92b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInotifyEventInputStream.java
@@ -19,11 +19,10 @@
package org.apache.hadoop.hdfs;
import com.google.common.collect.Iterators;
-import com.google.common.util.concurrent.UncheckedExecutionException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.inotify.MissingEventsException;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.util.Time;
@@ -33,13 +32,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Iterator;
import java.util.Random;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
/**
* Stream for reading inotify events. DFSInotifyEventInputStreams should not
@@ -52,7 +45,7 @@ public class DFSInotifyEventInputStream {
.class);
private final ClientProtocol namenode;
- private Iterator<Event> it;
+ private Iterator<EventBatch> it;
private long lastReadTxid;
/**
* The most recent txid the NameNode told us it has sync'ed -- helps us
@@ -78,22 +71,22 @@ public class DFSInotifyEventInputStream {
}
/**
- * Returns the next event in the stream or null if no new events are currently
- * available.
+ * Returns the next batch of events in the stream or null if no new
+ * batches are currently available.
*
* @throws IOException because of network error or edit log
* corruption. Also possible if JournalNodes are unresponsive in the
* QJM setting (even one unresponsive JournalNode is enough in rare cases),
* so catching this exception and retrying at least a few times is
* recommended.
- * @throws MissingEventsException if we cannot return the next event in the
- * stream because the data for the event (and possibly some subsequent events)
- * has been deleted (generally because this stream is a very large number of
- * events behind the current state of the NameNode). It is safe to continue
- * reading from the stream after this exception is thrown -- the next
- * available event will be returned.
+ * @throws MissingEventsException if we cannot return the next batch in the
+ * stream because the data for the events (and possibly some subsequent
+ * events) has been deleted (generally because this stream is a very large
+ * number of transactions behind the current state of the NameNode). It is
+ * safe to continue reading from the stream after this exception is thrown
+ * The next available batch of events will be returned.
*/
- public Event poll() throws IOException, MissingEventsException {
+ public EventBatch poll() throws IOException, MissingEventsException {
// need to keep retrying until the NN sends us the latest committed txid
if (lastReadTxid == -1) {
LOG.debug("poll(): lastReadTxid is -1, reading current txid from NN");
@@ -101,14 +94,14 @@ public class DFSInotifyEventInputStream {
return null;
}
if (!it.hasNext()) {
- EventsList el = namenode.getEditsFromTxid(lastReadTxid + 1);
+ EventBatchList el = namenode.getEditsFromTxid(lastReadTxid + 1);
if (el.getLastTxid() != -1) {
// we only want to set syncTxid when we were actually able to read some
// edits on the NN -- otherwise it will seem like edits are being
// generated faster than we can read them when the problem is really
// that we are temporarily unable to read edits
syncTxid = el.getSyncTxid();
- it = el.getEvents().iterator();
+ it = el.getBatches().iterator();
long formerLastReadTxid = lastReadTxid;
lastReadTxid = el.getLastTxid();
if (el.getFirstTxid() != formerLastReadTxid + 1) {
@@ -131,18 +124,18 @@ public class DFSInotifyEventInputStream {
}
/**
- * Return a estimate of how many events behind the NameNode's current state
- * this stream is. Clients should periodically call this method and check if
- * its result is steadily increasing, which indicates that they are falling
- * behind (i.e. events are being generated faster than the client is reading
- * them). If a client falls too far behind events may be deleted before the
- * client can read them.
+ * Return a estimate of how many transaction IDs behind the NameNode's
+ * current state this stream is. Clients should periodically call this method
+ * and check if its result is steadily increasing, which indicates that they
+ * are falling behind (i.e. transaction are being generated faster than the
+ * client is reading them). If a client falls too far behind events may be
+ * deleted before the client can read them.
* <p/>
* A return value of -1 indicates that an estimate could not be produced, and
* should be ignored. The value returned by this method is really only useful
* when compared to previous or subsequent returned values.
*/
- public long getEventsBehindEstimate() {
+ public long getTxidsBehindEstimate() {
if (syncTxid == 0) {
return -1;
} else {
@@ -155,8 +148,8 @@ public class DFSInotifyEventInputStream {
}
/**
- * Returns the next event in the stream, waiting up to the specified amount of
- * time for a new event. Returns null if a new event is not available at the
+ * Returns the next event batch in the stream, waiting up to the specified
+ * amount of time for a new batch. Returns null if one is not available at the
* end of the specified amount of time. The time before the method returns may
* exceed the specified amount of time by up to the time required for an RPC
* to the NameNode.
@@ -168,12 +161,12 @@ public class DFSInotifyEventInputStream {
* see {@link DFSInotifyEventInputStream#poll()}
* @throws InterruptedException if the calling thread is interrupted
*/
- public Event poll(long time, TimeUnit tu) throws IOException,
+ public EventBatch poll(long time, TimeUnit tu) throws IOException,
InterruptedException, MissingEventsException {
long initialTime = Time.monotonicNow();
long totalWait = TimeUnit.MILLISECONDS.convert(time, tu);
long nextWait = INITIAL_WAIT_MS;
- Event next = null;
+ EventBatch next = null;
while ((next = poll()) == null) {
long timeLeft = totalWait - (Time.monotonicNow() - initialTime);
if (timeLeft <= 0) {
@@ -193,17 +186,17 @@ public class DFSInotifyEventInputStream {
}
/**
- * Returns the next event in the stream, waiting indefinitely if a new event
- * is not immediately available.
+ * Returns the next batch of events in the stream, waiting indefinitely if
+ * a new batch is not immediately available.
*
* @throws IOException see {@link DFSInotifyEventInputStream#poll()}
* @throws MissingEventsException see
* {@link DFSInotifyEventInputStream#poll()}
* @throws InterruptedException if the calling thread is interrupted
*/
- public Event take() throws IOException, InterruptedException,
+ public EventBatch take() throws IOException, InterruptedException,
MissingEventsException {
- Event next = null;
+ EventBatch next = null;
int nextWaitMin = INITIAL_WAIT_MS;
while ((next = poll()) == null) {
// sleep for a random period between nextWaitMin and nextWaitMin * 2
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java
new file mode 100644
index 0000000..0ad1070
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatch.java
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.inotify;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * A batch of events that all happened on the same transaction ID.
+ */
+@InterfaceAudience.Public
+public class EventBatch {
+ private final long txid;
+ private final Event[] events;
+
+ public EventBatch(long txid, Event[] events) {
+ this.txid = txid;
+ this.events = events;
+ }
+
+ public long getTxid() {
+ return txid;
+ }
+
+ public Event[] getEvents() { return events; }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java
new file mode 100644
index 0000000..9c97038
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventBatchList.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.inotify;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+import java.util.List;
+
+/**
+ * Contains a list of event batches, the transaction ID in the edit log up to
+ * which we read to produce these events, and the first txid we observed when
+ * producing these events (the last of which is for the purpose of determining
+ * whether we have missed events due to edit deletion). Also contains the most
+ * recent txid that the NameNode has sync'ed, so the client can determine how
+ * far behind in the edit log it is.
+ */
+@InterfaceAudience.Private
+public class EventBatchList {
+ private List<EventBatch> batches;
+ private long firstTxid;
+ private long lastTxid;
+ private long syncTxid;
+
+ public EventBatchList(List<EventBatch> batches, long firstTxid,
+ long lastTxid, long syncTxid) {
+ this.batches = batches;
+ this.firstTxid = firstTxid;
+ this.lastTxid = lastTxid;
+ this.syncTxid = syncTxid;
+ }
+
+ public List<EventBatch> getBatches() {
+ return batches;
+ }
+
+ public long getFirstTxid() {
+ return firstTxid;
+ }
+
+ public long getLastTxid() {
+ return lastTxid;
+ }
+
+ public long getSyncTxid() {
+ return syncTxid;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java
deleted file mode 100644
index 6d02d3c..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/EventsList.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.inotify;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-import java.util.List;
-
-/**
- * Contains a set of events, the transaction ID in the edit log up to which we
- * read to produce these events, and the first txid we observed when producing
- * these events (the last of which is for the purpose of determining whether we
- * have missed events due to edit deletion). Also contains the most recent txid
- * that the NameNode has sync'ed, so the client can determine how far behind in
- * the edit log it is.
- */
-@InterfaceAudience.Private
-public class EventsList {
- private List<Event> events;
- private long firstTxid;
- private long lastTxid;
- private long syncTxid;
-
- public EventsList(List<Event> events, long firstTxid, long lastTxid,
- long syncTxid) {
- this.events = events;
- this.firstTxid = firstTxid;
- this.lastTxid = lastTxid;
- this.syncTxid = syncTxid;
- }
-
- public List<Event> getEvents() {
- return events;
- }
-
- public long getFirstTxid() {
- return firstTxid;
- }
-
- public long getLastTxid() {
- return lastTxid;
- }
-
- public long getSyncTxid() {
- return syncTxid;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index d29d2eb..f3a390a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -1405,9 +1405,9 @@ public interface ClientProtocol {
public long getCurrentEditLogTxid() throws IOException;
/**
- * Get an ordered list of events corresponding to the edit log transactions
- * from txid onwards.
+ * Get an ordered list of batches of events corresponding to the edit log
+ * transactions for txids equal to or greater than txid.
*/
@Idempotent
- public EventsList getEditsFromTxid(long txid) throws IOException;
+ public EventBatchList getEditsFromTxid(long txid) throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 077a3e9..7a2dd15 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -44,7 +44,7 @@ import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
@@ -1480,7 +1480,7 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
- public EventsList getEditsFromTxid(long txid) throws IOException {
+ public EventBatchList getEditsFromTxid(long txid) throws IOException {
GetEditsFromTxidRequestProto req = GetEditsFromTxidRequestProto.newBuilder()
.setTxid(txid).build();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index ae9cb3e..c52588f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -46,11 +46,12 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.proto.HAServiceProtocolProtos;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveEntry;
import org.apache.hadoop.hdfs.protocol.CacheDirectiveInfo;
@@ -2516,173 +2517,197 @@ public class PBHelper {
}
}
- public static EventsList convert(GetEditsFromTxidResponseProto resp) throws
+ public static EventBatchList convert(GetEditsFromTxidResponseProto resp) throws
IOException {
- List<Event> events = Lists.newArrayList();
- for (InotifyProtos.EventProto p : resp.getEventsList().getEventsList()) {
- switch(p.getType()) {
- case EVENT_CLOSE:
- InotifyProtos.CloseEventProto close =
- InotifyProtos.CloseEventProto.parseFrom(p.getContents());
- events.add(new Event.CloseEvent(close.getPath(), close.getFileSize(),
- close.getTimestamp()));
- break;
- case EVENT_CREATE:
- InotifyProtos.CreateEventProto create =
- InotifyProtos.CreateEventProto.parseFrom(p.getContents());
- events.add(new Event.CreateEvent.Builder()
- .iNodeType(createTypeConvert(create.getType()))
- .path(create.getPath())
- .ctime(create.getCtime())
- .ownerName(create.getOwnerName())
- .groupName(create.getGroupName())
- .perms(convert(create.getPerms()))
- .replication(create.getReplication())
- .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
- create.getSymlinkTarget())
- .overwrite(create.getOverwrite()).build());
- break;
- case EVENT_METADATA:
- InotifyProtos.MetadataUpdateEventProto meta =
- InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
- events.add(new Event.MetadataUpdateEvent.Builder()
- .path(meta.getPath())
- .metadataType(metadataUpdateTypeConvert(meta.getType()))
- .mtime(meta.getMtime())
- .atime(meta.getAtime())
- .replication(meta.getReplication())
- .ownerName(
- meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
- .groupName(
- meta.getGroupName().isEmpty() ? null : meta.getGroupName())
- .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
- .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
- meta.getAclsList()))
- .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
- meta.getXAttrsList()))
- .xAttrsRemoved(meta.getXAttrsRemoved())
- .build());
- break;
- case EVENT_RENAME:
- InotifyProtos.RenameEventProto rename =
- InotifyProtos.RenameEventProto.parseFrom(p.getContents());
- events.add(new Event.RenameEvent(rename.getSrcPath(), rename.getDestPath(),
- rename.getTimestamp()));
- break;
- case EVENT_APPEND:
- InotifyProtos.AppendEventProto reopen =
- InotifyProtos.AppendEventProto.parseFrom(p.getContents());
- events.add(new Event.AppendEvent(reopen.getPath()));
- break;
- case EVENT_UNLINK:
- InotifyProtos.UnlinkEventProto unlink =
- InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
- events.add(new Event.UnlinkEvent(unlink.getPath(), unlink.getTimestamp()));
- break;
- default:
- throw new RuntimeException("Unexpected inotify event type: " +
- p.getType());
+ final InotifyProtos.EventsListProto list = resp.getEventsList();
+ final long firstTxid = list.getFirstTxid();
+ final long lastTxid = list.getLastTxid();
+
+ List<EventBatch> batches = Lists.newArrayList();
+ if (list.getEventsList().size() > 0) {
+ throw new IOException("Can't handle old inotify server response.");
+ }
+ for (InotifyProtos.EventBatchProto bp : list.getBatchList()) {
+ long txid = bp.getTxid();
+ if ((txid != -1) && ((txid < firstTxid) || (txid > lastTxid))) {
+ throw new IOException("Error converting TxidResponseProto: got a " +
+ "transaction id " + txid + " that was outside the range of [" +
+ firstTxid + ", " + lastTxid + "].");
+ }
+ List<Event> events = Lists.newArrayList();
+ for (InotifyProtos.EventProto p : bp.getEventsList()) {
+ switch (p.getType()) {
+ case EVENT_CLOSE:
+ InotifyProtos.CloseEventProto close =
+ InotifyProtos.CloseEventProto.parseFrom(p.getContents());
+ events.add(new Event.CloseEvent(close.getPath(),
+ close.getFileSize(), close.getTimestamp()));
+ break;
+ case EVENT_CREATE:
+ InotifyProtos.CreateEventProto create =
+ InotifyProtos.CreateEventProto.parseFrom(p.getContents());
+ events.add(new Event.CreateEvent.Builder()
+ .iNodeType(createTypeConvert(create.getType()))
+ .path(create.getPath())
+ .ctime(create.getCtime())
+ .ownerName(create.getOwnerName())
+ .groupName(create.getGroupName())
+ .perms(convert(create.getPerms()))
+ .replication(create.getReplication())
+ .symlinkTarget(create.getSymlinkTarget().isEmpty() ? null :
+ create.getSymlinkTarget())
+ .overwrite(create.getOverwrite()).build());
+ break;
+ case EVENT_METADATA:
+ InotifyProtos.MetadataUpdateEventProto meta =
+ InotifyProtos.MetadataUpdateEventProto.parseFrom(p.getContents());
+ events.add(new Event.MetadataUpdateEvent.Builder()
+ .path(meta.getPath())
+ .metadataType(metadataUpdateTypeConvert(meta.getType()))
+ .mtime(meta.getMtime())
+ .atime(meta.getAtime())
+ .replication(meta.getReplication())
+ .ownerName(
+ meta.getOwnerName().isEmpty() ? null : meta.getOwnerName())
+ .groupName(
+ meta.getGroupName().isEmpty() ? null : meta.getGroupName())
+ .perms(meta.hasPerms() ? convert(meta.getPerms()) : null)
+ .acls(meta.getAclsList().isEmpty() ? null : convertAclEntry(
+ meta.getAclsList()))
+ .xAttrs(meta.getXAttrsList().isEmpty() ? null : convertXAttrs(
+ meta.getXAttrsList()))
+ .xAttrsRemoved(meta.getXAttrsRemoved())
+ .build());
+ break;
+ case EVENT_RENAME:
+ InotifyProtos.RenameEventProto rename =
+ InotifyProtos.RenameEventProto.parseFrom(p.getContents());
+ events.add(new Event.RenameEvent(rename.getSrcPath(),
+ rename.getDestPath(), rename.getTimestamp()));
+ break;
+ case EVENT_APPEND:
+ InotifyProtos.AppendEventProto reopen =
+ InotifyProtos.AppendEventProto.parseFrom(p.getContents());
+ events.add(new Event.AppendEvent(reopen.getPath()));
+ break;
+ case EVENT_UNLINK:
+ InotifyProtos.UnlinkEventProto unlink =
+ InotifyProtos.UnlinkEventProto.parseFrom(p.getContents());
+ events.add(new Event.UnlinkEvent(unlink.getPath(),
+ unlink.getTimestamp()));
+ break;
+ default:
+ throw new RuntimeException("Unexpected inotify event type: " +
+ p.getType());
+ }
}
+ batches.add(new EventBatch(txid, events.toArray(new Event[0])));
}
- return new EventsList(events, resp.getEventsList().getFirstTxid(),
+ return new EventBatchList(batches, resp.getEventsList().getFirstTxid(),
resp.getEventsList().getLastTxid(), resp.getEventsList().getSyncTxid());
}
- public static GetEditsFromTxidResponseProto convertEditsResponse(EventsList el) {
+ public static GetEditsFromTxidResponseProto convertEditsResponse(EventBatchList el) {
InotifyProtos.EventsListProto.Builder builder =
InotifyProtos.EventsListProto.newBuilder();
- for (Event e : el.getEvents()) {
- switch(e.getEventType()) {
- case CLOSE:
- Event.CloseEvent ce = (Event.CloseEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_CLOSE)
- .setContents(
- InotifyProtos.CloseEventProto.newBuilder()
- .setPath(ce.getPath())
- .setFileSize(ce.getFileSize())
- .setTimestamp(ce.getTimestamp()).build().toByteString()
- ).build());
- break;
- case CREATE:
- Event.CreateEvent ce2 = (Event.CreateEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_CREATE)
- .setContents(
- InotifyProtos.CreateEventProto.newBuilder()
- .setType(createTypeConvert(ce2.getiNodeType()))
- .setPath(ce2.getPath())
- .setCtime(ce2.getCtime())
- .setOwnerName(ce2.getOwnerName())
- .setGroupName(ce2.getGroupName())
- .setPerms(convert(ce2.getPerms()))
- .setReplication(ce2.getReplication())
- .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
- "" : ce2.getSymlinkTarget())
- .setOverwrite(ce2.getOverwrite()).build().toByteString()
- ).build());
- break;
- case METADATA:
- Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
- InotifyProtos.MetadataUpdateEventProto.Builder metaB =
- InotifyProtos.MetadataUpdateEventProto.newBuilder()
- .setPath(me.getPath())
- .setType(metadataUpdateTypeConvert(me.getMetadataType()))
- .setMtime(me.getMtime())
- .setAtime(me.getAtime())
- .setReplication(me.getReplication())
- .setOwnerName(me.getOwnerName() == null ? "" :
- me.getOwnerName())
- .setGroupName(me.getGroupName() == null ? "" :
- me.getGroupName())
- .addAllAcls(me.getAcls() == null ?
- Lists.<AclEntryProto>newArrayList() :
- convertAclEntryProto(me.getAcls()))
- .addAllXAttrs(me.getxAttrs() == null ?
- Lists.<XAttrProto>newArrayList() :
- convertXAttrProto(me.getxAttrs()))
- .setXAttrsRemoved(me.isxAttrsRemoved());
- if (me.getPerms() != null) {
- metaB.setPerms(convert(me.getPerms()));
+ for (EventBatch b : el.getBatches()) {
+ List<InotifyProtos.EventProto> events = Lists.newArrayList();
+ for (Event e : b.getEvents()) {
+ switch (e.getEventType()) {
+ case CLOSE:
+ Event.CloseEvent ce = (Event.CloseEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_CLOSE)
+ .setContents(
+ InotifyProtos.CloseEventProto.newBuilder()
+ .setPath(ce.getPath())
+ .setFileSize(ce.getFileSize())
+ .setTimestamp(ce.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case CREATE:
+ Event.CreateEvent ce2 = (Event.CreateEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_CREATE)
+ .setContents(
+ InotifyProtos.CreateEventProto.newBuilder()
+ .setType(createTypeConvert(ce2.getiNodeType()))
+ .setPath(ce2.getPath())
+ .setCtime(ce2.getCtime())
+ .setOwnerName(ce2.getOwnerName())
+ .setGroupName(ce2.getGroupName())
+ .setPerms(convert(ce2.getPerms()))
+ .setReplication(ce2.getReplication())
+ .setSymlinkTarget(ce2.getSymlinkTarget() == null ?
+ "" : ce2.getSymlinkTarget())
+ .setOverwrite(ce2.getOverwrite()).build().toByteString()
+ ).build());
+ break;
+ case METADATA:
+ Event.MetadataUpdateEvent me = (Event.MetadataUpdateEvent) e;
+ InotifyProtos.MetadataUpdateEventProto.Builder metaB =
+ InotifyProtos.MetadataUpdateEventProto.newBuilder()
+ .setPath(me.getPath())
+ .setType(metadataUpdateTypeConvert(me.getMetadataType()))
+ .setMtime(me.getMtime())
+ .setAtime(me.getAtime())
+ .setReplication(me.getReplication())
+ .setOwnerName(me.getOwnerName() == null ? "" :
+ me.getOwnerName())
+ .setGroupName(me.getGroupName() == null ? "" :
+ me.getGroupName())
+ .addAllAcls(me.getAcls() == null ?
+ Lists.<AclEntryProto>newArrayList() :
+ convertAclEntryProto(me.getAcls()))
+ .addAllXAttrs(me.getxAttrs() == null ?
+ Lists.<XAttrProto>newArrayList() :
+ convertXAttrProto(me.getxAttrs()))
+ .setXAttrsRemoved(me.isxAttrsRemoved());
+ if (me.getPerms() != null) {
+ metaB.setPerms(convert(me.getPerms()));
+ }
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_METADATA)
+ .setContents(metaB.build().toByteString())
+ .build());
+ break;
+ case RENAME:
+ Event.RenameEvent re = (Event.RenameEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_RENAME)
+ .setContents(
+ InotifyProtos.RenameEventProto.newBuilder()
+ .setSrcPath(re.getSrcPath())
+ .setDestPath(re.getDstPath())
+ .setTimestamp(re.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ case APPEND:
+ Event.AppendEvent re2 = (Event.AppendEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_APPEND)
+ .setContents(
+ InotifyProtos.AppendEventProto.newBuilder()
+ .setPath(re2.getPath()).build().toByteString()
+ ).build());
+ break;
+ case UNLINK:
+ Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
+ events.add(InotifyProtos.EventProto.newBuilder()
+ .setType(InotifyProtos.EventType.EVENT_UNLINK)
+ .setContents(
+ InotifyProtos.UnlinkEventProto.newBuilder()
+ .setPath(ue.getPath())
+ .setTimestamp(ue.getTimestamp()).build().toByteString()
+ ).build());
+ break;
+ default:
+ throw new RuntimeException("Unexpected inotify event: " + e);
}
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_METADATA)
- .setContents(metaB.build().toByteString())
- .build());
- break;
- case RENAME:
- Event.RenameEvent re = (Event.RenameEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_RENAME)
- .setContents(
- InotifyProtos.RenameEventProto.newBuilder()
- .setSrcPath(re.getSrcPath())
- .setDestPath(re.getDstPath())
- .setTimestamp(re.getTimestamp()).build().toByteString()
- ).build());
- break;
- case APPEND:
- Event.AppendEvent re2 = (Event.AppendEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_APPEND)
- .setContents(
- InotifyProtos.AppendEventProto.newBuilder()
- .setPath(re2.getPath()).build().toByteString()
- ).build());
- break;
- case UNLINK:
- Event.UnlinkEvent ue = (Event.UnlinkEvent) e;
- builder.addEvents(InotifyProtos.EventProto.newBuilder()
- .setType(InotifyProtos.EventType.EVENT_UNLINK)
- .setContents(
- InotifyProtos.UnlinkEventProto.newBuilder()
- .setPath(ue.getPath())
- .setTimestamp(ue.getTimestamp()).build().toByteString()
- ).build());
- break;
- default:
- throw new RuntimeException("Unexpected inotify event: " + e);
}
+ builder.addBatch(InotifyProtos.EventBatchProto.newBuilder().
+ setTxid(b.getTxid()).
+ addAllEvents(events));
}
builder.setFirstTxid(el.getFirstTxid());
builder.setLastTxid(el.getLastTxid());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
index 00a5f25..cd3fc23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/InotifyFSEditLogOpTranslator.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.collect.Lists;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.Block;
import java.util.List;
@@ -39,32 +40,35 @@ public class InotifyFSEditLogOpTranslator {
return size;
}
- public static Event[] translate(FSEditLogOp op) {
+ public static EventBatch translate(FSEditLogOp op) {
switch(op.opCode) {
case OP_ADD:
FSEditLogOp.AddOp addOp = (FSEditLogOp.AddOp) op;
if (addOp.blocks.length == 0) { // create
- return new Event[] { new Event.CreateEvent.Builder().path(addOp.path)
+ return new EventBatch(op.txid,
+ new Event[] { new Event.CreateEvent.Builder().path(addOp.path)
.ctime(addOp.atime)
.replication(addOp.replication)
.ownerName(addOp.permissions.getUserName())
.groupName(addOp.permissions.getGroupName())
.perms(addOp.permissions.getPermission())
.overwrite(addOp.overwrite)
- .iNodeType(Event.CreateEvent.INodeType.FILE).build() };
+ .iNodeType(Event.CreateEvent.INodeType.FILE).build() });
} else {
- return new Event[] { new Event.AppendEvent(addOp.path) };
+ return new EventBatch(op.txid,
+ new Event[] { new Event.AppendEvent(addOp.path) });
}
case OP_CLOSE:
FSEditLogOp.CloseOp cOp = (FSEditLogOp.CloseOp) op;
- return new Event[] {
- new Event.CloseEvent(cOp.path, getSize(cOp), cOp.mtime) };
+ return new EventBatch(op.txid, new Event[] {
+ new Event.CloseEvent(cOp.path, getSize(cOp), cOp.mtime) });
case OP_SET_REPLICATION:
FSEditLogOp.SetReplicationOp setRepOp = (FSEditLogOp.SetReplicationOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.REPLICATION)
.path(setRepOp.path)
- .replication(setRepOp.replication).build() };
+ .replication(setRepOp.replication).build() });
case OP_CONCAT_DELETE:
FSEditLogOp.ConcatDeleteOp cdOp = (FSEditLogOp.ConcatDeleteOp) op;
List<Event> events = Lists.newArrayList();
@@ -73,73 +77,83 @@ public class InotifyFSEditLogOpTranslator {
events.add(new Event.UnlinkEvent(src, cdOp.timestamp));
}
events.add(new Event.CloseEvent(cdOp.trg, -1, cdOp.timestamp));
- return events.toArray(new Event[0]);
+ return new EventBatch(op.txid, events.toArray(new Event[0]));
case OP_RENAME_OLD:
FSEditLogOp.RenameOldOp rnOpOld = (FSEditLogOp.RenameOldOp) op;
- return new Event[] {
- new Event.RenameEvent(rnOpOld.src, rnOpOld.dst, rnOpOld.timestamp) };
+ return new EventBatch(op.txid, new Event[] {
+ new Event.RenameEvent(rnOpOld.src,
+ rnOpOld.dst, rnOpOld.timestamp) });
case OP_RENAME:
FSEditLogOp.RenameOp rnOp = (FSEditLogOp.RenameOp) op;
- return new Event[] {
- new Event.RenameEvent(rnOp.src, rnOp.dst, rnOp.timestamp) };
+ return new EventBatch(op.txid, new Event[] {
+ new Event.RenameEvent(rnOp.src, rnOp.dst, rnOp.timestamp) });
case OP_DELETE:
FSEditLogOp.DeleteOp delOp = (FSEditLogOp.DeleteOp) op;
- return new Event[] { new Event.UnlinkEvent(delOp.path, delOp.timestamp) };
+ return new EventBatch(op.txid, new Event[] {
+ new Event.UnlinkEvent(delOp.path, delOp.timestamp) });
case OP_MKDIR:
FSEditLogOp.MkdirOp mkOp = (FSEditLogOp.MkdirOp) op;
- return new Event[] { new Event.CreateEvent.Builder().path(mkOp.path)
+ return new EventBatch(op.txid,
+ new Event[] { new Event.CreateEvent.Builder().path(mkOp.path)
.ctime(mkOp.timestamp)
.ownerName(mkOp.permissions.getUserName())
.groupName(mkOp.permissions.getGroupName())
.perms(mkOp.permissions.getPermission())
- .iNodeType(Event.CreateEvent.INodeType.DIRECTORY).build() };
+ .iNodeType(Event.CreateEvent.INodeType.DIRECTORY).build() });
case OP_SET_PERMISSIONS:
FSEditLogOp.SetPermissionsOp permOp = (FSEditLogOp.SetPermissionsOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.PERMS)
.path(permOp.src)
- .perms(permOp.permissions).build() };
+ .perms(permOp.permissions).build() });
case OP_SET_OWNER:
FSEditLogOp.SetOwnerOp ownOp = (FSEditLogOp.SetOwnerOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.OWNER)
.path(ownOp.src)
- .ownerName(ownOp.username).groupName(ownOp.groupname).build() };
+ .ownerName(ownOp.username).groupName(ownOp.groupname).build() });
case OP_TIMES:
FSEditLogOp.TimesOp timesOp = (FSEditLogOp.TimesOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.TIMES)
.path(timesOp.path)
- .atime(timesOp.atime).mtime(timesOp.mtime).build() };
+ .atime(timesOp.atime).mtime(timesOp.mtime).build() });
case OP_SYMLINK:
FSEditLogOp.SymlinkOp symOp = (FSEditLogOp.SymlinkOp) op;
- return new Event[] { new Event.CreateEvent.Builder().path(symOp.path)
+ return new EventBatch(op.txid,
+ new Event[] { new Event.CreateEvent.Builder().path(symOp.path)
.ctime(symOp.atime)
.ownerName(symOp.permissionStatus.getUserName())
.groupName(symOp.permissionStatus.getGroupName())
.perms(symOp.permissionStatus.getPermission())
.symlinkTarget(symOp.value)
- .iNodeType(Event.CreateEvent.INodeType.SYMLINK).build() };
+ .iNodeType(Event.CreateEvent.INodeType.SYMLINK).build() });
case OP_REMOVE_XATTR:
FSEditLogOp.RemoveXAttrOp rxOp = (FSEditLogOp.RemoveXAttrOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.XATTRS)
.path(rxOp.src)
.xAttrs(rxOp.xAttrs)
- .xAttrsRemoved(true).build() };
+ .xAttrsRemoved(true).build() });
case OP_SET_XATTR:
FSEditLogOp.SetXAttrOp sxOp = (FSEditLogOp.SetXAttrOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.XATTRS)
.path(sxOp.src)
.xAttrs(sxOp.xAttrs)
- .xAttrsRemoved(false).build() };
+ .xAttrsRemoved(false).build() });
case OP_SET_ACL:
FSEditLogOp.SetAclOp saOp = (FSEditLogOp.SetAclOp) op;
- return new Event[] { new Event.MetadataUpdateEvent.Builder()
+ return new EventBatch(op.txid,
+ new Event[] { new Event.MetadataUpdateEvent.Builder()
.metadataType(Event.MetadataUpdateEvent.MetadataType.ACLS)
.path(saOp.src)
- .acls(saOp.aclEntries).build() };
+ .acls(saOp.aclEntries).build() });
default:
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 690d7e1..16cec5c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -34,7 +34,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import com.google.common.collect.Lists;
import org.apache.commons.logging.Log;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.conf.Configuration;
@@ -55,8 +54,8 @@ import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.ha.HAServiceStatus;
import org.apache.hadoop.ha.HealthCheckFailedException;
@@ -67,8 +66,8 @@ import org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
-import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventsList;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
+import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -139,10 +138,16 @@ import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.ipc.WritableRpcEngine;
+import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshProtocolService;
+import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueProtocolService;
+import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB;
+import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
+import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolServerSideTranslatorPB;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -155,19 +160,12 @@ import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolP
import org.apache.hadoop.security.protocolPB.RefreshAuthorizationPolicyProtocolServerSideTranslatorPB;
import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolPB;
import org.apache.hadoop.security.protocolPB.RefreshUserMappingsProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolPB;
-import org.apache.hadoop.ipc.protocolPB.RefreshCallQueueProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ipc.proto.RefreshCallQueueProtocolProtos.RefreshCallQueueProtocolService;
-import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolPB;
-import org.apache.hadoop.ipc.protocolPB.GenericRefreshProtocolServerSideTranslatorPB;
-import org.apache.hadoop.ipc.proto.GenericRefreshProtocolProtos.GenericRefreshProtocolService;
import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.tools.proto.GetUserMappingsProtocolProtos.GetUserMappingsProtocolService;
import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolPB;
import org.apache.hadoop.tools.protocolPB.GetUserMappingsProtocolServerSideTranslatorPB;
import org.apache.hadoop.tracing.SpanReceiverInfo;
-import org.apache.hadoop.tracing.TraceAdminPB;
import org.apache.hadoop.tracing.TraceAdminPB.TraceAdminService;
import org.apache.hadoop.tracing.TraceAdminProtocolPB;
import org.apache.hadoop.tracing.TraceAdminProtocolServerSideTranslatorPB;
@@ -175,6 +173,7 @@ import org.apache.hadoop.util.VersionInfo;
import org.apache.hadoop.util.VersionUtil;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.Lists;
import com.google.protobuf.BlockingService;
/**
@@ -1670,7 +1669,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
- public EventsList getEditsFromTxid(long txid) throws IOException {
+ public EventBatchList getEditsFromTxid(long txid) throws IOException {
checkNNStartup();
namesystem.checkOperation(OperationCategory.READ); // only active
namesystem.checkSuperuserPrivilege();
@@ -1689,13 +1688,14 @@ class NameNodeRpcServer implements NamenodeProtocols {
// guaranteed to have been written by this NameNode.)
boolean readInProgress = syncTxid > 0;
- List<Event> events = Lists.newArrayList();
+ List<EventBatch> batches = Lists.newArrayList();
+ int totalEvents = 0;
long maxSeenTxid = -1;
long firstSeenTxid = -1;
if (syncTxid > 0 && txid > syncTxid) {
// we can't read past syncTxid, so there's no point in going any further
- return new EventsList(events, firstSeenTxid, maxSeenTxid, syncTxid);
+ return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
Collection<EditLogInputStream> streams = null;
@@ -1707,7 +1707,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
// will result
LOG.info("NN is transitioning from active to standby and FSEditLog " +
"is closed -- could not read edits");
- return new EventsList(events, firstSeenTxid, maxSeenTxid, syncTxid);
+ return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
boolean breakOuter = false;
@@ -1725,9 +1725,10 @@ class NameNodeRpcServer implements NamenodeProtocols {
break;
}
- Event[] eventsFromOp = InotifyFSEditLogOpTranslator.translate(op);
- if (eventsFromOp != null) {
- events.addAll(Arrays.asList(eventsFromOp));
+ EventBatch eventBatch = InotifyFSEditLogOpTranslator.translate(op);
+ if (eventBatch != null) {
+ batches.add(eventBatch);
+ totalEvents += eventBatch.getEvents().length;
}
if (op.getTransactionId() > maxSeenTxid) {
maxSeenTxid = op.getTransactionId();
@@ -1735,7 +1736,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
if (firstSeenTxid == -1) {
firstSeenTxid = op.getTransactionId();
}
- if (events.size() >= maxEventsPerRPC || (syncTxid > 0 &&
+ if (totalEvents >= maxEventsPerRPC || (syncTxid > 0 &&
op.getTransactionId() == syncTxid)) {
// we're done
breakOuter = true;
@@ -1750,7 +1751,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
}
}
- return new EventsList(events, firstSeenTxid, maxSeenTxid, syncTxid);
+ return new EventBatchList(batches, firstSeenTxid, maxSeenTxid, syncTxid);
}
@Override // TraceAdminProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
index a1d4d92..e51c02c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/inotify.proto
@@ -48,6 +48,11 @@ message EventProto {
required bytes contents = 2;
}
+message EventBatchProto {
+ required int64 txid = 1;
+ repeated EventProto events = 2;
+}
+
enum INodeType {
I_TYPE_FILE = 0x0;
I_TYPE_DIRECTORY = 0x1;
@@ -111,8 +116,9 @@ message UnlinkEventProto {
}
message EventsListProto {
- repeated EventProto events = 1;
+ repeated EventProto events = 1; // deprecated
required int64 firstTxid = 2;
required int64 lastTxid = 3;
required int64 syncTxid = 4;
-}
\ No newline at end of file
+ repeated EventBatchProto batch = 5;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43631451/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index a608ba8..82db110 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.inotify.MissingEventsException;
import org.apache.hadoop.hdfs.qjournal.MiniQJMHACluster;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
@@ -49,11 +50,17 @@ public class TestDFSInotifyEventInputStream {
private static final Log LOG = LogFactory.getLog(
TestDFSInotifyEventInputStream.class);
- private static Event waitForNextEvent(DFSInotifyEventInputStream eis)
+ private static EventBatch waitForNextEvents(DFSInotifyEventInputStream eis)
throws IOException, MissingEventsException {
- Event next = null;
- while ((next = eis.poll()) == null);
- return next;
+ EventBatch batch = null;
+ while ((batch = eis.poll()) == null);
+ return batch;
+ }
+
+ private static long checkTxid(EventBatch batch, long prevTxid){
+ Assert.assertTrue("Previous txid " + prevTxid + " was not less than " +
+ "new txid " + batch.getTxid(), prevTxid < batch.getTxid());
+ return batch.getTxid();
}
/**
@@ -64,7 +71,7 @@ public class TestDFSInotifyEventInputStream {
*/
@Test
public void testOpcodeCount() {
- Assert.assertTrue(FSEditLogOpCodes.values().length == 47);
+ Assert.assertEquals(47, FSEditLogOpCodes.values().length);
}
@@ -127,30 +134,36 @@ public class TestDFSInotifyEventInputStream {
"user::rwx,user:foo:rw-,group::r--,other::---", true));
client.removeAcl("/file5"); // SetAclOp -> MetadataUpdateEvent
- Event next = null;
+ EventBatch batch = null;
// RenameOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.RENAME);
- Event.RenameEvent re = (Event.RenameEvent) next;
- Assert.assertTrue(re.getDstPath().equals("/file4"));
- Assert.assertTrue(re.getSrcPath().equals("/file"));
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ long txid = batch.getTxid();
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ Event.RenameEvent re = (Event.RenameEvent) batch.getEvents()[0];
+ Assert.assertEquals("/file4", re.getDstPath());
+ Assert.assertEquals("/file", re.getSrcPath());
Assert.assertTrue(re.getTimestamp() > 0);
- long eventsBehind = eis.getEventsBehindEstimate();
+ long eventsBehind = eis.getTxidsBehindEstimate();
// RenameOldOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.RENAME);
- Event.RenameEvent re2 = (Event.RenameEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ Event.RenameEvent re2 = (Event.RenameEvent) batch.getEvents()[0];
Assert.assertTrue(re2.getDstPath().equals("/file2"));
Assert.assertTrue(re2.getSrcPath().equals("/file4"));
Assert.assertTrue(re.getTimestamp() > 0);
// AddOp with overwrite
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
- Event.CreateEvent ce = (Event.CreateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ Event.CreateEvent ce = (Event.CreateEvent) batch.getEvents()[0];
Assert.assertTrue(ce.getiNodeType() == Event.CreateEvent.INodeType.FILE);
Assert.assertTrue(ce.getPath().equals("/file2"));
Assert.assertTrue(ce.getCtime() > 0);
@@ -159,66 +172,80 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce.getOverwrite());
// CloseOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CLOSE);
- Event.CloseEvent ce2 = (Event.CloseEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CLOSE);
+ Event.CloseEvent ce2 = (Event.CloseEvent) batch.getEvents()[0];
Assert.assertTrue(ce2.getPath().equals("/file2"));
Assert.assertTrue(ce2.getFileSize() > 0);
Assert.assertTrue(ce2.getTimestamp() > 0);
// AddOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.APPEND);
- Assert.assertTrue(((Event.AppendEvent) next).getPath().equals("/file2"));
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.APPEND);
+ Assert.assertTrue(((Event.AppendEvent) batch.getEvents()[0]).getPath().equals("/file2"));
// CloseOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CLOSE);
- Assert.assertTrue(((Event.CloseEvent) next).getPath().equals("/file2"));
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CLOSE);
+ Assert.assertTrue(((Event.CloseEvent) batch.getEvents()[0]).getPath().equals("/file2"));
// TimesOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue.getPath().equals("/file2"));
Assert.assertTrue(mue.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.TIMES);
// SetReplicationOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue2 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue2 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue2.getPath().equals("/file2"));
Assert.assertTrue(mue2.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.REPLICATION);
Assert.assertTrue(mue2.getReplication() == 1);
// ConcatDeleteOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.APPEND);
- Assert.assertTrue(((Event.AppendEvent) next).getPath().equals("/file2"));
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.UNLINK);
- Event.UnlinkEvent ue2 = (Event.UnlinkEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(3, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.APPEND);
+ Assert.assertTrue(((Event.AppendEvent) batch.getEvents()[0]).getPath().equals("/file2"));
+ Assert.assertTrue(batch.getEvents()[1].getEventType() == Event.EventType.UNLINK);
+ Event.UnlinkEvent ue2 = (Event.UnlinkEvent) batch.getEvents()[1];
Assert.assertTrue(ue2.getPath().equals("/file3"));
Assert.assertTrue(ue2.getTimestamp() > 0);
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CLOSE);
- Event.CloseEvent ce3 = (Event.CloseEvent) next;
+ Assert.assertTrue(batch.getEvents()[2].getEventType() == Event.EventType.CLOSE);
+ Event.CloseEvent ce3 = (Event.CloseEvent) batch.getEvents()[2];
Assert.assertTrue(ce3.getPath().equals("/file2"));
Assert.assertTrue(ce3.getTimestamp() > 0);
// DeleteOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.UNLINK);
- Event.UnlinkEvent ue = (Event.UnlinkEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK);
+ Event.UnlinkEvent ue = (Event.UnlinkEvent) batch.getEvents()[0];
Assert.assertTrue(ue.getPath().equals("/file2"));
Assert.assertTrue(ue.getTimestamp() > 0);
// MkdirOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
- Event.CreateEvent ce4 = (Event.CreateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ Event.CreateEvent ce4 = (Event.CreateEvent) batch.getEvents()[0];
Assert.assertTrue(ce4.getiNodeType() ==
Event.CreateEvent.INodeType.DIRECTORY);
Assert.assertTrue(ce4.getPath().equals("/dir"));
@@ -227,18 +254,22 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce4.getSymlinkTarget() == null);
// SetPermissionsOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue3 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue3 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue3.getPath().equals("/dir"));
Assert.assertTrue(mue3.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.PERMS);
Assert.assertTrue(mue3.getPerms().toString().contains("rw-rw-rw-"));
// SetOwnerOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue4 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue4 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue4.getPath().equals("/dir"));
Assert.assertTrue(mue4.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.OWNER);
@@ -246,9 +277,11 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(mue4.getGroupName().equals("groupname"));
// SymlinkOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
- Event.CreateEvent ce5 = (Event.CreateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ Event.CreateEvent ce5 = (Event.CreateEvent) batch.getEvents()[0];
Assert.assertTrue(ce5.getiNodeType() ==
Event.CreateEvent.INodeType.SYMLINK);
Assert.assertTrue(ce5.getPath().equals("/dir2"));
@@ -257,9 +290,11 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(ce5.getSymlinkTarget().equals("/dir"));
// SetXAttrOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue5 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue5 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue5.getPath().equals("/file5"));
Assert.assertTrue(mue5.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.XATTRS);
@@ -268,9 +303,11 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(!mue5.isxAttrsRemoved());
// RemoveXAttrOp
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue6 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue6 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue6.getPath().equals("/file5"));
Assert.assertTrue(mue6.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.XATTRS);
@@ -279,9 +316,11 @@ public class TestDFSInotifyEventInputStream {
Assert.assertTrue(mue6.isxAttrsRemoved());
// SetAclOp (1)
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue7 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue7 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue7.getPath().equals("/file5"));
Assert.assertTrue(mue7.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.ACLS);
@@ -289,9 +328,11 @@ public class TestDFSInotifyEventInputStream {
AclEntry.parseAclEntry("user::rwx", true)));
// SetAclOp (2)
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.METADATA);
- Event.MetadataUpdateEvent mue8 = (Event.MetadataUpdateEvent) next;
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ txid = checkTxid(batch, txid);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.METADATA);
+ Event.MetadataUpdateEvent mue8 = (Event.MetadataUpdateEvent) batch.getEvents()[0];
Assert.assertTrue(mue8.getPath().equals("/file5"));
Assert.assertTrue(mue8.getMetadataType() ==
Event.MetadataUpdateEvent.MetadataType.ACLS);
@@ -305,7 +346,7 @@ public class TestDFSInotifyEventInputStream {
// and we should not have been behind at all when eventsBehind was set
// either, since there were few enough events that they should have all
// been read to the client during the first poll() call
- Assert.assertTrue(eis.getEventsBehindEstimate() == eventsBehind);
+ Assert.assertTrue(eis.getTxidsBehindEstimate() == eventsBehind);
} finally {
cluster.shutdown();
@@ -329,13 +370,14 @@ public class TestDFSInotifyEventInputStream {
}
cluster.getDfsCluster().shutdownNameNode(0);
cluster.getDfsCluster().transitionToActive(1);
- Event next = null;
+ EventBatch batch = null;
// we can read all of the edits logged by the old active from the new
// active
for (int i = 0; i < 10; i++) {
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
- Assert.assertTrue(((Event.CreateEvent) next).getPath().equals("/dir" +
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
i));
}
Assert.assertTrue(eis.poll() == null);
@@ -369,11 +411,12 @@ public class TestDFSInotifyEventInputStream {
// make sure that the old active can't read any further than the edits
// it logged itself (it has no idea whether the in-progress edits from
// the other writer have actually been committed)
- Event next = null;
+ EventBatch batch = null;
for (int i = 0; i < 10; i++) {
- next = waitForNextEvent(eis);
- Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
- Assert.assertTrue(((Event.CreateEvent) next).getPath().equals("/dir" +
+ batch = waitForNextEvents(eis);
+ Assert.assertEquals(1, batch.getEvents().length);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ Assert.assertTrue(((Event.CreateEvent) batch.getEvents()[0]).getPath().equals("/dir" +
i));
}
Assert.assertTrue(eis.poll() == null);
@@ -414,13 +457,13 @@ public class TestDFSInotifyEventInputStream {
}, 1, TimeUnit.SECONDS);
// a very generous wait period -- the edit will definitely have been
// processed by the time this is up
- Event next = eis.poll(5, TimeUnit.SECONDS);
- Assert.assertTrue(next != null);
- Assert.assertTrue(next.getEventType() == Event.EventType.CREATE);
- Assert.assertTrue(((Event.CreateEvent) next).getPath().equals("/dir"));
+ EventBatch batch = eis.poll(5, TimeUnit.SECONDS);
+ Assert.assertNotNull(batch);
+ Assert.assertEquals(1, batch.getEvents().length);
+ Assert.assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ Assert.assertEquals("/dir", ((Event.CreateEvent) batch.getEvents()[0]).getPath());
} finally {
cluster.shutdown();
}
}
-
}
[07/50] [abbrv] hadoop git commit: YARN-3092. Created a common
ResourceUsage class to track labeled resource usages in Capacity Scheduler.
Contributed by Wangda Tan (cherry picked from commit
6f9fe76918bbc79109653edc6cde85df05148ba3)
Posted by sj...@apache.org.
YARN-3092. Created a common ResourceUsage class to track labeled resource usages in Capacity Scheduler. Contributed by Wangda Tan
(cherry picked from commit 6f9fe76918bbc79109653edc6cde85df05148ba3)
(cherry picked from commit 61b4116b4b3c0eec8f514f079debd88bc757b28e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b0ad5538
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b0ad5538
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b0ad5538
Branch: refs/heads/branch-2.6
Commit: b0ad55384191122af160e3268ef448f5b2bc0780
Parents: 419e18c
Author: Jian He <ji...@apache.org>
Authored: Mon Jan 26 15:21:22 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:19 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/ResourceUsage.java | 332 +++++++++++++++++++
.../scheduler/TestResourceUsage.java | 138 ++++++++
3 files changed, 473 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0ad5538/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index be9e2d5..975e4e1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -16,6 +16,9 @@ Release 2.6.1 - UNRELEASED
YARN-1809. Synchronize RM and TimeLineServer Web-UIs. (Zhijie Shen and
Xuan Gong via jianhe)
+ YARN-3092. Created a common ResourceUsage class to track labeled resource
+ usages in Capacity Scheduler. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0ad5538/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
new file mode 100644
index 0000000..5a4cced
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -0,0 +1,332 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.util.resource.Resources;
+
+/**
+ * Resource Usage by Labels for following fields by label - AM resource (to
+ * enforce max-am-resource-by-label after YARN-2637) - Used resource (includes
+ * AM resource usage) - Reserved resource - Pending resource - Headroom
+ *
+ * This class can be used to track resource usage in queue/user/app.
+ *
+ * And it is thread-safe
+ */
+public class ResourceUsage {
+ private ReadLock readLock;
+ private WriteLock writeLock;
+ private Map<String, UsageByLabel> usages;
+ // short for no-label :)
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+
+ public ResourceUsage() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ usages = new HashMap<String, UsageByLabel>();
+ }
+
+ // Usage enum here to make implement cleaner
+ private enum ResourceType {
+ USED(0), PENDING(1), AMUSED(2), RESERVED(3), HEADROOM(4);
+
+ private int value;
+
+ private ResourceType(int value) {
+ this.value = value;
+ }
+
+ public int getValue() {
+ return this.value;
+ }
+ }
+
+ private static class UsageByLabel {
+ // usage by label, contains all UsageType
+ private Resource[] resArr;
+
+ public UsageByLabel(String label) {
+ resArr = new Resource[ResourceType.values().length];
+ for (int i = 0; i < resArr.length; i++) {
+ resArr[i] = Resource.newInstance(0, 0);
+ }
+ }
+
+ public Resource get(ResourceType type) {
+ return resArr[type.getValue()];
+ }
+
+ public void set(ResourceType type, Resource res) {
+ resArr[type.getValue()] = res;
+ }
+
+ public void inc(ResourceType type, Resource res) {
+ Resources.addTo(resArr[type.getValue()], res);
+ }
+
+ public void dec(ResourceType type, Resource res) {
+ Resources.subtractFrom(resArr[type.getValue()], res);
+ }
+ }
+
+ /*
+ * Used
+ */
+ public Resource getUsed() {
+ return getUsed(NL);
+ }
+
+ public Resource getUsed(String label) {
+ return internalGet(label, ResourceType.USED);
+ }
+
+ public void incUsed(String label, Resource res) {
+ internalInc(label, ResourceType.USED, res);
+ }
+
+ public void incUsed(Resource res) {
+ incUsed(NL, res);
+ }
+
+ public void decUsed(Resource res) {
+ decUsed(NL, res);
+ }
+
+ public void decUsed(String label, Resource res) {
+ internalDec(label, ResourceType.USED, res);
+ }
+
+ public void setUsed(Resource res) {
+ setUsed(NL, res);
+ }
+
+ public void setUsed(String label, Resource res) {
+ internalSet(label, ResourceType.USED, res);
+ }
+
+ /*
+ * Pending
+ */
+ public Resource getPending() {
+ return getPending(NL);
+ }
+
+ public Resource getPending(String label) {
+ return internalGet(label, ResourceType.PENDING);
+ }
+
+ public void incPending(String label, Resource res) {
+ internalInc(label, ResourceType.PENDING, res);
+ }
+
+ public void incPending(Resource res) {
+ incPending(NL, res);
+ }
+
+ public void decPending(Resource res) {
+ decPending(NL, res);
+ }
+
+ public void decPending(String label, Resource res) {
+ internalDec(label, ResourceType.PENDING, res);
+ }
+
+ public void setPending(Resource res) {
+ setPending(NL, res);
+ }
+
+ public void setPending(String label, Resource res) {
+ internalSet(label, ResourceType.PENDING, res);
+ }
+
+ /*
+ * Reserved
+ */
+ public Resource getReserved() {
+ return getReserved(NL);
+ }
+
+ public Resource getReserved(String label) {
+ return internalGet(label, ResourceType.RESERVED);
+ }
+
+ public void incReserved(String label, Resource res) {
+ internalInc(label, ResourceType.RESERVED, res);
+ }
+
+ public void incReserved(Resource res) {
+ incReserved(NL, res);
+ }
+
+ public void decReserved(Resource res) {
+ decReserved(NL, res);
+ }
+
+ public void decReserved(String label, Resource res) {
+ internalDec(label, ResourceType.RESERVED, res);
+ }
+
+ public void setReserved(Resource res) {
+ setReserved(NL, res);
+ }
+
+ public void setReserved(String label, Resource res) {
+ internalSet(label, ResourceType.RESERVED, res);
+ }
+
+ /*
+ * Headroom
+ */
+ public Resource getHeadroom() {
+ return getHeadroom(NL);
+ }
+
+ public Resource getHeadroom(String label) {
+ return internalGet(label, ResourceType.HEADROOM);
+ }
+
+ public void incHeadroom(String label, Resource res) {
+ internalInc(label, ResourceType.HEADROOM, res);
+ }
+
+ public void incHeadroom(Resource res) {
+ incHeadroom(NL, res);
+ }
+
+ public void decHeadroom(Resource res) {
+ decHeadroom(NL, res);
+ }
+
+ public void decHeadroom(String label, Resource res) {
+ internalDec(label, ResourceType.HEADROOM, res);
+ }
+
+ public void setHeadroom(Resource res) {
+ setHeadroom(NL, res);
+ }
+
+ public void setHeadroom(String label, Resource res) {
+ internalSet(label, ResourceType.HEADROOM, res);
+ }
+
+ /*
+ * AM-Used
+ */
+ public Resource getAMUsed() {
+ return getAMUsed(NL);
+ }
+
+ public Resource getAMUsed(String label) {
+ return internalGet(label, ResourceType.AMUSED);
+ }
+
+ public void incAMUsed(String label, Resource res) {
+ internalInc(label, ResourceType.AMUSED, res);
+ }
+
+ public void incAMUsed(Resource res) {
+ incAMUsed(NL, res);
+ }
+
+ public void decAMUsed(Resource res) {
+ decAMUsed(NL, res);
+ }
+
+ public void decAMUsed(String label, Resource res) {
+ internalDec(label, ResourceType.AMUSED, res);
+ }
+
+ public void setAMUsed(Resource res) {
+ setAMUsed(NL, res);
+ }
+
+ public void setAMUsed(String label, Resource res) {
+ internalSet(label, ResourceType.AMUSED, res);
+ }
+
+ private static Resource normalize(Resource res) {
+ if (res == null) {
+ return Resources.none();
+ }
+ return res;
+ }
+
+ private Resource internalGet(String label, ResourceType type) {
+ try {
+ readLock.lock();
+ UsageByLabel usage = usages.get(label);
+ if (null == usage) {
+ return Resources.none();
+ }
+ return normalize(usage.get(type));
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private UsageByLabel getAndAddIfMissing(String label) {
+ if (!usages.containsKey(label)) {
+ UsageByLabel u = new UsageByLabel(label);
+ usages.put(label, u);
+ return u;
+ }
+
+ return usages.get(label);
+ }
+
+ private void internalSet(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.set(type, res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void internalInc(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.inc(type, res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ private void internalDec(String label, ResourceType type, Resource res) {
+ try {
+ writeLock.lock();
+ UsageByLabel usage = getAndAddIfMissing(label);
+ usage.dec(type, res);
+ } finally {
+ writeLock.unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b0ad5538/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
new file mode 100644
index 0000000..b6dfacb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
@@ -0,0 +1,138 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestResourceUsage {
+ private static final Log LOG = LogFactory.getLog(TestResourceUsage.class);
+ private String suffix;
+
+ @Parameterized.Parameters
+ public static Collection<String[]> getParameters() {
+ return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
+ { "Headroom" }, { "Reserved" }, { "AMUsed" } });
+ }
+
+ public TestResourceUsage(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private static void dec(ResourceUsage obj, String suffix, Resource res,
+ String label) throws Exception {
+ executeByName(obj, "dec" + suffix, res, label);
+ }
+
+ private static void inc(ResourceUsage obj, String suffix, Resource res,
+ String label) throws Exception {
+ executeByName(obj, "inc" + suffix, res, label);
+ }
+
+ private static void set(ResourceUsage obj, String suffix, Resource res,
+ String label) throws Exception {
+ executeByName(obj, "set" + suffix, res, label);
+ }
+
+ private static Resource get(ResourceUsage obj, String suffix, String label)
+ throws Exception {
+ return executeByName(obj, "get" + suffix, null, label);
+ }
+
+ // Use reflection to avoid too much avoid code
+ private static Resource executeByName(ResourceUsage obj, String methodName,
+ Resource arg, String label) throws Exception {
+ // We have 4 kinds of method
+ // 1. getXXX() : Resource
+ // 2. getXXX(label) : Resource
+ // 3. set/inc/decXXX(res) : void
+ // 4. set/inc/decXXX(label, res) : void
+ if (methodName.startsWith("get")) {
+ Resource result;
+ if (label == null) {
+ // 1.
+ Method method = ResourceUsage.class.getDeclaredMethod(methodName);
+ result = (Resource) method.invoke(obj);
+ } else {
+ // 2.
+ Method method =
+ ResourceUsage.class.getDeclaredMethod(methodName, String.class);
+ result = (Resource) method.invoke(obj, label);
+ }
+ return result;
+ } else {
+ if (label == null) {
+ // 3.
+ Method method =
+ ResourceUsage.class.getDeclaredMethod(methodName, Resource.class);
+ method.invoke(obj, arg);
+ } else {
+ // 4.
+ Method method =
+ ResourceUsage.class.getDeclaredMethod(methodName, String.class,
+ Resource.class);
+ method.invoke(obj, label, arg);
+ }
+ return null;
+ }
+ }
+
+ private void internalTestModifyAndRead(String label) throws Exception {
+ ResourceUsage usage = new ResourceUsage();
+ Resource res;
+
+ // First get returns 0 always
+ res = get(usage, suffix, label);
+ check(0, 0, res);
+
+ // Add 1,1 should returns 1,1
+ inc(usage, suffix, Resource.newInstance(1, 1), label);
+ check(1, 1, get(usage, suffix, label));
+
+ // Set 2,2
+ set(usage, suffix, Resource.newInstance(2, 2), label);
+ check(2, 2, get(usage, suffix, label));
+
+ // dec 2,2
+ dec(usage, suffix, Resource.newInstance(2, 2), label);
+ check(0, 0, get(usage, suffix, label));
+ }
+
+ void check(int mem, int cpu, Resource res) {
+ Assert.assertEquals(mem, res.getMemory());
+ Assert.assertEquals(cpu, res.getVirtualCores());
+ }
+
+ @Test
+ public void testModifyAndRead() throws Exception {
+ LOG.info("Test - " + suffix);
+ internalTestModifyAndRead(null);
+ internalTestModifyAndRead("label");
+ }
+}
[33/50] [abbrv] hadoop git commit: YARN-3171. Sort by Application id,
AppAttempt and ContainerID doesn't work in ATS / RM web ui.
Contributed by Naganarasimha G R
Posted by sj...@apache.org.
YARN-3171. Sort by Application id, AppAttempt and ContainerID doesn't
work in ATS / RM web ui. Contributed by Naganarasimha G R
(cherry picked from commit 3ff1ba2a7b00fdf06270d00b2193bde4b56b06b3)
(cherry picked from commit b6eb36dbdc1c75f412616426cc7599f485e9b86e)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85ec6eb3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85ec6eb3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85ec6eb3
Branch: refs/heads/branch-2.6
Commit: 85ec6eb37a9a62f62a5c2b7840f49ea8b0bae31c
Parents: f4154bd
Author: Xuan <xg...@apache.org>
Authored: Sun Mar 15 20:26:10 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Fri Sep 11 11:49:11 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/yarn/server/webapp/WebPageUtils.java | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec6eb3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c82c892..cab1f0c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -208,6 +208,9 @@ Release 2.6.1 - 2015-09-09
YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM
web page. (Xuan Gong via zjshen)
+ YARN-3171. Sort by Application id, AppAttempt and ContainerID doesn't work
+ in ATS / RM web ui. (Naganarasimha G R via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85ec6eb3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
index 384a976..5acabf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/WebPageUtils.java
@@ -44,7 +44,7 @@ public class WebPageUtils {
StringBuilder sb = new StringBuilder();
return sb
.append("[\n")
- .append("{'sType':'numeric', 'aTargets': [0]")
+ .append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': " +
(isFairSchedulerPage ? "[6, 7]": "[5, 6]"))
@@ -63,7 +63,7 @@ public class WebPageUtils {
private static String getAttemptsTableColumnDefs() {
StringBuilder sb = new StringBuilder();
- return sb.append("[\n").append("{'sType':'numeric', 'aTargets': [0]")
+ return sb.append("[\n").append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }")
.append("\n, {'sType':'numeric', 'aTargets': [1]")
.append(", 'mRender': renderHadoopDate }]").toString();
@@ -79,7 +79,7 @@ public class WebPageUtils {
private static String getContainersTableColumnDefs() {
StringBuilder sb = new StringBuilder();
- return sb.append("[\n").append("{'sType':'numeric', 'aTargets': [0]")
+ return sb.append("[\n").append("{'sType':'string', 'aTargets': [0]")
.append(", 'mRender': parseHadoopID }]").toString();
}
[48/50] [abbrv] hadoop git commit: YARN-4101. RM should print alert
messages if Zookeeper and Resourcemanager gets connection issue. Contributed
by Xuan Gong
Posted by sj...@apache.org.
YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager gets connection issue. Contributed by Xuan Gong
(cherry picked from commit 214fd1408c21f596d1d15217c11b58b34561aab7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c09bb465
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c09bb465
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c09bb465
Branch: refs/heads/branch-2.6
Commit: c09bb46579631a6e3b8dd50e964e77c9357ea14b
Parents: cc30002
Author: Jian He <ji...@apache.org>
Authored: Wed Sep 2 17:45:23 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 16:26:18 2015 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/ha/ActiveStandbyElector.java | 4 ++++
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/AdminService.java | 9 +++++++++
.../yarn/server/resourcemanager/EmbeddedElectorService.java | 4 ++++
.../yarn/server/resourcemanager/webapp/AboutBlock.java | 2 ++
.../hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java | 5 +++++
.../yarn/server/resourcemanager/webapp/RMWebAppFilter.java | 5 ++++-
.../yarn/server/resourcemanager/webapp/dao/ClusterInfo.java | 6 ++++++
.../server/resourcemanager/webapp/TestRMWebServices.java | 8 ++++++--
9 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
index 123f119..f8f720e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/ActiveStandbyElector.java
@@ -1102,4 +1102,8 @@ public class ActiveStandbyElector implements StatCallback, StringCallback {
((appData == null) ? "null" : StringUtils.byteToHexString(appData)) +
" cb=" + appClient;
}
+
+ public String getHAZookeeperConnectionState() {
+ return this.zkConnectionState.name();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a2e0653..6d016b7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -11,6 +11,9 @@ Release 2.6.2 - UNRELEASED
YARN-4092. Fixed UI redirection to print useful messages when both RMs are
in standby mode. (Xuan Gong via jianhe)
+ YARN-4101. RM should print alert messages if Zookeeper and Resourcemanager
+ gets connection issue. (Xuan Gong via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index 14109f9..1660e4a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -722,4 +722,13 @@ public class AdminService extends CompositeService implements
throw RPCUtil.getRemoteException(ioe);
}
}
+
+ public String getHAZookeeperConnectionState() {
+ if (!rmContext.isHAEnabled()) {
+ return "ResourceManager HA is not enabled.";
+ } else if (!autoFailoverEnabled) {
+ return "Auto Failover is not enabled.";
+ }
+ return this.embeddedElector.getHAZookeeperConnectionState();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
index c7b7768..73bdca0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/EmbeddedElectorService.java
@@ -205,4 +205,8 @@ public class EmbeddedElectorService extends AbstractService
elector.quitElection(false);
elector.joinElection(localActiveNodeInfo);
}
+
+ public String getHAZookeeperConnectionState() {
+ return elector.getHAZookeeperConnectionState();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
index ea5c48a..4225afd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AboutBlock.java
@@ -44,6 +44,8 @@ public class AboutBlock extends HtmlBlock {
_("Cluster ID:", cinfo.getClusterId()).
_("ResourceManager state:", cinfo.getState()).
_("ResourceManager HA state:", cinfo.getHAState()).
+ _("ResourceManager HA zookeeper connection state:",
+ cinfo.getHAZookeeperConnectionState()).
_("ResourceManager RMStateStore:", cinfo.getRMStateStore()).
_("ResourceManager started on:", Times.format(cinfo.getStartedOn())).
_("ResourceManager version:", cinfo.getRMBuildVersion() +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
index 066ba52..2d19822 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebApp.java
@@ -113,4 +113,9 @@ public class RMWebApp extends WebApp implements YarnWebParams {
}
return path;
}
+
+ public String getHAZookeeperConnectionState() {
+ return rm.getRMContext().getRMAdminService()
+ .getHAZookeeperConnectionState();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 61600dd..024a8d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -58,6 +58,7 @@ public class RMWebAppFilter extends GuiceContainer {
private String path;
private static final int BASIC_SLEEP_TIME = 5;
private static final int MAX_SLEEP_TIME = 5 * 60;
+ private static final Random randnum = new Random();
@Inject
public RMWebAppFilter(Injector injector, Configuration conf) {
@@ -126,6 +127,8 @@ public class RMWebAppFilter extends GuiceContainer {
String redirectMsg =
doRetry ? "Can not find any active RM. Will retry in next " + next
+ " seconds." : "There is no active RM right now.";
+ redirectMsg += "\nHA Zookeeper Connection State: "
+ + rmWebApp.getHAZookeeperConnectionState();
PrintWriter out = response.getWriter();
out.println(redirectMsg);
if (doRetry) {
@@ -172,6 +175,6 @@ public class RMWebAppFilter extends GuiceContainer {
private static int calculateExponentialTime(int retries) {
long baseTime = BASIC_SLEEP_TIME * (1L << retries);
- return (int) (baseTime * ((new Random()).nextDouble() + 0.5));
+ return (int) (baseTime * (randnum.nextDouble() + 0.5));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
index b529f21..512a5c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ClusterInfo.java
@@ -43,6 +43,7 @@ public class ClusterInfo {
protected String hadoopVersion;
protected String hadoopBuildVersion;
protected String hadoopVersionBuiltOn;
+ protected String haZooKeeperConnectionState;
public ClusterInfo() {
} // JAXB needs this
@@ -62,6 +63,8 @@ public class ClusterInfo {
this.hadoopVersion = VersionInfo.getVersion();
this.hadoopBuildVersion = VersionInfo.getBuildVersion();
this.hadoopVersionBuiltOn = VersionInfo.getDate();
+ this.haZooKeeperConnectionState =
+ rm.getRMContext().getRMAdminService().getHAZookeeperConnectionState();
}
public String getState() {
@@ -108,4 +111,7 @@ public class ClusterInfo {
return this.startedOn;
}
+ public String getHAZookeeperConnectionState() {
+ return this.haZooKeeperConnectionState;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c09bb465/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
index 9f091d2..bd31ff3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServices.java
@@ -261,6 +261,8 @@ public class TestRMWebServices extends JerseyTest {
WebServicesTestUtils.getXmlLong(element, "startedOn"),
WebServicesTestUtils.getXmlString(element, "state"),
WebServicesTestUtils.getXmlString(element, "haState"),
+ WebServicesTestUtils.getXmlString(
+ element, "haZooKeeperConnectionState"),
WebServicesTestUtils.getXmlString(element, "hadoopVersionBuiltOn"),
WebServicesTestUtils.getXmlString(element, "hadoopBuildVersion"),
WebServicesTestUtils.getXmlString(element, "hadoopVersion"),
@@ -276,9 +278,10 @@ public class TestRMWebServices extends JerseyTest {
Exception {
assertEquals("incorrect number of elements", 1, json.length());
JSONObject info = json.getJSONObject("clusterInfo");
- assertEquals("incorrect number of elements", 11, info.length());
+ assertEquals("incorrect number of elements", 12, info.length());
verifyClusterGeneric(info.getLong("id"), info.getLong("startedOn"),
info.getString("state"), info.getString("haState"),
+ info.getString("haZooKeeperConnectionState"),
info.getString("hadoopVersionBuiltOn"),
info.getString("hadoopBuildVersion"), info.getString("hadoopVersion"),
info.getString("resourceManagerVersionBuiltOn"),
@@ -288,7 +291,8 @@ public class TestRMWebServices extends JerseyTest {
}
public void verifyClusterGeneric(long clusterid, long startedon,
- String state, String haState, String hadoopVersionBuiltOn,
+ String state, String haState, String haZooKeeperConnectionState,
+ String hadoopVersionBuiltOn,
String hadoopBuildVersion, String hadoopVersion,
String resourceManagerVersionBuiltOn, String resourceManagerBuildVersion,
String resourceManagerVersion) {
[12/50] [abbrv] hadoop git commit: YARN-3124. Fixed CS
LeafQueue/ParentQueue to use QueueCapacities to track capacities-by-label.
Contributed by Wangda Tan (cherry picked from commit
18a594257e052e8f10a03e5594e6cc6901dc56be)
Posted by sj...@apache.org.
YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track capacities-by-label. Contributed by Wangda Tan
(cherry picked from commit 18a594257e052e8f10a03e5594e6cc6901dc56be)
(cherry picked from commit 1be2d64ddddaa6322909073cfaf7f2f2eb46e18d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee2b6bc2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee2b6bc2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee2b6bc2
Branch: refs/heads/branch-2.6
Commit: ee2b6bc248eaa3d204bd6985d44ac8aea56b9728
Parents: 637e7f9
Author: Jian He <ji...@apache.org>
Authored: Thu Feb 12 14:58:09 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sun Sep 6 11:54:40 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 2 +
.../scheduler/capacity/AbstractCSQueue.java | 229 ++++++-------------
.../scheduler/capacity/CSQueue.java | 41 +---
.../scheduler/capacity/CSQueueUtils.java | 160 +++++++++----
.../CapacitySchedulerConfiguration.java | 73 ++----
.../scheduler/capacity/LeafQueue.java | 208 +++++------------
.../scheduler/capacity/ParentQueue.java | 92 ++------
.../scheduler/capacity/PlanQueue.java | 13 +-
.../scheduler/capacity/QueueCapacities.java | 68 +++++-
.../scheduler/capacity/ReservationQueue.java | 10 +-
.../scheduler/capacity/TestCSQueueUtils.java | 24 +-
.../capacity/TestCapacityScheduler.java | 6 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 3 +-
.../scheduler/capacity/TestQueueCapacities.java | 2 +-
.../scheduler/capacity/TestQueueParsing.java | 14 +-
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
17 files changed, 412 insertions(+), 540 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2c654dc..a6bbdf3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -189,6 +189,9 @@ Release 2.6.1 - UNRELEASED
YARN-2694. Ensure only single node label specified in ResourceRequest.
(Wangda Tan via jianhe)
+ YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track
+ capacities-by-label. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 971acea..ece5cff 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -204,6 +204,8 @@
<Field name="absoluteNodeLabelCapacities" />
<Field name="reservationsContinueLooking" />
<Field name="absoluteCapacityByNodeLabels" />
+ <Field name="authorizer" />
+ <Field name="parent" />
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 3c1663f..52629b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -39,25 +39,17 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
public abstract class AbstractCSQueue implements CSQueue {
CSQueue parent;
final String queueName;
-
- float capacity;
- float maximumCapacity;
- float absoluteCapacity;
- float absoluteMaxCapacity;
- float absoluteUsedCapacity = 0.0f;
- float usedCapacity = 0.0f;
volatile int numContainers;
- final Resource minimumAllocation;
- final Resource maximumAllocation;
+ Resource minimumAllocation;
+ Resource maximumAllocation;
QueueState state;
final QueueMetrics metrics;
@@ -65,10 +57,6 @@ public abstract class AbstractCSQueue implements CSQueue {
Set<String> accessibleLabels;
RMNodeLabelsManager labelManager;
String defaultLabelExpression;
- Map<String, Float> absoluteCapacityByNodeLabels;
- Map<String, Float> capacitiyByNodeLabels;
- Map<String, Float> absoluteMaxCapacityByNodeLabels;
- Map<String, Float> maxCapacityByNodeLabels;
Map<QueueACL, AccessControlList> acls =
new HashMap<QueueACL, AccessControlList>();
@@ -77,13 +65,16 @@ public abstract class AbstractCSQueue implements CSQueue {
// Track resource usage-by-label like used-resource/pending-resource, etc.
ResourceUsage queueUsage;
+ // Track capacities like used-capcity/abs-used-capacity/capacity/abs-capacity,
+ // etc.
+ QueueCapacities queueCapacities;
+
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
+ protected CapacitySchedulerContext csContext;
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
- this.minimumAllocation = cs.getMinimumResourceCapability();
- this.maximumAllocation = cs.getMaximumResourceCapability();
this.labelManager = cs.getRMContext().getNodeLabelManager();
this.parent = parent;
this.queueName = queueName;
@@ -94,65 +85,53 @@ public abstract class AbstractCSQueue implements CSQueue {
QueueMetrics.forQueue(getQueuePath(), parent,
cs.getConfiguration().getEnableUserMetrics(),
cs.getConf());
-
- // get labels
- this.accessibleLabels = cs.getConfiguration().getAccessibleNodeLabels(getQueuePath());
- this.defaultLabelExpression = cs.getConfiguration()
- .getDefaultNodeLabelExpression(getQueuePath());
-
- // inherit from parent if labels not set
- if (this.accessibleLabels == null && parent != null) {
- this.accessibleLabels = parent.getAccessibleNodeLabels();
- }
- SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
- this.accessibleLabels);
-
- // inherit from parent if labels not set
- if (this.defaultLabelExpression == null && parent != null
- && this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
- this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
- }
-
- // set capacity by labels
- capacitiyByNodeLabels =
- cs.getConfiguration().getNodeLabelCapacities(getQueuePath(), accessibleLabels,
- labelManager);
+ this.csContext = cs;
- // set maximum capacity by labels
- maxCapacityByNodeLabels =
- cs.getConfiguration().getMaximumNodeLabelCapacities(getQueuePath(),
- accessibleLabels, labelManager);
+ // initialize ResourceUsage
queueUsage = new ResourceUsage();
+
+ // initialize QueueCapacities
+ queueCapacities = new QueueCapacities(parent == null);
+ }
+
+ protected void setupConfigurableCapacities() {
+ CSQueueUtils.loadUpdateAndCheckCapacities(
+ getQueuePath(),
+ accessibleLabels,
+ csContext.getConfiguration(),
+ queueCapacities,
+ parent == null ? null : parent.getQueueCapacities(),
+ csContext.getRMContext().getNodeLabelManager());
}
@Override
public synchronized float getCapacity() {
- return capacity;
+ return queueCapacities.getCapacity();
}
@Override
public synchronized float getAbsoluteCapacity() {
- return absoluteCapacity;
+ return queueCapacities.getAbsoluteCapacity();
}
@Override
public float getAbsoluteMaximumCapacity() {
- return absoluteMaxCapacity;
+ return queueCapacities.getAbsoluteMaximumCapacity();
}
@Override
public synchronized float getAbsoluteUsedCapacity() {
- return absoluteUsedCapacity;
+ return queueCapacities.getAbsoluteUsedCapacity();
}
@Override
public float getMaximumCapacity() {
- return maximumCapacity;
+ return queueCapacities.getMaximumCapacity();
}
@Override
public synchronized float getUsedCapacity() {
- return usedCapacity;
+ return queueCapacities.getUsedCapacity();
}
@Override
@@ -210,12 +189,12 @@ public abstract class AbstractCSQueue implements CSQueue {
@Override
public synchronized void setUsedCapacity(float usedCapacity) {
- this.usedCapacity = usedCapacity;
+ queueCapacities.setUsedCapacity(usedCapacity);
}
@Override
public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) {
- this.absoluteUsedCapacity = absUsedCapacity;
+ queueCapacities.setAbsoluteUsedCapacity(absUsedCapacity);
}
/**
@@ -224,21 +203,16 @@ public abstract class AbstractCSQueue implements CSQueue {
*/
synchronized void setMaxCapacity(float maximumCapacity) {
// Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
+ CSQueueUtils.checkMaxCapacity(getQueueName(),
+ queueCapacities.getCapacity(), maximumCapacity);
float absMaxCapacity =
CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absoluteCapacity,
+ CSQueueUtils.checkAbsoluteCapacity(getQueueName(),
+ queueCapacities.getAbsoluteCapacity(),
absMaxCapacity);
- this.maximumCapacity = maximumCapacity;
- this.absoluteMaxCapacity = absMaxCapacity;
- }
-
- @Override
- public float getAbsActualCapacity() {
- // for now, simply return actual capacity = guaranteed capacity for parent
- // queue
- return absoluteCapacity;
+ queueCapacities.setMaximumCapacity(maximumCapacity);
+ queueCapacities.setAbsoluteMaximumCapacity(absMaxCapacity);
}
@Override
@@ -246,39 +220,35 @@ public abstract class AbstractCSQueue implements CSQueue {
return defaultLabelExpression;
}
- synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
- float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity,
- QueueState state, Map<QueueACL, AccessControlList> acls,
- Set<String> labels, String defaultLabelExpression,
- Map<String, Float> nodeLabelCapacities,
- Map<String, Float> maximumNodeLabelCapacities,
- boolean reservationContinueLooking)
+ synchronized void setupQueueConfigs(Resource clusterResource)
throws IOException {
- // Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absoluteCapacity,
- absoluteMaxCapacity);
-
- this.capacity = capacity;
- this.absoluteCapacity = absoluteCapacity;
-
- this.maximumCapacity = maximumCapacity;
- this.absoluteMaxCapacity = absoluteMaxCapacity;
-
- this.state = state;
+ // get labels
+ this.accessibleLabels =
+ csContext.getConfiguration().getAccessibleNodeLabels(getQueuePath());
+ this.defaultLabelExpression = csContext.getConfiguration()
+ .getDefaultNodeLabelExpression(getQueuePath());
- this.acls = acls;
+ // inherit from parent if labels not set
+ if (this.accessibleLabels == null && parent != null) {
+ this.accessibleLabels = parent.getAccessibleNodeLabels();
+ }
+ SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
+ this.accessibleLabels);
- // set labels
- this.accessibleLabels = labels;
+ // inherit from parent if labels not set
+ if (this.defaultLabelExpression == null && parent != null
+ && this.accessibleLabels.containsAll(parent.getAccessibleNodeLabels())) {
+ this.defaultLabelExpression = parent.getDefaultNodeLabelExpression();
+ }
+
+ // After we setup labels, we can setup capacities
+ setupConfigurableCapacities();
- // set label expression
- this.defaultLabelExpression = defaultLabelExpression;
+ this.minimumAllocation = csContext.getMinimumResourceCapability();
+ this.maximumAllocation = csContext.getMaximumResourceCapability();
- // copy node label capacity
- this.capacitiyByNodeLabels = new HashMap<String, Float>(nodeLabelCapacities);
- this.maxCapacityByNodeLabels =
- new HashMap<String, Float>(maximumNodeLabelCapacities);
+ this.state = csContext.getConfiguration().getState(getQueuePath());
+ this.acls = csContext.getConfiguration().getAcls(getQueuePath());
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -305,30 +275,18 @@ public abstract class AbstractCSQueue implements CSQueue {
}
}
}
-
- // calculate absolute capacity by each node label
- this.absoluteCapacityByNodeLabels =
- CSQueueUtils.computeAbsoluteCapacityByNodeLabels(
- this.capacitiyByNodeLabels, parent);
-
- // calculate maximum capacity by each node label
- this.absoluteMaxCapacityByNodeLabels =
- CSQueueUtils.computeAbsoluteMaxCapacityByNodeLabels(
- maximumNodeLabelCapacities, parent);
-
- // check absoluteMaximumNodeLabelCapacities is valid
- CSQueueUtils.checkAbsoluteCapacitiesByLabel(getQueueName(),
- absoluteCapacityByNodeLabels, absoluteCapacityByNodeLabels);
-
- this.reservationsContinueLooking = reservationContinueLooking;
+
+ this.reservationsContinueLooking = csContext.getConfiguration()
+ .getReservationContinueLook();
+
}
protected QueueInfo getQueueInfo() {
QueueInfo queueInfo = recordFactory.newRecordInstance(QueueInfo.class);
queueInfo.setQueueName(queueName);
queueInfo.setAccessibleNodeLabels(accessibleLabels);
- queueInfo.setCapacity(capacity);
- queueInfo.setMaximumCapacity(maximumCapacity);
+ queueInfo.setCapacity(queueCapacities.getCapacity());
+ queueInfo.setMaximumCapacity(queueCapacities.getMaximumCapacity());
queueInfo.setQueueState(state);
queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
queueInfo.setCurrentCapacity(getUsedCapacity());
@@ -379,51 +337,6 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Private
- public float getCapacityByNodeLabel(String label) {
- if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) {
- if (null == parent) {
- return 1f;
- }
- return getCapacity();
- }
-
- if (!capacitiyByNodeLabels.containsKey(label)) {
- return 0f;
- } else {
- return capacitiyByNodeLabels.get(label);
- }
- }
-
- @Private
- public float getAbsoluteCapacityByNodeLabel(String label) {
- if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) {
- if (null == parent) {
- return 1f;
- }
- return getAbsoluteCapacity();
- }
-
- if (!absoluteCapacityByNodeLabels.containsKey(label)) {
- return 0f;
- } else {
- return absoluteCapacityByNodeLabels.get(label);
- }
- }
-
- @Private
- public float getAbsoluteMaximumCapacityByNodeLabel(String label) {
- if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL)) {
- return getAbsoluteMaximumCapacity();
- }
-
- if (!absoluteMaxCapacityByNodeLabels.containsKey(label)) {
- return 0f;
- } else {
- return absoluteMaxCapacityByNodeLabels.get(label);
- }
- }
-
- @Private
public boolean getReservationContinueLooking() {
return reservationsContinueLooking;
}
@@ -432,14 +345,14 @@ public abstract class AbstractCSQueue implements CSQueue {
public Map<QueueACL, AccessControlList> getACLs() {
return acls;
}
-
+
@Private
- public Resource getUsedResourceByLabel(String nodeLabel) {
- return queueUsage.getUsed(nodeLabel);
+ public QueueCapacities getQueueCapacities() {
+ return queueCapacities;
}
- @VisibleForTesting
- public ResourceUsage getResourceUsage() {
+ @Private
+ public ResourceUsage getQueueResourceUsage() {
return queueUsage;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
index 07a7e0e..1dfb872 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -75,15 +76,6 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
* @return configured queue capacity
*/
public float getCapacity();
-
- /**
- * Get actual <em>capacity</em> of the queue, this may be different from
- * configured capacity when mis-config take place, like add labels to the
- * cluster
- *
- * @return actual queue capacity
- */
- public float getAbsActualCapacity();
/**
* Get capacity of the parent of the queue as a function of the
@@ -144,14 +136,6 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
public Resource getUsedResources();
/**
- * Get the currently utilized resources which allocated at nodes with label
- * specified
- *
- * @return used resources by the queue and it's children
- */
- public Resource getUsedResourceByLabel(String nodeLabel);
-
- /**
* Get the current run-state of the queue
* @return current run-state
*/
@@ -279,25 +263,16 @@ extends org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue {
*/
public void attachContainer(Resource clusterResource,
FiCaSchedulerApp application, RMContainer container);
-
+
/**
- * Get absolute capacity by label of this queue can use
- * @param nodeLabel
- * @return absolute capacity by label of this queue can use
+ * Get QueueCapacities of this queue
+ * @return queueCapacities
*/
- public float getAbsoluteCapacityByNodeLabel(String nodeLabel);
+ public QueueCapacities getQueueCapacities();
/**
- * Get absolute max capacity by label of this queue can use
- * @param nodeLabel
- * @return absolute capacity by label of this queue can use
- */
- public float getAbsoluteMaximumCapacityByNodeLabel(String nodeLabel);
-
- /**
- * Get capacity by node label
- * @param nodeLabel
- * @return capacity by node label
+ * Get ResourceUsage of this queue
+ * @return resourceUsage
*/
- public float getCapacityByNodeLabel(String nodeLabel);
+ public ResourceUsage getQueueResourceUsage();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index f458057..865b0b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -17,13 +17,14 @@
*/
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Map.Entry;
+import java.util.HashSet;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.utils.Lock;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -34,6 +35,9 @@ class CSQueueUtils {
final static float EPSILON = 0.0001f;
+ /*
+ * Used only by tests
+ */
public static void checkMaxCapacity(String queueName,
float capacity, float maximumCapacity) {
if (maximumCapacity < 0.0f || maximumCapacity > 1.0f) {
@@ -43,6 +47,9 @@ class CSQueueUtils {
}
}
+ /*
+ * Used only by tests
+ */
public static void checkAbsoluteCapacity(String queueName,
float absCapacity, float absMaxCapacity) {
if (absMaxCapacity < (absCapacity - EPSILON)) {
@@ -53,19 +60,33 @@ class CSQueueUtils {
}
}
- public static void checkAbsoluteCapacitiesByLabel(String queueName,
- Map<String, Float> absCapacities,
- Map<String, Float> absMaximumCapacities) {
- for (Entry<String, Float> entry : absCapacities.entrySet()) {
- String label = entry.getKey();
- float absCapacity = entry.getValue();
- float absMaxCapacity = absMaximumCapacities.get(label);
- if (absMaxCapacity < (absCapacity - EPSILON)) {
- throw new IllegalArgumentException("Illegal call to setMaxCapacity. "
- + "Queue '" + queueName + "' has " + "an absolute capacity ("
- + absCapacity + ") greater than "
- + "its absolute maximumCapacity (" + absMaxCapacity + ") of label="
- + label);
+ /**
+ * Check sanity of capacities:
+ * - capacity <= maxCapacity
+ * - absCapacity <= absMaximumCapacity
+ */
+ private static void capacitiesSanityCheck(String queueName,
+ QueueCapacities queueCapacities) {
+ for (String label : queueCapacities.getExistingNodeLabels()) {
+ float capacity = queueCapacities.getCapacity(label);
+ float maximumCapacity = queueCapacities.getMaximumCapacity(label);
+ if (capacity > maximumCapacity) {
+ throw new IllegalArgumentException("Illegal queue capacity setting, "
+ + "(capacity=" + capacity + ") > (maximum-capacity="
+ + maximumCapacity + "). When label=[" + label + "]");
+ }
+
+ // Actually, this may not needed since we have verified capacity <=
+ // maximumCapacity. And the way we compute absolute capacity (abs(x) =
+ // cap(x) * cap(x.parent) * ...) is a monotone increasing function. But
+ // just keep it here to make sure our compute abs capacity method works
+ // correctly.
+ float absCapacity = queueCapacities.getAbsoluteCapacity(label);
+ float absMaxCapacity = queueCapacities.getAbsoluteMaximumCapacity(label);
+ if (absCapacity > absMaxCapacity) {
+ throw new IllegalArgumentException("Illegal queue capacity setting, "
+ + "(abs-capacity=" + absCapacity + ") > (abs-maximum-capacity="
+ + absMaxCapacity + "). When label=[" + label + "]");
}
}
}
@@ -77,37 +98,94 @@ class CSQueueUtils {
return (parentAbsMaxCapacity * maximumCapacity);
}
- public static Map<String, Float> computeAbsoluteCapacityByNodeLabels(
- Map<String, Float> nodeLabelToCapacities, CSQueue parent) {
- if (parent == null) {
- return nodeLabelToCapacities;
+ /**
+ * This method intends to be used by ReservationQueue, ReservationQueue will
+ * not appear in configuration file, so we shouldn't do load capacities
+ * settings in configuration for reservation queue.
+ */
+ public static void updateAndCheckCapacitiesByLabel(String queuePath,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
+
+ capacitiesSanityCheck(queuePath, queueCapacities);
+ }
+
+ /**
+ * Do following steps for capacities
+ * - Load capacities from configuration
+ * - Update absolute capacities for new capacities
+ * - Check if capacities/absolute-capacities legal
+ */
+ public static void loadUpdateAndCheckCapacities(String queuePath,
+ Set<String> accessibleLabels, CapacitySchedulerConfiguration csConf,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities,
+ RMNodeLabelsManager nlm) {
+ loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm,
+ queueCapacities, csConf);
+
+ updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
+
+ capacitiesSanityCheck(queuePath, queueCapacities);
+ }
+
+ // Considered NO_LABEL, ANY and null cases
+ private static Set<String> normalizeAccessibleNodeLabels(Set<String> labels,
+ RMNodeLabelsManager mgr) {
+ Set<String> accessibleLabels = new HashSet<String>();
+ if (labels != null) {
+ accessibleLabels.addAll(labels);
}
-
- Map<String, Float> absoluteCapacityByNodeLabels =
- new HashMap<String, Float>();
- for (Entry<String, Float> entry : nodeLabelToCapacities.entrySet()) {
- String label = entry.getKey();
- float capacity = entry.getValue();
- absoluteCapacityByNodeLabels.put(label,
- capacity * parent.getAbsoluteCapacityByNodeLabel(label));
+ if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) {
+ accessibleLabels.addAll(mgr.getClusterNodeLabels());
}
- return absoluteCapacityByNodeLabels;
+ accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL);
+
+ return accessibleLabels;
}
- public static Map<String, Float> computeAbsoluteMaxCapacityByNodeLabels(
- Map<String, Float> maximumNodeLabelToCapacities, CSQueue parent) {
- if (parent == null) {
- return maximumNodeLabelToCapacities;
+ private static void loadCapacitiesByLabelsFromConf(String queuePath,
+ Set<String> labels, RMNodeLabelsManager mgr,
+ QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) {
+ queueCapacities.clearConfigurableFields();
+ labels = normalizeAccessibleNodeLabels(labels, mgr);
+
+ for (String label : labels) {
+ if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
+ queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL,
+ csConf.getNonLabeledQueueCapacity(queuePath) / 100);
+ queueCapacities.setMaximumCapacity(CommonNodeLabelsManager.NO_LABEL,
+ csConf.getNonLabeledQueueMaximumCapacity(queuePath) / 100);
+ } else {
+ queueCapacities.setCapacity(label,
+ csConf.getLabeledQueueCapacity(queuePath, label) / 100);
+ queueCapacities.setMaximumCapacity(label,
+ csConf.getLabeledQueueMaximumCapacity(queuePath, label) / 100);
+ }
}
- Map<String, Float> absoluteMaxCapacityByNodeLabels =
- new HashMap<String, Float>();
- for (Entry<String, Float> entry : maximumNodeLabelToCapacities.entrySet()) {
- String label = entry.getKey();
- float maxCapacity = entry.getValue();
- absoluteMaxCapacityByNodeLabels.put(label,
- maxCapacity * parent.getAbsoluteMaximumCapacityByNodeLabel(label));
+ }
+
+ // Set absolute capacities for {capacity, maximum-capacity}
+ private static void updateAbsoluteCapacitiesByNodeLabels(
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ for (String label : queueCapacities.getExistingNodeLabels()) {
+ float capacity = queueCapacities.getCapacity(label);
+ if (capacity > 0f) {
+ queueCapacities.setAbsoluteCapacity(
+ label,
+ capacity
+ * (parentQueueCapacities == null ? 1 : parentQueueCapacities
+ .getAbsoluteCapacity(label)));
+ }
+
+ float maxCapacity = queueCapacities.getMaximumCapacity(label);
+ if (maxCapacity > 0f) {
+ queueCapacities.setAbsoluteMaximumCapacity(
+ label,
+ maxCapacity
+ * (parentQueueCapacities == null ? 1 : parentQueueCapacities
+ .getAbsoluteMaximumCapacity(label)));
+ }
}
- return absoluteMaxCapacityByNodeLabels;
}
@Lock(CSQueue.class)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 23bf381..f5e86b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -287,6 +287,9 @@ public class CapacitySchedulerConfiguration extends Configuration {
}
private String getNodeLabelPrefix(String queue, String label) {
+ if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
+ return getQueuePrefix(queue);
+ }
return getQueuePrefix(queue) + ACCESSIBLE_NODE_LABELS + DOT + label + DOT;
}
@@ -325,7 +328,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
getMaximumApplicationMasterResourcePercent());
}
- public float getCapacity(String queue) {
+ public float getNonLabeledQueueCapacity(String queue) {
float capacity = queue.equals("root") ? 100.0f : getFloat(
getQueuePrefix(queue) + CAPACITY, UNDEFINED);
if (capacity < MINIMUM_CAPACITY_VALUE || capacity > MAXIMUM_CAPACITY_VALUE) {
@@ -347,7 +350,7 @@ public class CapacitySchedulerConfiguration extends Configuration {
", capacity=" + capacity);
}
- public float getMaximumCapacity(String queue) {
+ public float getNonLabeledQueueMaximumCapacity(String queue) {
float maxCapacity = getFloat(getQueuePrefix(queue) + MAXIMUM_CAPACITY,
MAXIMUM_CAPACITY_VALUE);
maxCapacity = (maxCapacity == DEFAULT_MAXIMUM_CAPACITY_VALUE) ?
@@ -451,57 +454,29 @@ public class CapacitySchedulerConfiguration extends Configuration {
return Collections.unmodifiableSet(set);
}
- public Map<String, Float> getNodeLabelCapacities(String queue,
- Set<String> labels, RMNodeLabelsManager mgr) {
- Map<String, Float> nodeLabelCapacities = new HashMap<String, Float>();
-
- if (labels == null) {
- return nodeLabelCapacities;
+ private float internalGetLabeledQueueCapacity(String queue, String label, String suffix,
+ float defaultValue) {
+ String capacityPropertyName = getNodeLabelPrefix(queue, label) + suffix;
+ float capacity = getFloat(capacityPropertyName, defaultValue);
+ if (capacity < MINIMUM_CAPACITY_VALUE
+ || capacity > MAXIMUM_CAPACITY_VALUE) {
+ throw new IllegalArgumentException("Illegal capacity of " + capacity
+ + " for node-label=" + label + " in queue=" + queue
+ + ", valid capacity should in range of [0, 100].");
}
-
- for (String label : labels.contains(CommonNodeLabelsManager.ANY) ? mgr
- .getClusterNodeLabels() : labels) {
- String capacityPropertyName = getNodeLabelPrefix(queue, label) + CAPACITY;
- float capacity = getFloat(capacityPropertyName, 0f);
- if (capacity < MINIMUM_CAPACITY_VALUE
- || capacity > MAXIMUM_CAPACITY_VALUE) {
- throw new IllegalArgumentException("Illegal capacity of " + capacity
- + " for node-label=" + label + " in queue=" + queue
- + ", valid capacity should in range of [0, 100].");
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("CSConf - getCapacityOfLabel: prefix="
- + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity);
- }
-
- nodeLabelCapacities.put(label, capacity / 100f);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("CSConf - getCapacityOfLabel: prefix="
+ + getNodeLabelPrefix(queue, label) + ", capacity=" + capacity);
}
- return nodeLabelCapacities;
+ return capacity;
}
- public Map<String, Float> getMaximumNodeLabelCapacities(String queue,
- Set<String> labels, RMNodeLabelsManager mgr) {
- Map<String, Float> maximumNodeLabelCapacities = new HashMap<String, Float>();
- if (labels == null) {
- return maximumNodeLabelCapacities;
- }
-
- for (String label : labels.contains(CommonNodeLabelsManager.ANY) ? mgr
- .getClusterNodeLabels() : labels) {
- float maxCapacity =
- getFloat(getNodeLabelPrefix(queue, label) + MAXIMUM_CAPACITY,
- 100f);
- if (maxCapacity < MINIMUM_CAPACITY_VALUE
- || maxCapacity > MAXIMUM_CAPACITY_VALUE) {
- throw new IllegalArgumentException("Illegal " + "capacity of "
- + maxCapacity + " for label=" + label + " in queue=" + queue);
- }
- LOG.debug("CSConf - getCapacityOfLabel: prefix="
- + getNodeLabelPrefix(queue, label) + ", capacity=" + maxCapacity);
-
- maximumNodeLabelCapacities.put(label, maxCapacity / 100f);
- }
- return maximumNodeLabelCapacities;
+ public float getLabeledQueueCapacity(String queue, String label) {
+ return internalGetLabeledQueueCapacity(queue, label, CAPACITY, 0f);
+ }
+
+ public float getLabeledQueueMaximumCapacity(String queue, String label) {
+ return internalGetLabeledQueueCapacity(queue, label, MAXIMUM_CAPACITY, 100f);
}
public String getDefaultNodeLabelExpression(String queue) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6fbc8e4..f9f9ca9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -97,7 +97,7 @@ public class LeafQueue extends AbstractCSQueue {
Set<FiCaSchedulerApp> pendingApplications;
- private final float minimumAllocationFactor;
+ private float minimumAllocationFactor;
private Map<String, User> users = new HashMap<String, User>();
@@ -123,53 +123,7 @@ public class LeafQueue extends AbstractCSQueue {
super(cs, queueName, parent, old);
this.scheduler = cs;
- this.activeUsersManager = new ActiveUsersManager(metrics);
- this.minimumAllocationFactor =
- Resources.ratio(resourceCalculator,
- Resources.subtract(maximumAllocation, minimumAllocation),
- maximumAllocation);
-
- float capacity = getCapacityFromConf();
- float absoluteCapacity = parent.getAbsoluteCapacity() * capacity;
-
- float maximumCapacity =
- (float)cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
- float absoluteMaxCapacity =
- CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
-
- // Initially set to absoluteMax, will be updated to more accurate
- // max avail value during assignContainers
- absoluteMaxAvailCapacity = absoluteMaxCapacity;
-
- int userLimit = cs.getConfiguration().getUserLimit(getQueuePath());
- float userLimitFactor =
- cs.getConfiguration().getUserLimitFactor(getQueuePath());
-
- int maxApplications =
- cs.getConfiguration().getMaximumApplicationsPerQueue(getQueuePath());
- if (maxApplications < 0) {
- int maxSystemApps = cs.getConfiguration().getMaximumSystemApplications();
- maxApplications = (int)(maxSystemApps * absoluteCapacity);
- }
- maxApplicationsPerUser =
- (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
-
- float maxAMResourcePerQueuePercent = cs.getConfiguration()
- .getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
-
- QueueState state = cs.getConfiguration().getState(getQueuePath());
-
- Map<QueueACL, AccessControlList> acls =
- cs.getConfiguration().getAcls(getQueuePath());
-
- setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor,
- maxApplications, maxAMResourcePerQueuePercent, maxApplicationsPerUser,
- state, acls, cs.getConfiguration().getNodeLocalityDelay(), accessibleLabels,
- defaultLabelExpression, this.capacitiyByNodeLabels,
- this.maxCapacityByNodeLabels,
- cs.getConfiguration().getReservationContinueLook());
-
+ this.activeUsersManager = new ActiveUsersManager(metrics);
if(LOG.isDebugEnabled()) {
LOG.debug("LeafQueue:" + " name=" + queueName
+ ", fullname=" + getQueuePath());
@@ -180,34 +134,13 @@ public class LeafQueue extends AbstractCSQueue {
this.pendingApplications =
new TreeSet<FiCaSchedulerApp>(applicationComparator);
this.activeApplications = new TreeSet<FiCaSchedulerApp>(applicationComparator);
+
+ setupQueueConfigs(cs.getClusterResource());
}
-
- // externalizing in method, to allow overriding
- protected float getCapacityFromConf() {
- return (float)scheduler.getConfiguration().getCapacity(getQueuePath()) / 100;
- }
-
- protected synchronized void setupQueueConfigs(
- Resource clusterResource,
- float capacity, float absoluteCapacity,
- float maximumCapacity, float absoluteMaxCapacity,
- int userLimit, float userLimitFactor,
- int maxApplications, float maxAMResourcePerQueuePercent,
- int maxApplicationsPerUser, QueueState state,
- Map<QueueACL, AccessControlList> acls, int nodeLocalityDelay,
- Set<String> labels, String defaultLabelExpression,
- Map<String, Float> capacitieByLabel,
- Map<String, Float> maximumCapacitiesByLabel,
- boolean revervationContinueLooking) throws IOException {
- super.setupQueueConfigs(clusterResource, capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, state, acls, labels,
- defaultLabelExpression, capacitieByLabel, maximumCapacitiesByLabel,
- revervationContinueLooking);
- // Sanity check
- CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
- float absCapacity = getParent().getAbsoluteCapacity() * capacity;
- CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absCapacity,
- absoluteMaxCapacity);
+
+ protected synchronized void setupQueueConfigs(Resource clusterResource)
+ throws IOException {
+ super.setupQueueConfigs(clusterResource);
this.lastClusterResource = clusterResource;
updateAbsoluteCapacityResource(clusterResource);
@@ -217,16 +150,28 @@ public class LeafQueue extends AbstractCSQueue {
// and all queues may not be realized yet, we'll use (optimistic)
// absoluteMaxCapacity (it will be replaced with the more accurate
// absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
- updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
+ updateHeadroomInfo(clusterResource,
+ queueCapacities.getAbsoluteMaximumCapacity());
- this.absoluteCapacity = absCapacity;
+ CapacitySchedulerConfiguration conf = csContext.getConfiguration();
+ userLimit = conf.getUserLimit(getQueuePath());
+ userLimitFactor = conf.getUserLimitFactor(getQueuePath());
- this.userLimit = userLimit;
- this.userLimitFactor = userLimitFactor;
+ // Initially set to absoluteMax, will be updated to more accurate
+ // max avail value during assignContainers
+ absoluteMaxAvailCapacity = queueCapacities.getAbsoluteMaximumCapacity();
- this.maxApplications = maxApplications;
- this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
- this.maxApplicationsPerUser = maxApplicationsPerUser;
+ maxApplications = conf.getMaximumApplicationsPerQueue(getQueuePath());
+ if (maxApplications < 0) {
+ int maxSystemApps = conf.getMaximumSystemApplications();
+ maxApplications =
+ (int) (maxSystemApps * queueCapacities.getAbsoluteCapacity());
+ }
+ maxApplicationsPerUser =
+ (int)(maxApplications * (userLimit / 100.0f) * userLimitFactor);
+
+ maxAMResourcePerQueuePercent =
+ conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
this.defaultLabelExpression)) {
@@ -242,7 +187,12 @@ public class LeafQueue extends AbstractCSQueue {
getAccessibleNodeLabels().iterator(), ',')));
}
- this.nodeLocalityDelay = nodeLocalityDelay;
+ nodeLocalityDelay = conf.getNodeLocalityDelay();
+
+ this.minimumAllocationFactor =
+ Resources.ratio(resourceCalculator,
+ Resources.subtract(maximumAllocation, minimumAllocation),
+ maximumAllocation);
StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
@@ -250,21 +200,21 @@ public class LeafQueue extends AbstractCSQueue {
}
StringBuilder labelStrBuilder = new StringBuilder();
- if (labels != null) {
- for (String s : labels) {
+ if (accessibleLabels != null) {
+ for (String s : accessibleLabels) {
labelStrBuilder.append(s);
labelStrBuilder.append(",");
}
}
LOG.info("Initializing " + queueName + "\n" +
- "capacity = " + capacity +
+ "capacity = " + queueCapacities.getCapacity() +
" [= (float) configuredCapacity / 100 ]" + "\n" +
- "asboluteCapacity = " + absoluteCapacity +
+ "asboluteCapacity = " + queueCapacities.getAbsoluteCapacity() +
" [= parentAbsoluteCapacity * capacity ]" + "\n" +
- "maxCapacity = " + maximumCapacity +
+ "maxCapacity = " + queueCapacities.getMaximumCapacity() +
" [= configuredMaxCapacity ]" + "\n" +
- "absoluteMaxCapacity = " + absoluteMaxCapacity +
+ "absoluteMaxCapacity = " + queueCapacities.getAbsoluteMaximumCapacity() +
" [= 1.0 maximumCapacity undefined, " +
"(parentAbsoluteMaxCapacity * maximumCapacity) / 100 otherwise ]" +
"\n" +
@@ -279,7 +229,7 @@ public class LeafQueue extends AbstractCSQueue {
"maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
- "usedCapacity = " + usedCapacity +
+ "usedCapacity = " + queueCapacities.getUsedCapacity() +
" [= usedResourcesMemory / " +
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
"absoluteUsedCapacity = " + absoluteUsedCapacity +
@@ -435,8 +385,8 @@ public class LeafQueue extends AbstractCSQueue {
public String toString() {
return queueName + ": " +
- "capacity=" + capacity + ", " +
- "absoluteCapacity=" + absoluteCapacity + ", " +
+ "capacity=" + queueCapacities.getCapacity() + ", " +
+ "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
"usedResources=" + queueUsage.getUsed() + ", " +
"usedCapacity=" + getUsedCapacity() + ", " +
"absoluteUsedCapacity=" + getAbsoluteUsedCapacity() + ", " +
@@ -483,23 +433,7 @@ public class LeafQueue extends AbstractCSQueue {
" from " + newlyParsedQueue.getQueuePath());
}
- LeafQueue newlyParsedLeafQueue = (LeafQueue)newlyParsedQueue;
- setupQueueConfigs(
- clusterResource,
- newlyParsedLeafQueue.capacity, newlyParsedLeafQueue.absoluteCapacity,
- newlyParsedLeafQueue.maximumCapacity,
- newlyParsedLeafQueue.absoluteMaxCapacity,
- newlyParsedLeafQueue.userLimit, newlyParsedLeafQueue.userLimitFactor,
- newlyParsedLeafQueue.maxApplications,
- newlyParsedLeafQueue.maxAMResourcePerQueuePercent,
- newlyParsedLeafQueue.getMaxApplicationsPerUser(),
- newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
- newlyParsedLeafQueue.getNodeLocalityDelay(),
- newlyParsedLeafQueue.accessibleLabels,
- newlyParsedLeafQueue.defaultLabelExpression,
- newlyParsedLeafQueue.capacitiyByNodeLabels,
- newlyParsedLeafQueue.maxCapacityByNodeLabels,
- newlyParsedLeafQueue.reservationsContinueLooking);
+ setupQueueConfigs(clusterResource);
// queue metrics are updated, more resource may be available
// activate the pending applications if possible
@@ -1022,7 +956,8 @@ public class LeafQueue extends AbstractCSQueue {
application.getCurrentReservation()),
labelManager.getResourceByLabel(label, clusterResource));
- if (potentialNewWithoutReservedCapacity <= absoluteMaxCapacity) {
+ if (potentialNewWithoutReservedCapacity <= queueCapacities
+ .getAbsoluteMaximumCapacity()) {
if (LOG.isDebugEnabled()) {
LOG.debug("try to use reserved: "
+ getQueueName()
@@ -1036,8 +971,9 @@ public class LeafQueue extends AbstractCSQueue {
+ Resources.divide(resourceCalculator, clusterResource,
queueUsage.getUsed(), clusterResource) + " required " + required
+ " potentialNewWithoutReservedCapacity: "
- + potentialNewWithoutReservedCapacity + " ( " + " max-capacity: "
- + absoluteMaxCapacity + ")");
+ + potentialNewWithoutReservedCapacity + " ( "
+ + " max-capacity: "
+ + queueCapacities.getAbsoluteMaximumCapacity() + ")");
}
// we could potentially use this node instead of reserved node
return true;
@@ -1046,7 +982,8 @@ public class LeafQueue extends AbstractCSQueue {
// Otherwise, if any of the label of this node beyond queue limit, we
// cannot allocate on this node. Consider a small epsilon here.
- if (potentialNewCapacity > getAbsoluteMaximumCapacityByNodeLabel(label) + 1e-4) {
+ if (potentialNewCapacity > queueCapacities
+ .getAbsoluteMaximumCapacity(label) + 1e-4) {
canAssign = false;
break;
}
@@ -1061,7 +998,8 @@ public class LeafQueue extends AbstractCSQueue {
queueUsage.getUsed(label),
labelManager.getResourceByLabel(label, clusterResource))
+ " potentialNewCapacity: " + potentialNewCapacity + " ( "
- + " max-capacity: " + absoluteMaxCapacity + ")");
+ + " max-capacity: " + queueCapacities.getAbsoluteMaximumCapacity()
+ + ")");
}
}
@@ -1144,7 +1082,7 @@ public class LeafQueue extends AbstractCSQueue {
Resources.multiplyAndNormalizeUp(resourceCalculator,
labelManager.getResourceByLabel(firstLabel,
clusterResource),
- getAbsoluteCapacityByNodeLabel(firstLabel),
+ queueCapacities.getAbsoluteCapacity(firstLabel),
minimumAllocation));
} else {
// else there's no label on request, just to use absolute capacity as
@@ -1152,7 +1090,7 @@ public class LeafQueue extends AbstractCSQueue {
queueCapacity =
Resources.multiplyAndNormalizeUp(resourceCalculator, labelManager
.getResourceByLabel(CommonNodeLabelsManager.NO_LABEL, clusterResource),
- absoluteCapacity, minimumAllocation);
+ queueCapacities.getAbsoluteCapacity(), minimumAllocation);
}
// Allow progress for queues with miniscule capacity
@@ -1797,12 +1735,9 @@ public class LeafQueue extends AbstractCSQueue {
}
private void updateAbsoluteCapacityResource(Resource clusterResource) {
-
- absoluteCapacityResource = Resources.multiplyAndNormalizeUp(
- resourceCalculator,
- clusterResource,
- absoluteCapacity, minimumAllocation);
-
+ absoluteCapacityResource =
+ Resources.multiplyAndNormalizeUp(resourceCalculator, clusterResource,
+ queueCapacities.getAbsoluteCapacity(), minimumAllocation);
}
@Override
@@ -1813,7 +1748,8 @@ public class LeafQueue extends AbstractCSQueue {
// Update headroom info based on new cluster resource value
// absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
// during allocation
- updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
+ updateHeadroomInfo(clusterResource,
+ queueCapacities.getAbsoluteMaximumCapacity());
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -1986,35 +1922,13 @@ public class LeafQueue extends AbstractCSQueue {
getParent().detachContainer(clusterResource, application, rmContainer);
}
}
-
- @Override
- public float getAbsActualCapacity() {
- //? Is this actually used by anything at present?
- // There is a findbugs warning -re lastClusterResource (now excluded),
- // when this is used, verify that the access is mt correct and remove
- // the findbugs exclusion if possible
- if (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
- lastClusterResource, Resources.none())) {
- return absoluteCapacity;
- }
-
- Resource resourceRespectLabels =
- labelManager == null ? lastClusterResource : labelManager
- .getQueueResource(queueName, accessibleLabels, lastClusterResource);
- float absActualCapacity =
- Resources.divide(resourceCalculator, lastClusterResource,
- resourceRespectLabels, lastClusterResource);
-
- return absActualCapacity > absoluteCapacity ? absoluteCapacity
- : absActualCapacity;
- }
public void setCapacity(float capacity) {
- this.capacity = capacity;
+ queueCapacities.setCapacity(capacity);
}
public void setAbsoluteCapacity(float absoluteCapacity) {
- this.absoluteCapacity = absoluteCapacity;
+ queueCapacities.setAbsoluteCapacity(absoluteCapacity);
}
public void setMaxApplications(int maxApplications) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index de92c9c..5f7db78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -86,7 +86,7 @@ public class ParentQueue extends AbstractCSQueue {
this.rootQueue = (parent == null);
- float rawCapacity = cs.getConfiguration().getCapacity(getQueuePath());
+ float rawCapacity = cs.getConfiguration().getNonLabeledQueueCapacity(getQueuePath());
if (rootQueue &&
(rawCapacity != CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE)) {
@@ -94,46 +94,20 @@ public class ParentQueue extends AbstractCSQueue {
"capacity of " + rawCapacity + " for queue " + queueName +
". Must be " + CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE);
}
-
- float capacity = (float) rawCapacity / 100;
- float parentAbsoluteCapacity =
- (rootQueue) ? 1.0f : parent.getAbsoluteCapacity();
- float absoluteCapacity = parentAbsoluteCapacity * capacity;
-
- float maximumCapacity =
- (float) cs.getConfiguration().getMaximumCapacity(getQueuePath()) / 100;
- float absoluteMaxCapacity =
- CSQueueUtils.computeAbsoluteMaximumCapacity(maximumCapacity, parent);
-
- QueueState state = cs.getConfiguration().getState(getQueuePath());
-
- Map<QueueACL, AccessControlList> acls =
- cs.getConfiguration().getAcls(getQueuePath());
-
- setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
- defaultLabelExpression, capacitiyByNodeLabels, maxCapacityByNodeLabels,
- cs.getConfiguration().getReservationContinueLook());
this.childQueues = new TreeSet<CSQueue>(queueComparator);
+
+ setupQueueConfigs(cs.getClusterResource());
LOG.info("Initialized parent-queue " + queueName +
" name=" + queueName +
", fullname=" + getQueuePath());
}
- synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
- float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity,
- QueueState state, Map<QueueACL, AccessControlList> acls,
- Set<String> accessibleLabels, String defaultLabelExpression,
- Map<String, Float> nodeLabelCapacities,
- Map<String, Float> maximumCapacitiesByLabel,
- boolean reservationContinueLooking) throws IOException {
- super.setupQueueConfigs(clusterResource, capacity, absoluteCapacity,
- maximumCapacity, absoluteMaxCapacity, state, acls, accessibleLabels,
- defaultLabelExpression, nodeLabelCapacities, maximumCapacitiesByLabel,
- reservationContinueLooking);
- StringBuilder aclsString = new StringBuilder();
+ synchronized void setupQueueConfigs(Resource clusterResource)
+ throws IOException {
+ super.setupQueueConfigs(clusterResource);
+ StringBuilder aclsString = new StringBuilder();
for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
@@ -147,10 +121,10 @@ public class ParentQueue extends AbstractCSQueue {
}
LOG.info(queueName +
- ", capacity=" + capacity +
- ", asboluteCapacity=" + absoluteCapacity +
- ", maxCapacity=" + maximumCapacity +
- ", asboluteMaxCapacity=" + absoluteMaxCapacity +
+ ", capacity=" + this.queueCapacities.getCapacity() +
+ ", asboluteCapacity=" + this.queueCapacities.getAbsoluteCapacity() +
+ ", maxCapacity=" + this.queueCapacities.getMaximumCapacity() +
+ ", asboluteMaxCapacity=" + this.queueCapacities.getAbsoluteMaximumCapacity() +
", state=" + state +
", acls=" + aclsString +
", labels=" + labelStrBuilder.toString() + "\n" +
@@ -166,19 +140,19 @@ public class ParentQueue extends AbstractCSQueue {
}
float delta = Math.abs(1.0f - childCapacities); // crude way to check
// allow capacities being set to 0, and enforce child 0 if parent is 0
- if (((capacity > 0) && (delta > PRECISION)) ||
- ((capacity == 0) && (childCapacities > 0))) {
+ if (((queueCapacities.getCapacity() > 0) && (delta > PRECISION)) ||
+ ((queueCapacities.getCapacity() == 0) && (childCapacities > 0))) {
throw new IllegalArgumentException("Illegal" +
" capacity of " + childCapacities +
" for children of queue " + queueName);
}
// check label capacities
for (String nodeLabel : labelManager.getClusterNodeLabels()) {
- float capacityByLabel = getCapacityByNodeLabel(nodeLabel);
+ float capacityByLabel = queueCapacities.getCapacity(nodeLabel);
// check children's labels
float sum = 0;
for (CSQueue queue : childQueues) {
- sum += queue.getCapacityByNodeLabel(nodeLabel);
+ sum += queue.getQueueCapacities().getCapacity(nodeLabel);
}
if ((capacityByLabel > 0 && Math.abs(1.0f - sum) > PRECISION)
|| (capacityByLabel == 0) && (sum > 0)) {
@@ -254,8 +228,8 @@ public class ParentQueue extends AbstractCSQueue {
public String toString() {
return queueName + ": " +
"numChildQueue= " + childQueues.size() + ", " +
- "capacity=" + capacity + ", " +
- "absoluteCapacity=" + absoluteCapacity + ", " +
+ "capacity=" + queueCapacities.getCapacity() + ", " +
+ "absoluteCapacity=" + queueCapacities.getAbsoluteCapacity() + ", " +
"usedResources=" + queueUsage.getUsed() +
"usedCapacity=" + getUsedCapacity() + ", " +
"numApps=" + getNumApplications() + ", " +
@@ -263,9 +237,8 @@ public class ParentQueue extends AbstractCSQueue {
}
@Override
- public synchronized void reinitialize(
- CSQueue newlyParsedQueue, Resource clusterResource)
- throws IOException {
+ public synchronized void reinitialize(CSQueue newlyParsedQueue,
+ Resource clusterResource) throws IOException {
// Sanity check
if (!(newlyParsedQueue instanceof ParentQueue) ||
!newlyParsedQueue.getQueuePath().equals(getQueuePath())) {
@@ -276,18 +249,7 @@ public class ParentQueue extends AbstractCSQueue {
ParentQueue newlyParsedParentQueue = (ParentQueue)newlyParsedQueue;
// Set new configs
- setupQueueConfigs(clusterResource,
- newlyParsedParentQueue.capacity,
- newlyParsedParentQueue.absoluteCapacity,
- newlyParsedParentQueue.maximumCapacity,
- newlyParsedParentQueue.absoluteMaxCapacity,
- newlyParsedParentQueue.state,
- newlyParsedParentQueue.acls,
- newlyParsedParentQueue.accessibleLabels,
- newlyParsedParentQueue.defaultLabelExpression,
- newlyParsedParentQueue.capacitiyByNodeLabels,
- newlyParsedParentQueue.maxCapacityByNodeLabels,
- newlyParsedParentQueue.reservationsContinueLooking);
+ setupQueueConfigs(clusterResource);
// Re-configure existing child queues and add new ones
// The CS has already checked to ensure all existing child queues are present!
@@ -512,7 +474,7 @@ public class ParentQueue extends AbstractCSQueue {
labelManager.getResourceByLabel(label, clusterResource));
// if any of the label doesn't beyond limit, we can allocate on this node
if (currentAbsoluteLabelUsedCapacity >=
- getAbsoluteMaximumCapacityByNodeLabel(label)) {
+ queueCapacities.getAbsoluteMaximumCapacity(label)) {
if (LOG.isDebugEnabled()) {
LOG.debug(getQueueName() + " used=" + queueUsage.getUsed()
+ " current-capacity (" + queueUsage.getUsed(label) + ") "
@@ -540,7 +502,8 @@ public class ParentQueue extends AbstractCSQueue {
Resources.subtract(queueUsage.getUsed(), reservedResources),
clusterResource);
- if (capacityWithoutReservedCapacity <= absoluteMaxCapacity) {
+ if (capacityWithoutReservedCapacity <= queueCapacities
+ .getAbsoluteMaximumCapacity()) {
if (LOG.isDebugEnabled()) {
LOG.debug("parent: try to use reserved: " + getQueueName()
+ " usedResources: " + queueUsage.getUsed().getMemory()
@@ -550,7 +513,7 @@ public class ParentQueue extends AbstractCSQueue {
/ clusterResource.getMemory()
+ " potentialNewWithoutReservedCapacity: "
+ capacityWithoutReservedCapacity + " ( " + " max-capacity: "
- + absoluteMaxCapacity + ")");
+ + queueCapacities.getAbsoluteMaximumCapacity() + ")");
}
// we could potentially use this node instead of reserved node
return true;
@@ -760,13 +723,6 @@ public class ParentQueue extends AbstractCSQueue {
}
}
}
-
- @Override
- public float getAbsActualCapacity() {
- // for now, simply return actual capacity = guaranteed capacity for parent
- // queue
- return absoluteCapacity;
- }
public synchronized int getNumApplications() {
return numApplications;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
index 0725959..f91a6d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/PlanQueue.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.io.IOException;
-import java.util.HashMap;
-import java.util.HashSet;
import java.util.Iterator;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -101,16 +99,7 @@ public class PlanQueue extends ParentQueue {
}
// Set new configs
- setupQueueConfigs(clusterResource, newlyParsedParentQueue.getCapacity(),
- newlyParsedParentQueue.getAbsoluteCapacity(),
- newlyParsedParentQueue.getMaximumCapacity(),
- newlyParsedParentQueue.getAbsoluteMaximumCapacity(),
- newlyParsedParentQueue.getState(), newlyParsedParentQueue.getACLs(),
- newlyParsedParentQueue.accessibleLabels,
- newlyParsedParentQueue.defaultLabelExpression,
- newlyParsedParentQueue.capacitiyByNodeLabels,
- newlyParsedParentQueue.maxCapacityByNodeLabels,
- newlyParsedParentQueue.getReservationContinueLooking());
+ setupQueueConfigs(clusterResource);
updateQuotas(newlyParsedParentQueue.userLimit,
newlyParsedParentQueue.userLimitFactor,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index a0e6d8c..962a636 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -19,12 +19,18 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
+
+import com.google.common.collect.Sets;
public class QueueCapacities {
private static final String NL = CommonNodeLabelsManager.NO_LABEL;
@@ -32,13 +38,15 @@ public class QueueCapacities {
private Map<String, Capacities> capacitiesMap;
private ReadLock readLock;
private WriteLock writeLock;
+ private final boolean isRoot;
- public QueueCapacities() {
+ public QueueCapacities(boolean isRoot) {
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
readLock = lock.readLock();
writeLock = lock.writeLock();
capacitiesMap = new HashMap<String, Capacities>();
+ this.isRoot = isRoot;
}
// Usage enum here to make implement cleaner
@@ -58,6 +66,18 @@ public class QueueCapacities {
public Capacities() {
capacitiesArr = new float[CapacityType.values().length];
}
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append("{used=" + capacitiesArr[0] + "%, ");
+ sb.append("abs_used=" + capacitiesArr[1] + "%, ");
+ sb.append("max_cap=" + capacitiesArr[2] + "%, ");
+ sb.append("abs_max_cap=" + capacitiesArr[3] + "%, ");
+ sb.append("cap=" + capacitiesArr[4] + "%, ");
+ sb.append("abs_cap=" + capacitiesArr[5] + "%}");
+ return sb.toString();
+ }
}
private float _get(String label, CapacityType type) {
@@ -127,6 +147,10 @@ public class QueueCapacities {
}
public float getCapacity(String label) {
+ if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL) && isRoot) {
+ return 1f;
+ }
+
return _get(label, CapacityType.CAP);
}
@@ -144,6 +168,9 @@ public class QueueCapacities {
}
public float getAbsoluteCapacity(String label) {
+ if (StringUtils.equals(label, RMNodeLabelsManager.NO_LABEL) && isRoot) {
+ return 1f;
+ }
return _get(label, CapacityType.ABS_CAP);
}
@@ -188,4 +215,43 @@ public class QueueCapacities {
public void setAbsoluteMaximumCapacity(String label, float value) {
_set(label, CapacityType.ABS_MAX_CAP, value);
}
+
+ /**
+ * Clear configurable fields, like
+ * (absolute)capacity/(absolute)maximum-capacity, this will be used by queue
+ * reinitialize, when we reinitialize a queue, we will first clear all
+ * configurable fields, and load new values
+ */
+ public void clearConfigurableFields() {
+ try {
+ writeLock.lock();
+ for (String label : capacitiesMap.keySet()) {
+ _set(label, CapacityType.CAP, 0);
+ _set(label, CapacityType.MAX_CAP, 0);
+ _set(label, CapacityType.ABS_CAP, 0);
+ _set(label, CapacityType.ABS_MAX_CAP, 0);
+ }
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ public Set<String> getExistingNodeLabels() {
+ try {
+ readLock.lock();
+ return new HashSet<String>(capacitiesMap.keySet());
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
+ public String toString() {
+ try {
+ readLock.lock();
+ return this.capacitiesMap.toString();
+ } finally {
+ readLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
index c4424b5..a8d17cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ReservationQueue.java
@@ -62,11 +62,11 @@ public class ReservationQueue extends LeafQueue {
throw new IOException("Trying to reinitialize " + getQueuePath()
+ " from " + newlyParsedQueue.getQueuePath());
}
+ super.reinitialize(newlyParsedQueue, clusterResource);
CSQueueUtils.updateQueueStatistics(
parent.schedulerContext.getResourceCalculator(), newlyParsedQueue,
parent, parent.schedulerContext.getClusterResource(),
parent.schedulerContext.getMinimumResourceCapability());
- super.reinitialize(newlyParsedQueue, clusterResource);
updateQuotas(parent.getUserLimitForReservation(),
parent.getUserLimitFactor(),
parent.getMaxApplicationsForReservations(),
@@ -108,9 +108,9 @@ public class ReservationQueue extends LeafQueue {
maxApplicationsPerUser = maxAppsPerUserForReservation;
}
- // used by the super constructor, we initialize to zero
- protected float getCapacityFromConf() {
- return 0f;
+ @Override
+ protected void setupConfigurableCapacities() {
+ CSQueueUtils.updateAndCheckCapacitiesByLabel(getQueuePath(),
+ queueCapacities, parent == null ? null : parent.getQueueCapacities());
}
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
index d643c9d..5135ba9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCSQueueUtils.java
@@ -202,33 +202,33 @@ public class TestCSQueueUtils {
LOG.info("t2 l2q2 " + result);
//some usage, but below the base capacity
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
- l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q2.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.4f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//usage gt base on parent sibling
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
- l1q2.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
+ l1q2.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.3f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//same as last, but with usage also on direct parent
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
- l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
+ l1q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.1f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
LOG.info("t2 l2q2 " + result);
//add to direct sibling, below the threshold of effect at present
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.3f, result, 0.000001f);
@@ -236,9 +236,9 @@ public class TestCSQueueUtils {
//add to direct sibling, now above the threshold of effect
//(it's cumulative with prior tests)
- root.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l1q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
- l2q1.getResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ root.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l1q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
+ l2q1.getQueueResourceUsage().incUsed(Resources.multiply(clusterResource, 0.2f));
result = CSQueueUtils.getAbsoluteMaxAvailCapacity(
resourceCalculator, clusterResource, l2q2);
assertEquals( 0.1f, result, 0.000001f);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index edcd871..456178d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -389,11 +389,11 @@ public class TestCapacityScheduler {
public void testMaximumCapacitySetup() {
float delta = 0.0000001f;
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
- assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+ assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
conf.setMaximumCapacity(A, 50.0f);
- assertEquals(50.0f, conf.getMaximumCapacity(A),delta);
+ assertEquals(50.0f, conf.getNonLabeledQueueMaximumCapacity(A),delta);
conf.setMaximumCapacity(A, -1);
- assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getMaximumCapacity(A),delta);
+ assertEquals(CapacitySchedulerConfiguration.MAXIMUM_CAPACITY_VALUE,conf.getNonLabeledQueueMaximumCapacity(A),delta);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
index 261fa01..4853241 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacitySchedulerNodeLabelUpdate.java
@@ -91,7 +91,8 @@ public class TestCapacitySchedulerNodeLabelUpdate {
String label) {
CapacityScheduler scheduler = (CapacityScheduler) rm.getResourceScheduler();
CSQueue queue = scheduler.getQueue(queueName);
- Assert.assertEquals(memory, queue.getUsedResourceByLabel(label).getMemory());
+ Assert.assertEquals(memory, queue.getQueueResourceUsage().getUsed(label)
+ .getMemory());
}
@Test (timeout = 30000)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
index 89a5311..a468230 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -99,7 +99,7 @@ public class TestQueueCapacities {
}
private void internalTestModifyAndRead(String label) throws Exception {
- QueueCapacities qc = new QueueCapacities();
+ QueueCapacities qc = new QueueCapacities(false);
// First get returns 0 always
Assert.assertEquals(0f, get(qc, suffix, label), 1e-8);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index cf2e5ce..fc03581 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -418,18 +418,18 @@ public class TestQueueParsing {
// check capacity of A2
CSQueue qA2 = capacityScheduler.getQueue("a2");
Assert.assertEquals(0.7, qA2.getCapacity(), DELTA);
- Assert.assertEquals(0.5, qA2.getCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.5, qA2.getQueueCapacities().getCapacity("red"), DELTA);
Assert.assertEquals(0.07, qA2.getAbsoluteCapacity(), DELTA);
- Assert.assertEquals(0.25, qA2.getAbsoluteCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.25, qA2.getQueueCapacities().getAbsoluteCapacity("red"), DELTA);
Assert.assertEquals(0.1275, qA2.getAbsoluteMaximumCapacity(), DELTA);
- Assert.assertEquals(0.3, qA2.getAbsoluteMaximumCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.3, qA2.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
// check capacity of B3
CSQueue qB3 = capacityScheduler.getQueue("b3");
Assert.assertEquals(0.18, qB3.getAbsoluteCapacity(), DELTA);
- Assert.assertEquals(0.125, qB3.getAbsoluteCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(0.125, qB3.getQueueCapacities().getAbsoluteCapacity("red"), DELTA);
Assert.assertEquals(0.35, qB3.getAbsoluteMaximumCapacity(), DELTA);
- Assert.assertEquals(1, qB3.getAbsoluteMaximumCapacityByNodeLabel("red"), DELTA);
+ Assert.assertEquals(1, qB3.getQueueCapacities().getAbsoluteMaximumCapacity("red"), DELTA);
}
private void
@@ -641,8 +641,8 @@ public class TestQueueParsing {
// check root queue's capacity by label -- they should be all zero
CSQueue root = capacityScheduler.getQueue(CapacitySchedulerConfiguration.ROOT);
- Assert.assertEquals(0, root.getCapacityByNodeLabel("red"), DELTA);
- Assert.assertEquals(0, root.getCapacityByNodeLabel("blue"), DELTA);
+ Assert.assertEquals(0, root.getQueueCapacities().getCapacity("red"), DELTA);
+ Assert.assertEquals(0, root.getQueueCapacities().getCapacity("blue"), DELTA);
CSQueue a = capacityScheduler.getQueue("a");
Assert.assertEquals(0.10, a.getAbsoluteCapacity(), DELTA);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee2b6bc2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
index d418dab..63a48bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesCapacitySched.java
@@ -395,9 +395,9 @@ public class TestRMWebServicesCapacitySched extends JerseyTest {
String qshortName = qArr[qArr.length - 1];
assertEquals("usedCapacity doesn't match", 0, info.usedCapacity, 1e-3f);
- assertEquals("capacity doesn't match", csConf.getCapacity(q),
+ assertEquals("capacity doesn't match", csConf.getNonLabeledQueueCapacity(q),
info.capacity, 1e-3f);
- float expectCapacity = csConf.getMaximumCapacity(q);
+ float expectCapacity = csConf.getNonLabeledQueueMaximumCapacity(q);
float expectAbsMaxCapacity = parentAbsMaxCapacity * (info.maxCapacity/100);
if (CapacitySchedulerConfiguration.UNDEFINED == expectCapacity) {
expectCapacity = 100;
[13/50] [abbrv] hadoop git commit: YARN-2918. RM should not fail on
startup if queue's configured labels do not exist in cluster-node-labels.
Contributed by Wangda Tan (cherry picked from commit
f489a4ec969f3727d03c8e85d51af1018fc0b2a1)
Posted by sj...@apache.org.
YARN-2918. RM should not fail on startup if queue's configured labels do not exist in cluster-node-labels. Contributed by Wangda Tan
(cherry picked from commit f489a4ec969f3727d03c8e85d51af1018fc0b2a1)
(cherry picked from commit d817fbb34d6e34991c6e512c20d71387750a98f4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23362649
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23362649
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23362649
Branch: refs/heads/branch-2.6
Commit: 2336264900feb9dc265a96e660c6c5d1dbd17e2d
Parents: ee2b6bc
Author: Jian He <ji...@apache.org>
Authored: Thu May 7 17:35:41 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sun Sep 6 14:15:33 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../ApplicationMasterService.java | 2 +-
.../server/resourcemanager/RMAppManager.java | 2 +-
.../server/resourcemanager/RMServerUtils.java | 5 +-
.../scheduler/SchedulerUtils.java | 67 +++++-----
.../scheduler/capacity/AbstractCSQueue.java | 6 +-
.../scheduler/capacity/CSQueueUtils.java | 28 +---
.../CapacitySchedulerConfiguration.java | 33 +++++
.../scheduler/capacity/LeafQueue.java | 4 +-
.../scheduler/TestSchedulerUtils.java | 134 +++++++++++++++----
.../scheduler/capacity/TestQueueParsing.java | 104 +++++++++++++-
11 files changed, 294 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a6bbdf3..1ad0199 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -192,6 +192,9 @@ Release 2.6.1 - UNRELEASED
YARN-3124. Fixed CS LeafQueue/ParentQueue to use QueueCapacities to track
capacities-by-label. (Wangda Tan via jianhe)
+ YARN-2918. RM should not fail on startup if queue's configured labels do
+ not exist in cluster-node-labels. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
index c896bf5..8ed1621 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java
@@ -501,7 +501,7 @@ public class ApplicationMasterService extends AbstractService implements
try {
RMServerUtils.normalizeAndValidateRequests(ask,
rScheduler.getMaximumResourceCapability(), app.getQueue(),
- rScheduler);
+ rScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("Invalid resource ask by application " + appAttemptId, e);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index fbae710..e7d6f4c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -375,7 +375,7 @@ public class RMAppManager implements EventHandler<RMAppManagerEvent>,
try {
SchedulerUtils.normalizeAndValidateRequest(amReq,
scheduler.getMaximumResourceCapability(),
- submissionContext.getQueue(), scheduler, isRecovery);
+ submissionContext.getQueue(), scheduler, isRecovery, rmContext);
} catch (InvalidResourceRequestException e) {
LOG.warn("RM app submission failed in validating AM resource request"
+ " for application " + submissionContext.getApplicationId(), e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index 800cfec..bbab0a9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -90,11 +90,12 @@ public class RMServerUtils {
* requested memory/vcore is non-negative and not greater than max
*/
public static void normalizeAndValidateRequests(List<ResourceRequest> ask,
- Resource maximumResource, String queueName, YarnScheduler scheduler)
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext)
throws InvalidResourceRequestException {
for (ResourceRequest resReq : ask) {
SchedulerUtils.normalizeAndvalidateRequest(resReq, maximumResource,
- queueName, scheduler);
+ queueName, scheduler, rmContext);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index bdc9da1..d135485 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -209,7 +210,7 @@ public class SchedulerUtils {
public static void normalizeAndValidateRequest(ResourceRequest resReq,
Resource maximumResource, String queueName, YarnScheduler scheduler,
- boolean isRecovery)
+ boolean isRecovery, RMContext rmContext)
throws InvalidResourceRequestException {
QueueInfo queueInfo = null;
@@ -221,15 +222,16 @@ public class SchedulerUtils {
}
SchedulerUtils.normalizeNodeLabelExpressionInRequest(resReq, queueInfo);
if (!isRecovery) {
- validateResourceRequest(resReq, maximumResource, queueInfo);
+ validateResourceRequest(resReq, maximumResource, queueInfo, rmContext);
}
}
public static void normalizeAndvalidateRequest(ResourceRequest resReq,
- Resource maximumResource, String queueName, YarnScheduler scheduler)
+ Resource maximumResource, String queueName, YarnScheduler scheduler,
+ RMContext rmContext)
throws InvalidResourceRequestException {
normalizeAndValidateRequest(resReq, maximumResource, queueName, scheduler,
- false);
+ false, rmContext);
}
/**
@@ -239,8 +241,8 @@ public class SchedulerUtils {
* @throws <code>InvalidResourceRequestException</code> when there is invalid
* request
*/
- public static void validateResourceRequest(ResourceRequest resReq,
- Resource maximumResource, QueueInfo queueInfo)
+ private static void validateResourceRequest(ResourceRequest resReq,
+ Resource maximumResource, QueueInfo queueInfo, RMContext rmContext)
throws InvalidResourceRequestException {
if (resReq.getCapability().getMemory() < 0 ||
resReq.getCapability().getMemory() > maximumResource.getMemory()) {
@@ -283,7 +285,7 @@ public class SchedulerUtils {
if (labelExp != null && !labelExp.trim().isEmpty() && queueInfo != null) {
if (!checkQueueLabelExpression(queueInfo.getAccessibleNodeLabels(),
- labelExp)) {
+ labelExp, rmContext)) {
throw new InvalidResourceRequestException("Invalid resource request"
+ ", queue="
+ queueInfo.getQueueName()
@@ -316,26 +318,6 @@ public class SchedulerUtils {
return false;
}
- public static void checkIfLabelInClusterNodeLabels(RMNodeLabelsManager mgr,
- Set<String> labels) throws IOException {
- if (mgr == null) {
- if (labels != null && !labels.isEmpty()) {
- throw new IOException("NodeLabelManager is null, please check");
- }
- return;
- }
-
- if (labels != null) {
- for (String label : labels) {
- if (!label.equals(RMNodeLabelsManager.ANY)
- && !mgr.containsNodeLabel(label)) {
- throw new IOException("NodeLabelManager doesn't include label = "
- + label + ", please check.");
- }
- }
- }
- }
-
public static boolean checkNodeLabelExpression(Set<String> nodeLabels,
String labelExpression) {
// empty label expression can only allocate on node with empty labels
@@ -356,19 +338,36 @@ public class SchedulerUtils {
return true;
}
+ /**
+ * Check queue label expression, check if node label in queue's
+ * node-label-expression existed in clusterNodeLabels if rmContext != null
+ */
public static boolean checkQueueLabelExpression(Set<String> queueLabels,
- String labelExpression) {
- if (queueLabels != null && queueLabels.contains(RMNodeLabelsManager.ANY)) {
- return true;
- }
+ String labelExpression, RMContext rmContext) {
// if label expression is empty, we can allocate container on any node
if (labelExpression == null) {
return true;
}
for (String str : labelExpression.split("&&")) {
- if (!str.trim().isEmpty()
- && (queueLabels == null || !queueLabels.contains(str.trim()))) {
- return false;
+ str = str.trim();
+ if (!str.trim().isEmpty()) {
+ // check queue label
+ if (queueLabels == null) {
+ return false;
+ } else {
+ if (!queueLabels.contains(str)
+ && !queueLabels.contains(RMNodeLabelsManager.ANY)) {
+ return false;
+ }
+ }
+
+ // check node label manager contains this label
+ if (null != rmContext) {
+ RMNodeLabelsManager nlm = rmContext.getNodeLabelManager();
+ if (nlm != null && !nlm.containsNodeLabel(str)) {
+ return false;
+ }
+ }
}
}
return true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 52629b3..f7d49c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -97,11 +97,9 @@ public abstract class AbstractCSQueue implements CSQueue {
protected void setupConfigurableCapacities() {
CSQueueUtils.loadUpdateAndCheckCapacities(
getQueuePath(),
- accessibleLabels,
csContext.getConfiguration(),
queueCapacities,
- parent == null ? null : parent.getQueueCapacities(),
- csContext.getRMContext().getNodeLabelManager());
+ parent == null ? null : parent.getQueueCapacities());
}
@Override
@@ -232,8 +230,6 @@ public abstract class AbstractCSQueue implements CSQueue {
if (this.accessibleLabels == null && parent != null) {
this.accessibleLabels = parent.getAccessibleNodeLabels();
}
- SchedulerUtils.checkIfLabelInClusterNodeLabels(labelManager,
- this.accessibleLabels);
// inherit from parent if labels not set
if (this.defaultLabelExpression == null && parent != null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 865b0b4..cbc20b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -117,10 +117,9 @@ class CSQueueUtils {
* - Check if capacities/absolute-capacities legal
*/
public static void loadUpdateAndCheckCapacities(String queuePath,
- Set<String> accessibleLabels, CapacitySchedulerConfiguration csConf,
- QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities,
- RMNodeLabelsManager nlm) {
- loadCapacitiesByLabelsFromConf(queuePath, accessibleLabels, nlm,
+ CapacitySchedulerConfiguration csConf,
+ QueueCapacities queueCapacities, QueueCapacities parentQueueCapacities) {
+ loadCapacitiesByLabelsFromConf(queuePath,
queueCapacities, csConf);
updateAbsoluteCapacitiesByNodeLabels(queueCapacities, parentQueueCapacities);
@@ -128,28 +127,13 @@ class CSQueueUtils {
capacitiesSanityCheck(queuePath, queueCapacities);
}
- // Considered NO_LABEL, ANY and null cases
- private static Set<String> normalizeAccessibleNodeLabels(Set<String> labels,
- RMNodeLabelsManager mgr) {
- Set<String> accessibleLabels = new HashSet<String>();
- if (labels != null) {
- accessibleLabels.addAll(labels);
- }
- if (accessibleLabels.contains(CommonNodeLabelsManager.ANY)) {
- accessibleLabels.addAll(mgr.getClusterNodeLabels());
- }
- accessibleLabels.add(CommonNodeLabelsManager.NO_LABEL);
-
- return accessibleLabels;
- }
-
private static void loadCapacitiesByLabelsFromConf(String queuePath,
- Set<String> labels, RMNodeLabelsManager mgr,
QueueCapacities queueCapacities, CapacitySchedulerConfiguration csConf) {
queueCapacities.clearConfigurableFields();
- labels = normalizeAccessibleNodeLabels(labels, mgr);
+ Set<String> configuredNodelabels =
+ csConf.getConfiguredNodeLabels(queuePath);
- for (String label : labels) {
+ for (String label : configuredNodelabels) {
if (label.equals(CommonNodeLabelsManager.NO_LABEL)) {
queueCapacities.setCapacity(CommonNodeLabelsManager.NO_LABEL,
csConf.getNonLabeledQueueCapacity(queuePath) / 100);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index f5e86b3..7bee438 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -23,8 +23,10 @@ import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import java.util.StringTokenizer;
@@ -780,4 +782,35 @@ public class CapacitySchedulerConfiguration extends Configuration {
DEFAULT_RESERVATION_ENFORCEMENT_WINDOW);
return enforcementWindow;
}
+
+ /**
+ * Get configured node labels in a given queuePath
+ */
+ public Set<String> getConfiguredNodeLabels(String queuePath) {
+ Set<String> configuredNodeLabels = new HashSet<String>();
+ Entry<String, String> e = null;
+
+ Iterator<Entry<String, String>> iter = iterator();
+ while (iter.hasNext()) {
+ e = iter.next();
+ String key = e.getKey();
+
+ if (key.startsWith(getQueuePrefix(queuePath) + ACCESSIBLE_NODE_LABELS
+ + DOT)) {
+ // Find <label-name> in
+ // <queue-path>.accessible-node-labels.<label-name>.property
+ int labelStartIdx =
+ key.indexOf(ACCESSIBLE_NODE_LABELS)
+ + ACCESSIBLE_NODE_LABELS.length() + 1;
+ int labelEndIndx = key.indexOf('.', labelStartIdx);
+ String labelName = key.substring(labelStartIdx, labelEndIndx);
+ configuredNodeLabels.add(labelName);
+ }
+ }
+
+ // always add NO_LABEL
+ configuredNodeLabels.add(RMNodeLabelsManager.NO_LABEL);
+
+ return configuredNodeLabels;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f9f9ca9..f8a6af3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -173,8 +173,8 @@ public class LeafQueue extends AbstractCSQueue {
maxAMResourcePerQueuePercent =
conf.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
- if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
- this.defaultLabelExpression)) {
+ if (!SchedulerUtils.checkQueueLabelExpression(
+ this.accessibleLabels, this.defaultLabelExpression, null)) {
throw new IOException("Invalid default label expression of "
+ " queue="
+ getQueueName()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
index 8abaeb6..0fd598c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerUtils.java
@@ -64,8 +64,10 @@ import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
+import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MockRMWithAMS;
import org.apache.hadoop.yarn.server.resourcemanager.TestAMAuthorization.MyContainerManager;
+import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.MemoryRMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
@@ -83,12 +85,15 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.collect.ImmutableSet;
import com.google.common.collect.Sets;
public class TestSchedulerUtils {
private static final Log LOG = LogFactory.getLog(TestSchedulerUtils.class);
+ private RMContext rmContext = getMockRMContext();
+
@Test (timeout = 30000)
public void testNormalizeRequest() {
ResourceCalculator resourceCalculator = new DefaultResourceCalculator();
@@ -206,6 +211,8 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y"));
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -213,22 +220,44 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression(" ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is a subset of queue labels");
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
+ }
+
+ // same as above, but cluster node labels don't contains label being
+ // requested. should fail
+ try {
+ // set queue accessible node labesl to [x, y]
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), ResourceRequest.ANY, resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler, rmContext);
+
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
// queue has labels, failed cases (when ask a label not included by queue)
@@ -236,6 +265,8 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y"));
Resource resource = Resources.createResource(
0,
@@ -244,9 +275,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("z");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// we don't allow specify more than two node labels in a single expression
@@ -255,6 +289,8 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y"));
Resource resource = Resources.createResource(
0,
@@ -263,9 +299,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x && y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// queue doesn't have label, succeed (when request no label)
@@ -280,15 +319,15 @@ public class TestSchedulerUtils {
ResourceRequest resReq = BuilderUtils.newResourceRequest(
mock(Priority.class), ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression(" ");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when request labels is empty");
@@ -299,6 +338,9 @@ public class TestSchedulerUtils {
// set queue accessible node labels to empty
queueAccessibleNodeLabels.clear();
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x"));
+
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -306,9 +348,12 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x"));
}
// queue is "*", always succeeded
@@ -317,6 +362,9 @@ public class TestSchedulerUtils {
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x","y", "z"));
+
Resource resource = Resources.createResource(
0,
YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
@@ -324,18 +372,39 @@ public class TestSchedulerUtils {
mock(Priority.class), ResourceRequest.ANY, resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("y");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
resReq.setNodeLabelExpression("z");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
} catch (InvalidResourceRequestException e) {
e.printStackTrace();
fail("Should be valid when queue can access any labels");
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y", "z"));
+ }
+
+ // same as above, but cluster node labels don't contains label, should fail
+ try {
+ // set queue accessible node labels to empty
+ queueAccessibleNodeLabels.clear();
+ queueAccessibleNodeLabels.add(RMNodeLabelsManager.ANY);
+
+ Resource resource = Resources.createResource(
+ 0,
+ YarnConfiguration.DEFAULT_RM_SCHEDULER_MINIMUM_ALLOCATION_VCORES);
+ ResourceRequest resReq = BuilderUtils.newResourceRequest(
+ mock(Priority.class), ResourceRequest.ANY, resource, 1);
+ resReq.setNodeLabelExpression("x");
+ SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
+ scheduler, rmContext);
+ fail("Should fail");
+ } catch (InvalidResourceRequestException e) {
}
// we don't allow resource name other than ANY and specify label
@@ -343,6 +412,9 @@ public class TestSchedulerUtils {
// set queue accessible node labesl to [x, y]
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays.asList("x", "y"));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x", "y"));
+
Resource resource = Resources.createResource(
0,
@@ -351,9 +423,12 @@ public class TestSchedulerUtils {
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x", "y"));
}
// we don't allow resource name other than ANY and specify label even if
@@ -363,6 +438,8 @@ public class TestSchedulerUtils {
queueAccessibleNodeLabels.clear();
queueAccessibleNodeLabels.addAll(Arrays
.asList(CommonNodeLabelsManager.ANY));
+ rmContext.getNodeLabelManager().addToCluserNodeLabels(
+ ImmutableSet.of("x"));
Resource resource = Resources.createResource(
0,
@@ -371,9 +448,12 @@ public class TestSchedulerUtils {
mock(Priority.class), "rack", resource, 1);
resReq.setNodeLabelExpression("x");
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, "queue",
- scheduler);
+ scheduler, rmContext);
fail("Should fail");
} catch (InvalidResourceRequestException e) {
+ } finally {
+ rmContext.getNodeLabelManager().removeFromClusterNodeLabels(
+ Arrays.asList("x"));
}
}
@@ -395,7 +475,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Zero memory should be accepted");
}
@@ -409,7 +489,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Zero vcores should be accepted");
}
@@ -424,7 +504,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Max memory should be accepted");
}
@@ -439,7 +519,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
} catch (InvalidResourceRequestException e) {
fail("Max vcores should not be accepted");
}
@@ -453,7 +533,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("Negative memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -468,7 +548,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("Negative vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -484,7 +564,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("More than max memory should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -501,7 +581,7 @@ public class TestSchedulerUtils {
BuilderUtils.newResourceRequest(mock(Priority.class),
ResourceRequest.ANY, resource, 1);
SchedulerUtils.normalizeAndvalidateRequest(resReq, maxResource, null,
- mockScheduler);
+ mockScheduler, rmContext);
fail("More than max vcores should not be accepted");
} catch (InvalidResourceRequestException e) {
// expected
@@ -632,4 +712,12 @@ public class TestSchedulerUtils {
Assert.assertNull(applications.get(appId));
return app;
}
+
+ private static RMContext getMockRMContext() {
+ RMContext rmContext = mock(RMContext.class);
+ RMNodeLabelsManager nlm = new MemoryRMNodeLabelsManager();
+ nlm.init(new Configuration(false));
+ when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+ return rmContext;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23362649/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
index fc03581..d0c5d95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
@@ -501,7 +501,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(capacityScheduler);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -529,7 +529,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsInheritedNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -557,7 +557,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testSingleLevelQueueParsingWhenLabelsNotExistedInNodeLabelManager()
throws IOException {
YarnConfiguration conf = new YarnConfiguration();
@@ -585,7 +585,7 @@ public class TestQueueParsing {
ServiceOperations.stopQuietly(nodeLabelsManager);
}
- @Test(expected = Exception.class)
+ @Test
public void testQueueParsingWhenLabelsNotExist() throws IOException {
YarnConfiguration conf = new YarnConfiguration();
CapacitySchedulerConfiguration csConf =
@@ -659,4 +659,100 @@ public class TestQueueParsing {
DELTA);
capacityScheduler.stop();
}
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingFailWhenSumOfChildrenNonLabeledCapacityNot100Percent()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabels(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfiguration(csConf);
+ csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + ".c.c2", 5);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingFailWhenSumOfChildrenLabeledCapacityNot100Percent()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabels(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfigurationWithLabels(csConf);
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
+ "red", 24);
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
+
+ /**
+ * Test init a queue configuration, children's capacity for a given label
+ * doesn't equals to 100%. This expect IllegalArgumentException thrown.
+ */
+ @Test(expected = IllegalArgumentException.class)
+ public void testQueueParsingWithSumOfChildLabelCapacityNot100PercentWithWildCard()
+ throws IOException {
+ nodeLabelManager.addToCluserNodeLabels(ImmutableSet
+ .of("red", "blue"));
+
+ YarnConfiguration conf = new YarnConfiguration();
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration(conf);
+ setupQueueConfigurationWithLabels(csConf);
+ csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT + ".b.b3",
+ "red", 24);
+ csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT,
+ ImmutableSet.of(RMNodeLabelsManager.ANY));
+ csConf.setAccessibleNodeLabels(CapacitySchedulerConfiguration.ROOT + ".b",
+ ImmutableSet.of(RMNodeLabelsManager.ANY));
+
+ CapacityScheduler capacityScheduler = new CapacityScheduler();
+ RMContextImpl rmContext =
+ new RMContextImpl(null, null, null, null, null, null,
+ new RMContainerTokenSecretManager(csConf),
+ new NMTokenSecretManagerInRM(csConf),
+ new ClientToAMTokenSecretManagerInRM(), null);
+ rmContext.setNodeLabelManager(nodeLabelManager);
+ capacityScheduler.setConf(csConf);
+ capacityScheduler.setRMContext(rmContext);
+ capacityScheduler.init(csConf);
+ capacityScheduler.start();
+ ServiceOperations.stopQuietly(capacityScheduler);
+ }
}
[09/50] [abbrv] hadoop git commit: YARN-3098. Created common
QueueCapacities class in Capacity Scheduler to track capacities-by-labels of
queues. Contributed by Wangda Tan (cherry picked from commit
21d80b3dd90a8e33e51701887c8d9369ed4ab17d)
Posted by sj...@apache.org.
YARN-3098. Created common QueueCapacities class in Capacity Scheduler to track capacities-by-labels of queues. Contributed by Wangda Tan
(cherry picked from commit 21d80b3dd90a8e33e51701887c8d9369ed4ab17d)
(cherry picked from commit c0b1311a93614becc4a255af48fb7b697d491b80)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4c94f071
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4c94f071
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4c94f071
Branch: refs/heads/branch-2.6
Commit: 4c94f0714012e1a7cfce1ea7b60d3c86f0b5cce1
Parents: d9281fb
Author: Jian He <ji...@apache.org>
Authored: Tue Feb 3 11:43:12 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sat Sep 5 20:54:20 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/ResourceUsage.java | 80 +++-----
.../scheduler/capacity/QueueCapacities.java | 191 +++++++++++++++++++
.../scheduler/capacity/TestQueueCapacities.java | 127 ++++++++++++
4 files changed, 351 insertions(+), 50 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bac13ff..58c2139 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -19,6 +19,9 @@ Release 2.6.1 - UNRELEASED
YARN-3092. Created a common ResourceUsage class to track labeled resource
usages in Capacity Scheduler. (Wangda Tan via jianhe)
+ YARN-3098. Created common QueueCapacities class in Capacity Scheduler to
+ track capacities-by-labels of queues. (Wangda Tan via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 5a4cced..c651878 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -56,14 +56,10 @@ public class ResourceUsage {
private enum ResourceType {
USED(0), PENDING(1), AMUSED(2), RESERVED(3), HEADROOM(4);
- private int value;
+ private int idx;
private ResourceType(int value) {
- this.value = value;
- }
-
- public int getValue() {
- return this.value;
+ this.idx = value;
}
}
@@ -77,22 +73,6 @@ public class ResourceUsage {
resArr[i] = Resource.newInstance(0, 0);
}
}
-
- public Resource get(ResourceType type) {
- return resArr[type.getValue()];
- }
-
- public void set(ResourceType type, Resource res) {
- resArr[type.getValue()] = res;
- }
-
- public void inc(ResourceType type, Resource res) {
- Resources.addTo(resArr[type.getValue()], res);
- }
-
- public void dec(ResourceType type, Resource res) {
- Resources.subtractFrom(resArr[type.getValue()], res);
- }
}
/*
@@ -103,11 +83,11 @@ public class ResourceUsage {
}
public Resource getUsed(String label) {
- return internalGet(label, ResourceType.USED);
+ return _get(label, ResourceType.USED);
}
public void incUsed(String label, Resource res) {
- internalInc(label, ResourceType.USED, res);
+ _inc(label, ResourceType.USED, res);
}
public void incUsed(Resource res) {
@@ -119,7 +99,7 @@ public class ResourceUsage {
}
public void decUsed(String label, Resource res) {
- internalDec(label, ResourceType.USED, res);
+ _dec(label, ResourceType.USED, res);
}
public void setUsed(Resource res) {
@@ -127,7 +107,7 @@ public class ResourceUsage {
}
public void setUsed(String label, Resource res) {
- internalSet(label, ResourceType.USED, res);
+ _set(label, ResourceType.USED, res);
}
/*
@@ -138,11 +118,11 @@ public class ResourceUsage {
}
public Resource getPending(String label) {
- return internalGet(label, ResourceType.PENDING);
+ return _get(label, ResourceType.PENDING);
}
public void incPending(String label, Resource res) {
- internalInc(label, ResourceType.PENDING, res);
+ _inc(label, ResourceType.PENDING, res);
}
public void incPending(Resource res) {
@@ -154,7 +134,7 @@ public class ResourceUsage {
}
public void decPending(String label, Resource res) {
- internalDec(label, ResourceType.PENDING, res);
+ _dec(label, ResourceType.PENDING, res);
}
public void setPending(Resource res) {
@@ -162,7 +142,7 @@ public class ResourceUsage {
}
public void setPending(String label, Resource res) {
- internalSet(label, ResourceType.PENDING, res);
+ _set(label, ResourceType.PENDING, res);
}
/*
@@ -173,11 +153,11 @@ public class ResourceUsage {
}
public Resource getReserved(String label) {
- return internalGet(label, ResourceType.RESERVED);
+ return _get(label, ResourceType.RESERVED);
}
public void incReserved(String label, Resource res) {
- internalInc(label, ResourceType.RESERVED, res);
+ _inc(label, ResourceType.RESERVED, res);
}
public void incReserved(Resource res) {
@@ -189,7 +169,7 @@ public class ResourceUsage {
}
public void decReserved(String label, Resource res) {
- internalDec(label, ResourceType.RESERVED, res);
+ _dec(label, ResourceType.RESERVED, res);
}
public void setReserved(Resource res) {
@@ -197,7 +177,7 @@ public class ResourceUsage {
}
public void setReserved(String label, Resource res) {
- internalSet(label, ResourceType.RESERVED, res);
+ _set(label, ResourceType.RESERVED, res);
}
/*
@@ -208,11 +188,11 @@ public class ResourceUsage {
}
public Resource getHeadroom(String label) {
- return internalGet(label, ResourceType.HEADROOM);
+ return _get(label, ResourceType.HEADROOM);
}
public void incHeadroom(String label, Resource res) {
- internalInc(label, ResourceType.HEADROOM, res);
+ _inc(label, ResourceType.HEADROOM, res);
}
public void incHeadroom(Resource res) {
@@ -224,7 +204,7 @@ public class ResourceUsage {
}
public void decHeadroom(String label, Resource res) {
- internalDec(label, ResourceType.HEADROOM, res);
+ _dec(label, ResourceType.HEADROOM, res);
}
public void setHeadroom(Resource res) {
@@ -232,7 +212,7 @@ public class ResourceUsage {
}
public void setHeadroom(String label, Resource res) {
- internalSet(label, ResourceType.HEADROOM, res);
+ _set(label, ResourceType.HEADROOM, res);
}
/*
@@ -243,11 +223,11 @@ public class ResourceUsage {
}
public Resource getAMUsed(String label) {
- return internalGet(label, ResourceType.AMUSED);
+ return _get(label, ResourceType.AMUSED);
}
public void incAMUsed(String label, Resource res) {
- internalInc(label, ResourceType.AMUSED, res);
+ _inc(label, ResourceType.AMUSED, res);
}
public void incAMUsed(Resource res) {
@@ -259,7 +239,7 @@ public class ResourceUsage {
}
public void decAMUsed(String label, Resource res) {
- internalDec(label, ResourceType.AMUSED, res);
+ _dec(label, ResourceType.AMUSED, res);
}
public void setAMUsed(Resource res) {
@@ -267,7 +247,7 @@ public class ResourceUsage {
}
public void setAMUsed(String label, Resource res) {
- internalSet(label, ResourceType.AMUSED, res);
+ _set(label, ResourceType.AMUSED, res);
}
private static Resource normalize(Resource res) {
@@ -277,14 +257,14 @@ public class ResourceUsage {
return res;
}
- private Resource internalGet(String label, ResourceType type) {
+ private Resource _get(String label, ResourceType type) {
try {
readLock.lock();
UsageByLabel usage = usages.get(label);
if (null == usage) {
return Resources.none();
}
- return normalize(usage.get(type));
+ return normalize(usage.resArr[type.idx]);
} finally {
readLock.unlock();
}
@@ -300,31 +280,31 @@ public class ResourceUsage {
return usages.get(label);
}
- private void internalSet(String label, ResourceType type, Resource res) {
+ private void _set(String label, ResourceType type, Resource res) {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.set(type, res);
+ usage.resArr[type.idx] = res;
} finally {
writeLock.unlock();
}
}
- private void internalInc(String label, ResourceType type, Resource res) {
+ private void _inc(String label, ResourceType type, Resource res) {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.inc(type, res);
+ Resources.addTo(usage.resArr[type.idx], res);
} finally {
writeLock.unlock();
}
}
- private void internalDec(String label, ResourceType type, Resource res) {
+ private void _dec(String label, ResourceType type, Resource res) {
try {
writeLock.lock();
UsageByLabel usage = getAndAddIfMissing(label);
- usage.dec(type, res);
+ Resources.subtractFrom(usage.resArr[type.idx], res);
} finally {
writeLock.unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
new file mode 100644
index 0000000..a0e6d8c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -0,0 +1,191 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+
+public class QueueCapacities {
+ private static final String NL = CommonNodeLabelsManager.NO_LABEL;
+ private static final float LABEL_DOESNT_EXIST_CAP = 0f;
+ private Map<String, Capacities> capacitiesMap;
+ private ReadLock readLock;
+ private WriteLock writeLock;
+
+ public QueueCapacities() {
+ ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
+ readLock = lock.readLock();
+ writeLock = lock.writeLock();
+
+ capacitiesMap = new HashMap<String, Capacities>();
+ }
+
+ // Usage enum here to make implement cleaner
+ private enum CapacityType {
+ USED_CAP(0), ABS_USED_CAP(1), MAX_CAP(2), ABS_MAX_CAP(3), CAP(4), ABS_CAP(5);
+
+ private int idx;
+
+ private CapacityType(int idx) {
+ this.idx = idx;
+ }
+ }
+
+ private static class Capacities {
+ private float[] capacitiesArr;
+
+ public Capacities() {
+ capacitiesArr = new float[CapacityType.values().length];
+ }
+ }
+
+ private float _get(String label, CapacityType type) {
+ try {
+ readLock.lock();
+ Capacities cap = capacitiesMap.get(label);
+ if (null == cap) {
+ return LABEL_DOESNT_EXIST_CAP;
+ }
+ return cap.capacitiesArr[type.idx];
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ private void _set(String label, CapacityType type, float value) {
+ try {
+ writeLock.lock();
+ Capacities cap = capacitiesMap.get(label);
+ if (null == cap) {
+ cap = new Capacities();
+ capacitiesMap.put(label, cap);
+ }
+ cap.capacitiesArr[type.idx] = value;
+ } finally {
+ writeLock.unlock();
+ }
+ }
+
+ /* Used Capacity Getter and Setter */
+ public float getUsedCapacity() {
+ return _get(NL, CapacityType.USED_CAP);
+ }
+
+ public float getUsedCapacity(String label) {
+ return _get(label, CapacityType.USED_CAP);
+ }
+
+ public void setUsedCapacity(float value) {
+ _set(NL, CapacityType.USED_CAP, value);
+ }
+
+ public void setUsedCapacity(String label, float value) {
+ _set(label, CapacityType.USED_CAP, value);
+ }
+
+ /* Absolute Used Capacity Getter and Setter */
+ public float getAbsoluteUsedCapacity() {
+ return _get(NL, CapacityType.ABS_USED_CAP);
+ }
+
+ public float getAbsoluteUsedCapacity(String label) {
+ return _get(label, CapacityType.ABS_USED_CAP);
+ }
+
+ public void setAbsoluteUsedCapacity(float value) {
+ _set(NL, CapacityType.ABS_USED_CAP, value);
+ }
+
+ public void setAbsoluteUsedCapacity(String label, float value) {
+ _set(label, CapacityType.ABS_USED_CAP, value);
+ }
+
+ /* Capacity Getter and Setter */
+ public float getCapacity() {
+ return _get(NL, CapacityType.CAP);
+ }
+
+ public float getCapacity(String label) {
+ return _get(label, CapacityType.CAP);
+ }
+
+ public void setCapacity(float value) {
+ _set(NL, CapacityType.CAP, value);
+ }
+
+ public void setCapacity(String label, float value) {
+ _set(label, CapacityType.CAP, value);
+ }
+
+ /* Absolute Capacity Getter and Setter */
+ public float getAbsoluteCapacity() {
+ return _get(NL, CapacityType.ABS_CAP);
+ }
+
+ public float getAbsoluteCapacity(String label) {
+ return _get(label, CapacityType.ABS_CAP);
+ }
+
+ public void setAbsoluteCapacity(float value) {
+ _set(NL, CapacityType.ABS_CAP, value);
+ }
+
+ public void setAbsoluteCapacity(String label, float value) {
+ _set(label, CapacityType.ABS_CAP, value);
+ }
+
+ /* Maximum Capacity Getter and Setter */
+ public float getMaximumCapacity() {
+ return _get(NL, CapacityType.MAX_CAP);
+ }
+
+ public float getMaximumCapacity(String label) {
+ return _get(label, CapacityType.MAX_CAP);
+ }
+
+ public void setMaximumCapacity(float value) {
+ _set(NL, CapacityType.MAX_CAP, value);
+ }
+
+ public void setMaximumCapacity(String label, float value) {
+ _set(label, CapacityType.MAX_CAP, value);
+ }
+
+ /* Absolute Maximum Capacity Getter and Setter */
+ public float getAbsoluteMaximumCapacity() {
+ return _get(NL, CapacityType.ABS_MAX_CAP);
+ }
+
+ public float getAbsoluteMaximumCapacity(String label) {
+ return _get(label, CapacityType.ABS_MAX_CAP);
+ }
+
+ public void setAbsoluteMaximumCapacity(float value) {
+ _set(NL, CapacityType.ABS_MAX_CAP, value);
+ }
+
+ public void setAbsoluteMaximumCapacity(String label, float value) {
+ _set(label, CapacityType.ABS_MAX_CAP, value);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4c94f071/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
new file mode 100644
index 0000000..89a5311
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.Collection;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+@RunWith(Parameterized.class)
+public class TestQueueCapacities {
+ private static final Log LOG = LogFactory.getLog(TestQueueCapacities.class);
+ private String suffix;
+
+ @Parameterized.Parameters
+ public static Collection<String[]> getParameters() {
+ return Arrays.asList(new String[][] {
+ { "Capacity" },
+ { "AbsoluteCapacity" },
+ { "UsedCapacity" },
+ { "AbsoluteUsedCapacity" },
+ { "MaximumCapacity" },
+ { "AbsoluteMaximumCapacity" } });
+ }
+
+ public TestQueueCapacities(String suffix) {
+ this.suffix = suffix;
+ }
+
+ private static float get(QueueCapacities obj, String suffix,
+ String label) throws Exception {
+ return executeByName(obj, "get" + suffix, label, -1f);
+ }
+
+ private static void set(QueueCapacities obj, String suffix,
+ String label, float value) throws Exception {
+ executeByName(obj, "set" + suffix, label, value);
+ }
+
+ // Use reflection to avoid too much avoid code
+ private static float executeByName(QueueCapacities obj, String methodName,
+ String label, float value) throws Exception {
+ // We have 4 kinds of method
+ // 1. getXXX() : float
+ // 2. getXXX(label) : float
+ // 3. setXXX(float) : void
+ // 4. setXXX(label, float) : void
+ if (methodName.startsWith("get")) {
+ float result;
+ if (label == null) {
+ // 1.
+ Method method = QueueCapacities.class.getDeclaredMethod(methodName);
+ result = (Float) method.invoke(obj);
+ } else {
+ // 2.
+ Method method =
+ QueueCapacities.class.getDeclaredMethod(methodName, String.class);
+ result = (Float) method.invoke(obj, label);
+ }
+ return result;
+ } else {
+ if (label == null) {
+ // 3.
+ Method method =
+ QueueCapacities.class.getDeclaredMethod(methodName, Float.TYPE);
+ method.invoke(obj, value);
+ } else {
+ // 4.
+ Method method =
+ QueueCapacities.class.getDeclaredMethod(methodName, String.class,
+ Float.TYPE);
+ method.invoke(obj, label, value);
+ }
+ return -1f;
+ }
+ }
+
+ private void internalTestModifyAndRead(String label) throws Exception {
+ QueueCapacities qc = new QueueCapacities();
+
+ // First get returns 0 always
+ Assert.assertEquals(0f, get(qc, suffix, label), 1e-8);
+
+ // Set to 1, and check
+ set(qc, suffix, label, 1f);
+ Assert.assertEquals(1f, get(qc, suffix, label), 1e-8);
+
+ // Set to 2, and check
+ set(qc, suffix, label, 2f);
+ Assert.assertEquals(2f, get(qc, suffix, label), 1e-8);
+ }
+
+ void check(int mem, int cpu, Resource res) {
+ Assert.assertEquals(mem, res.getMemory());
+ Assert.assertEquals(cpu, res.getVirtualCores());
+ }
+
+ @Test
+ public void testModifyAndRead() throws Exception {
+ LOG.info("Test - " + suffix);
+ internalTestModifyAndRead(null);
+ internalTestModifyAndRead("label");
+ }
+}
\ No newline at end of file
[45/50] [abbrv] hadoop git commit: Creating 2.6.2 entries in
CHANGES.txt files.
Posted by sj...@apache.org.
Creating 2.6.2 entries in CHANGES.txt files.
(cherry picked from commit 33474939a048ac7bbe07df772f0000c84979722f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b99a8513
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b99a8513
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b99a8513
Branch: refs/heads/branch-2.6
Commit: b99a85139e0176d5d5c32ef50c002859650ffacf
Parents: 4cb7dba
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Thu Aug 20 17:21:06 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 16:00:06 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 12 ++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++++++++
hadoop-mapreduce-project/CHANGES.txt | 12 ++++++++++++
hadoop-yarn-project/CHANGES.txt | 12 ++++++++++++
4 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b99a8513/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 495f613..bae50e9 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1,5 +1,17 @@
Hadoop Change Log
+Release 2.6.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b99a8513/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2b0bf71..4bae4f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,5 +1,17 @@
Hadoop HDFS Change Log
+Release 2.6.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b99a8513/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index a33321b..cb0cca0 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1,5 +1,17 @@
Hadoop MapReduce Change Log
+Release 2.6.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b99a8513/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2f226dd..fe89d45 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1,5 +1,17 @@
Hadoop YARN Change Log
+Release 2.6.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
[43/50] [abbrv] hadoop git commit: Release process for 2.6.1: Set the
release date for 2.6.1
Posted by sj...@apache.org.
Release process for 2.6.1: Set the release date for 2.6.1
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41d19f47
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41d19f47
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41d19f47
Branch: refs/heads/branch-2.6
Commit: 41d19f47405652d821bbe74b645d29f048024bfe
Parents: e8c4544
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Wed Sep 23 15:15:36 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Wed Sep 23 15:15:36 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41d19f47/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1511483..495f613 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop Change Log
-Release 2.6.1 - 2015-09-16
+Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41d19f47/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2c38a58..2b0bf71 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop HDFS Change Log
-Release 2.6.1 - 2015-09-16
+Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41d19f47/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index d6a8f76..a33321b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop MapReduce Change Log
-Release 2.6.1 - 2015-09-16
+Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41d19f47/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ae4f089..2f226dd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop YARN Change Log
-Release 2.6.1 - 2015-09-16
+Release 2.6.1 - 2015-09-23
INCOMPATIBLE CHANGES
[32/50] [abbrv] hadoop git commit: YARN-1884. Added nodeHttpAddress
into ContainerReport and fixed the link to NM web page. Contributed by Xuan
Gong.
Posted by sj...@apache.org.
YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM web page. Contributed by Xuan Gong.
(cherry picked from commit 85f6d67fa78511f255fcfa810afc9a156a7b483b)
(cherry picked from commit 426535007bcdc67331f7a37b5d69cc20b37c26e0)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f4154bde
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f4154bde
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f4154bde
Branch: refs/heads/branch-2.6
Commit: f4154bdee8d52d655f5f35ba020c3f63f80b69be
Parents: 075d899
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Mar 11 19:35:19 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Fri Sep 11 11:45:29 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/api/records/ContainerReport.java | 17 ++++++++++++++-
.../src/main/proto/yarn_protos.proto | 1 +
.../hadoop/yarn/client/cli/ApplicationCLI.java | 11 +++++++---
.../hadoop/yarn/client/ProtocolHATestBase.java | 4 ++--
.../yarn/client/api/impl/TestAHSClient.java | 6 ++++--
.../yarn/client/api/impl/TestYarnClient.java | 7 +++++--
.../hadoop/yarn/client/cli/TestYarnCLI.java | 18 ++++++++++++----
.../records/impl/pb/ContainerReportPBImpl.java | 19 +++++++++++++++++
.../ApplicationHistoryManagerImpl.java | 2 +-
...pplicationHistoryManagerOnTimelineStore.java | 10 ++++++++-
.../metrics/ContainerMetricsConstants.java | 2 ++
.../yarn/server/webapp/AppAttemptBlock.java | 12 ++++++-----
.../hadoop/yarn/server/webapp/AppBlock.java | 11 ++++------
.../yarn/server/webapp/ContainerBlock.java | 7 ++++++-
.../yarn/server/webapp/dao/ContainerInfo.java | 5 +++++
.../metrics/ContainerCreatedEvent.java | 8 ++++++-
.../metrics/SystemMetricsPublisher.java | 5 ++++-
.../rmcontainer/RMContainer.java | 1 +
.../rmcontainer/RMContainerImpl.java | 22 ++++++++++++++++++--
.../metrics/TestSystemMetricsPublisher.java | 5 ++++-
21 files changed, 142 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b6bafb8..c82c892 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -205,6 +205,9 @@ Release 2.6.1 - 2015-09-09
YARN-4047. ClientRMService getApplications has high scheduler lock contention.
(Jason Lowe via jianhe)
+ YARN-1884. Added nodeHttpAddress into ContainerReport and fixed the link to NM
+ web page. (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
index 4cce77f..72b8edf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerReport.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.yarn.util.Records;
* <li>{@link ContainerState} of the container.</li>
* <li>Diagnostic information in case of errors.</li>
* <li>Log URL.</li>
+ * <li>nodeHttpAddress</li>
* </ul>
* </p>
*
@@ -54,7 +55,8 @@ public abstract class ContainerReport {
public static ContainerReport newInstance(ContainerId containerId,
Resource allocatedResource, NodeId assignedNode, Priority priority,
long creationTime, long finishTime, String diagnosticInfo, String logUrl,
- int containerExitStatus, ContainerState containerState) {
+ int containerExitStatus, ContainerState containerState,
+ String nodeHttpAddress) {
ContainerReport report = Records.newRecord(ContainerReport.class);
report.setContainerId(containerId);
report.setAllocatedResource(allocatedResource);
@@ -66,6 +68,7 @@ public abstract class ContainerReport {
report.setLogUrl(logUrl);
report.setContainerExitStatus(containerExitStatus);
report.setContainerState(containerState);
+ report.setNodeHttpAddress(nodeHttpAddress);
return report;
}
@@ -199,4 +202,16 @@ public abstract class ContainerReport {
@Unstable
public abstract void setContainerExitStatus(int containerExitStatus);
+ /**
+ * Get the Node Http address of the container
+ *
+ * @return the node http address of the container
+ */
+ @Public
+ @Unstable
+ public abstract String getNodeHttpAddress();
+
+ @Private
+ @Unstable
+ public abstract void setNodeHttpAddress(String nodeHttpAddress);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 5c86c2d..db01770 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -98,6 +98,7 @@ message ContainerReportProto {
optional string log_url = 8;
optional int32 container_exit_status = 9;
optional ContainerStateProto container_state = 10;
+ optional string node_http_address = 11;
}
enum YarnApplicationStateProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
index 83d212d..259f66b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/ApplicationCLI.java
@@ -58,7 +58,7 @@ public class ApplicationCLI extends YarnCLI {
"%30s\t%20s\t%35s\t%35s"
+ System.getProperty("line.separator");
private static final String CONTAINER_PATTERN =
- "%30s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ "%30s\t%20s\t%20s\t%20s\t%20s\t%20s\t%35s"
+ System.getProperty("line.separator");
private static final String APP_TYPE_CMD = "appTypes";
@@ -315,6 +315,9 @@ public class ApplicationCLI extends YarnCLI {
containerReportStr.println(containerReport.getLogUrl());
containerReportStr.print("\tHost : ");
containerReportStr.println(containerReport.getAssignedNode());
+ containerReportStr.print("\tNodeHttpAddress : ");
+ containerReportStr.println(containerReport.getNodeHttpAddress() == null
+ ? "N/A" : containerReport.getNodeHttpAddress());
containerReportStr.print("\tDiagnostics : ");
containerReportStr.print(containerReport.getDiagnosticsInfo());
} else {
@@ -535,7 +538,7 @@ public class ApplicationCLI extends YarnCLI {
.getContainers(ConverterUtils.toApplicationAttemptId(appAttemptId));
writer.println("Total number of containers " + ":" + appsReport.size());
writer.printf(CONTAINER_PATTERN, "Container-Id", "Start Time",
- "Finish Time", "State", "Host", "LOG-URL");
+ "Finish Time", "State", "Host", "Node Http Address", "LOG-URL");
for (ContainerReport containerReport : appsReport) {
writer.printf(
CONTAINER_PATTERN,
@@ -543,7 +546,9 @@ public class ApplicationCLI extends YarnCLI {
Times.format(containerReport.getCreationTime()),
Times.format(containerReport.getFinishTime()),
containerReport.getContainerState(), containerReport
- .getAssignedNode(), containerReport.getLogUrl());
+ .getAssignedNode(), containerReport.getNodeHttpAddress() == null
+ ? "N/A" : containerReport.getNodeHttpAddress(),
+ containerReport.getLogUrl());
}
writer.flush();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 4b2b4bc..ea9e361 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -77,7 +77,6 @@ import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenRequest;
import org.apache.hadoop.yarn.api.protocolrecords.RenewDelegationTokenResponse;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SubmitApplicationResponse;
-import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -715,7 +714,8 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public ContainerReport createFakeContainerReport() {
return ContainerReport.newInstance(createFakeContainerId(), null,
NodeId.newInstance("localhost", 0), null, 1000l, 1200l, "", "", 0,
- ContainerState.COMPLETE);
+ ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("localhost", 0).toString());
}
public List<ContainerReport> createFakeContainerReports() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
index a88189e..c3e3c41 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java
@@ -371,14 +371,16 @@ public class TestAHSClient {
ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1),
null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234,
- 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container);
ContainerReport container1 =
ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2),
null, NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234,
- 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ 5678, "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container1);
containers.put(attempt.getApplicationAttemptId(), containerReports);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
index 02f2882..3471343 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
@@ -501,13 +501,16 @@ public class TestYarnClient {
ContainerReport container = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 1), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
containerReports.add(container);
ContainerReport container1 = ContainerReport.newInstance(
ContainerId.newContainerId(attempt.getApplicationAttemptId(), 2), null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
+
containerReports.add(container1);
containers.put(attempt.getApplicationAttemptId(), containerReports);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index b4df551..62ed913 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -223,7 +223,8 @@ public class TestYarnCLI {
ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, 1234, 5678,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
when(client.getContainerReport(any(ContainerId.class))).thenReturn(
container);
int result = cli.run(new String[] { "container", "-status",
@@ -239,6 +240,7 @@ public class TestYarnCLI {
pw.println("\tState : COMPLETE");
pw.println("\tLOG-URL : logURL");
pw.println("\tHost : host:1234");
+ pw.println("\tNodeHttpAddress : http://host:2345");
pw.println("\tDiagnostics : diagnosticInfo");
pw.close();
String appReportStr = baos.toString("UTF-8");
@@ -258,19 +260,23 @@ public class TestYarnCLI {
long time1=1234,time2=5678;
ContainerReport container = ContainerReport.newInstance(containerId, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
ContainerReport container1 = ContainerReport.newInstance(containerId1, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1, time2,
- "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE);
+ "diagnosticInfo", "logURL", 0, ContainerState.COMPLETE,
+ "http://" + NodeId.newInstance("host", 2345).toString());
ContainerReport container2 = ContainerReport.newInstance(containerId2, null,
NodeId.newInstance("host", 1234), Priority.UNDEFINED, time1,0,
- "diagnosticInfo", "", 0, ContainerState.RUNNING);
+ "diagnosticInfo", "", 0, ContainerState.RUNNING,
+ "http://" + NodeId.newInstance("host", 2345).toString());
List<ContainerReport> reports = new ArrayList<ContainerReport>();
reports.add(container);
reports.add(container1);
reports.add(container2);
when(client.getContainers(any(ApplicationAttemptId.class))).thenReturn(
reports);
+ sysOutStream.reset();
int result = cli.run(new String[] { "container", "-list",
attemptId.toString() });
assertEquals(0, result);
@@ -283,24 +289,28 @@ public class TestYarnCLI {
pw.print("\t Finish Time");
pw.print("\t State");
pw.print("\t Host");
+ pw.print("\t Node Http Address");
pw.println("\t LOG-URL");
pw.print(" container_1234_0005_01_000001");
pw.print("\t"+Times.format(time1));
pw.print("\t"+Times.format(time2));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
+ pw.print("\t http://host:2345");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000002");
pw.print("\t"+Times.format(time1));
pw.print("\t"+Times.format(time2));
pw.print("\t COMPLETE");
pw.print("\t host:1234");
+ pw.print("\t http://host:2345");
pw.println("\t logURL");
pw.print(" container_1234_0005_01_000003");
pw.print("\t"+Times.format(time1));
pw.print("\t N/A");
pw.print("\t RUNNING");
pw.print("\t host:1234");
+ pw.print("\t http://host:2345");
pw.println("\t ");
pw.close();
String appReportStr = baos.toString("UTF-8");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
index 18c452f..1f0405f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
@@ -338,4 +338,23 @@ public class ContainerReportPBImpl extends ContainerReport {
ContainerStateProto containerState) {
return ProtoUtils.convertFromProtoFormat(containerState);
}
+
+ @Override
+ public String getNodeHttpAddress() {
+ ContainerReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNodeHttpAddress()) {
+ return null;
+ }
+ return (p.getNodeHttpAddress());
+ }
+
+ @Override
+ public void setNodeHttpAddress(String nodeHttpAddress) {
+ maybeInitBuilder();
+ if (nodeHttpAddress == null) {
+ builder.clearNodeHttpAddress();
+ return;
+ }
+ builder.setNodeHttpAddress(nodeHttpAddress);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
index 779b182..479858f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java
@@ -215,7 +215,7 @@ public class ApplicationHistoryManagerImpl extends AbstractService implements
containerHistory.getStartTime(), containerHistory.getFinishTime(),
containerHistory.getDiagnosticsInfo(), logUrl,
containerHistory.getContainerExitStatus(),
- containerHistory.getContainerState());
+ containerHistory.getContainerState(), null);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 56ad469..4a407c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -410,6 +410,7 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
String diagnosticsInfo = null;
int exitStatus = ContainerExitStatus.INVALID;
ContainerState state = null;
+ String nodeHttpAddress = null;
Map<String, Object> entityInfo = entity.getOtherInfo();
if (entityInfo != null) {
if (entityInfo
@@ -439,6 +440,12 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
allocatedPriority = (Integer) entityInfo.get(
ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO);
}
+ if (entityInfo.containsKey(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO)) {
+ nodeHttpAddress =
+ (String) entityInfo
+ .get(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO);
+ }
}
List<TimelineEvent> events = entity.getEvents();
if (events != null) {
@@ -488,7 +495,8 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
Resource.newInstance(allocatedMem, allocatedVcore),
NodeId.newInstance(allocatedHost, allocatedPort),
Priority.newInstance(allocatedPriority),
- createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state);
+ createdTime, finishedTime, diagnosticsInfo, logUrl, exitStatus, state,
+ nodeHttpAddress);
}
private ApplicationReportExt generateApplicationReport(TimelineEntity entity,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
index 8791da4..0d5540d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ContainerMetricsConstants.java
@@ -57,4 +57,6 @@ public class ContainerMetricsConstants {
public static final String STATE_EVENT_INFO =
"YARN_CONTAINER_STATE";
+ public static final String ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO =
+ "YARN_CONTAINER_ALLOCATED_HOST_HTTP_ADDRESS";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index ea33f4f..ae5f092 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -191,12 +191,14 @@ public class AppAttemptBlock extends HtmlBlock {
.append(url("container", container.getContainerId()))
.append("'>")
.append(container.getContainerId())
- .append("</a>\",\"<a href='")
- .append("#") // TODO: replace with node http address (YARN-1884)
+ .append("</a>\",\"<a ")
+ .append(
+ container.getNodeHttpAddress() == null ? "#" : "href='"
+ + container.getNodeHttpAddress())
.append("'>")
- .append(container.getAssignedNodeId() == null ? "N/A" :
+ .append(container.getNodeHttpAddress() == null ? "N/A" :
StringEscapeUtils.escapeJavaScript(StringEscapeUtils
- .escapeHtml(container.getAssignedNodeId())))
+ .escapeHtml(container.getNodeHttpAddress())))
.append("</a>\",\"")
.append(container.getContainerExitStatus()).append("\",\"<a href='")
.append(container.getLogUrl() == null ?
@@ -224,4 +226,4 @@ public class AppAttemptBlock extends HtmlBlock {
}
return false;
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index d52975b..e62c783 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -291,15 +291,12 @@ public class AppBlock extends HtmlBlock {
}
long startTime = 0L;
String logsLink = null;
+ String nodeLink = null;
if (containerReport != null) {
ContainerInfo container = new ContainerInfo(containerReport);
startTime = container.getStartedTime();
logsLink = containerReport.getLogUrl();
- }
- String nodeLink = null;
- if (appAttempt.getHost() != null && appAttempt.getRpcPort() >= 0
- && appAttempt.getRpcPort() < 65536) {
- nodeLink = appAttempt.getHost() + ":" + appAttempt.getRpcPort();
+ nodeLink = containerReport.getNodeHttpAddress();
}
// AppAttemptID numerical value parsed by parseHadoopID in
// yarn.dt.plugins.js
@@ -310,8 +307,8 @@ public class AppBlock extends HtmlBlock {
.append(appAttempt.getAppAttemptId())
.append("</a>\",\"")
.append(startTime)
- .append("\",\"<a href='")
- .append("#") // TODO: replace with node http address (YARN-1884)
+ .append("\",\"<a ")
+ .append(nodeLink == null ? "#" : "href='" + nodeLink)
.append("'>")
.append(nodeLink == null ? "N/A" : StringEscapeUtils
.escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
index ed50c7a..cae8d2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/ContainerBlock.java
@@ -104,7 +104,12 @@ public class ContainerBlock extends HtmlBlock {
container.getContainerState() == null ? UNAVAILABLE : container
.getContainerState())
._("Exit Status:", container.getContainerExitStatus())
- ._("Node:", container.getAssignedNodeId())
+ ._(
+ "Node:",
+ container.getNodeHttpAddress() == null ? "#" : container
+ .getNodeHttpAddress(),
+ container.getNodeHttpAddress() == null ? "N/A" : container
+ .getNodeHttpAddress())
._("Priority:", container.getPriority())
._("Started:", Times.format(container.getStartedTime()))
._(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
index bdcc7b2..0d18e7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/dao/ContainerInfo.java
@@ -42,6 +42,7 @@ public class ContainerInfo {
protected String logUrl;
protected int containerExitStatus;
protected ContainerState containerState;
+ protected String nodeHttpAddress;
public ContainerInfo() {
// JAXB needs this
@@ -64,6 +65,7 @@ public class ContainerInfo {
logUrl = container.getLogUrl();
containerExitStatus = container.getContainerExitStatus();
containerState = container.getContainerState();
+ nodeHttpAddress = container.getNodeHttpAddress();
}
public String getContainerId() {
@@ -114,4 +116,7 @@ public class ContainerInfo {
return containerState;
}
+ public String getNodeHttpAddress() {
+ return nodeHttpAddress;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
index eeda181..05b6781 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/ContainerCreatedEvent.java
@@ -29,18 +29,21 @@ public class ContainerCreatedEvent extends SystemMetricsEvent {
private Resource allocatedResource;
private NodeId allocatedNode;
private Priority allocatedPriority;
+ private String nodeHttpAddress;
public ContainerCreatedEvent(
ContainerId containerId,
Resource allocatedResource,
NodeId allocatedNode,
Priority allocatedPriority,
- long createdTime) {
+ long createdTime,
+ String nodeHttpAddress) {
super(SystemMetricsEventType.CONTAINER_CREATED, createdTime);
this.containerId = containerId;
this.allocatedResource = allocatedResource;
this.allocatedNode = allocatedNode;
this.allocatedPriority = allocatedPriority;
+ this.nodeHttpAddress = nodeHttpAddress;
}
@Override
@@ -64,4 +67,7 @@ public class ContainerCreatedEvent extends SystemMetricsEvent {
return allocatedPriority;
}
+ public String getNodeHttpAddress() {
+ return nodeHttpAddress;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index d8e7ef6..fb1edc62 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -179,7 +179,7 @@ public class SystemMetricsPublisher extends CompositeService {
container.getAllocatedResource(),
container.getAllocatedNode(),
container.getAllocatedPriority(),
- createdTime));
+ createdTime, container.getNodeHttpAddress()));
}
}
@@ -380,6 +380,9 @@ public class SystemMetricsPublisher extends CompositeService {
event.getAllocatedNode().getPort());
entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_ENTITY_INFO,
event.getAllocatedPriority().getPriority());
+ entityInfo.put(
+ ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_ENTITY_INFO,
+ event.getNodeHttpAddress());
entity.setOtherInfo(entityInfo);
TimelineEvent tEvent = new TimelineEvent();
tEvent.setEventType(ContainerMetricsConstants.CREATED_EVENT_TYPE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
index 9e9dcb9..20087f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
@@ -79,4 +79,5 @@ public interface RMContainer extends EventHandler<RMContainerEvent> {
List<ResourceRequest> getResourceRequests();
+ String getNodeHttpAddress();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index b69968f..7ee5467 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -42,7 +42,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppRunningOnNodeEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerAllocatedEvent;
@@ -608,10 +607,29 @@ public class RMContainerImpl implements RMContainer {
this.getAllocatedResource(), this.getAllocatedNode(),
this.getAllocatedPriority(), this.getCreationTime(),
this.getFinishTime(), this.getDiagnosticsInfo(), this.getLogURL(),
- this.getContainerExitStatus(), this.getContainerState());
+ this.getContainerExitStatus(), this.getContainerState(),
+ this.getNodeHttpAddress());
} finally {
this.readLock.unlock();
}
return containerReport;
}
+
+ @Override
+ public String getNodeHttpAddress() {
+ try {
+ readLock.lock();
+ if (container.getNodeHttpAddress() != null) {
+ StringBuilder httpAddress = new StringBuilder();
+ httpAddress.append(WebAppUtils.getHttpSchemePrefix(rmContext
+ .getYarnConfiguration()));
+ httpAddress.append(container.getNodeHttpAddress());
+ return httpAddress.toString();
+ } else {
+ return null;
+ }
+ } finally {
+ readLock.unlock();
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f4154bde/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
index 65c8547..2fa60dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/TestSystemMetricsPublisher.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer;
-import org.apache.hadoop.yarn.server.applicationhistoryservice.webapp.AHSWebApp;
import org.apache.hadoop.yarn.server.metrics.AppAttemptMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ApplicationMetricsConstants;
import org.apache.hadoop.yarn.server.metrics.ContainerMetricsConstants;
@@ -369,6 +368,10 @@ public class TestSystemMetricsPublisher {
when(container.getDiagnosticsInfo()).thenReturn("test diagnostics info");
when(container.getContainerExitStatus()).thenReturn(-1);
when(container.getContainerState()).thenReturn(ContainerState.COMPLETE);
+ Container mockContainer = mock(Container.class);
+ when(container.getContainer()).thenReturn(mockContainer);
+ when(mockContainer.getNodeHttpAddress())
+ .thenReturn("http://localhost:1234");
return container;
}
[36/50] [abbrv] hadoop git commit: YARN-3248. Display count of nodes
blacklisted by apps in the web UI. Contributed by Varun Vasudev
Posted by sj...@apache.org.
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
Contributed by Varun Vasudev
(cherry picked from commit 4728bdfa15809db4b8b235faa286c65de4a48cf6)
(cherry picked from commit e26b6e55e96b763063dfbd39977096367eafc1e3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7af5d6b4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7af5d6b4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7af5d6b4
Branch: refs/heads/branch-2.6
Commit: 7af5d6b4bae762e3c7fd0bb16f1f04d45e42ea25
Parents: e914220
Author: Xuan <xg...@apache.org>
Authored: Wed Apr 1 04:19:18 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:06 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/server/webapp/AppAttemptBlock.java | 66 +++++----
.../hadoop/yarn/server/webapp/AppBlock.java | 38 +++--
.../hadoop/yarn/server/webapp/AppsBlock.java | 95 +++++++-----
.../scheduler/AppSchedulingInfo.java | 4 +
.../scheduler/SchedulerApplicationAttempt.java | 4 +
.../webapp/AppsBlockWithMetrics.java | 2 +-
.../webapp/CapacitySchedulerPage.java | 2 +-
.../webapp/RMAppAttemptBlock.java | 68 +++++++++
.../resourcemanager/webapp/RMAppBlock.java | 110 ++++++++++++++
.../resourcemanager/webapp/RMAppsBlock.java | 146 +++++++++++++++++++
.../resourcemanager/webapp/RMWebServices.java | 3 +-
.../webapp/dao/AppAttemptInfo.java | 19 ++-
.../webapp/TestRMWebServicesApps.java | 2 +-
14 files changed, 476 insertions(+), 86 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e7d847f..c2c2bfa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -217,6 +217,9 @@ Release 2.6.1 - 2015-09-09
YARN-3379. Fixed missing data in localityTable and ResourceRequests table
in RM WebUI. (Xuan Gong via jianhe)
+ YARN-3248. Display count of nodes blacklisted by apps in the web UI.
+ (Varun Vasudev via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index abf7c86..50d4f72 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
import com.google.inject.Inject;
public class AppAttemptBlock extends HtmlBlock {
@@ -73,7 +74,7 @@ public class AppAttemptBlock extends HtmlBlock {
}
UserGroupInformation callerUGI = getCallerUGI();
- ApplicationAttemptReport appAttemptReport = null;
+ ApplicationAttemptReport appAttemptReport;
try {
final GetApplicationAttemptReportRequest request =
GetApplicationAttemptReportRequest.newInstance(appAttemptId);
@@ -136,33 +137,7 @@ public class AppAttemptBlock extends HtmlBlock {
&& appAttempt.getRpcPort() < 65536) {
node = appAttempt.getHost() + ":" + appAttempt.getRpcPort();
}
- info("Application Attempt Overview")
- ._(
- "Application Attempt State:",
- appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
- .getAppAttemptState())
- ._(
- "AM Container:",
- appAttempt.getAmContainerId() == null || containers == null
- || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
- ? null : root_url("container", appAttempt.getAmContainerId()),
- String.valueOf(appAttempt.getAmContainerId()))
- ._("Node:", node)
- ._(
- "Tracking URL:",
- appAttempt.getTrackingUrl() == null
- || appAttempt.getTrackingUrl() == UNAVAILABLE ? null
- : root_url(appAttempt.getTrackingUrl()),
- appAttempt.getTrackingUrl() == null
- || appAttempt.getTrackingUrl() == UNAVAILABLE
- ? "Unassigned"
- : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
- || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
- || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
- ? "History" : "ApplicationMaster")
- ._("Diagnostics Info:", appAttempt.getDiagnosticsInfo() == null ?
- "" : appAttempt.getDiagnosticsInfo());
-
+ generateOverview(appAttemptReport, containers, appAttempt, node);
html._(InfoBlock.class);
if (exceptionWhenGetContainerReports) {
@@ -216,7 +191,40 @@ public class AppAttemptBlock extends HtmlBlock {
tbody._()._();
}
- private boolean hasAMContainer(ContainerId containerId,
+ protected void generateOverview(ApplicationAttemptReport appAttemptReport,
+ Collection<ContainerReport> containers, AppAttemptInfo appAttempt,
+ String node) {
+ info("Application Attempt Overview")
+ ._(
+ "Application Attempt State:",
+ appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
+ .getAppAttemptState())
+ ._(
+ "AM Container:",
+ appAttempt.getAmContainerId() == null || containers == null
+ || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
+ ? null : root_url("container", appAttempt.getAmContainerId()),
+ String.valueOf(appAttempt.getAmContainerId()))
+ ._("Node:", node)
+ ._(
+ "Tracking URL:",
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null
+ : root_url(appAttempt.getTrackingUrl()),
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE)
+ ? "Unassigned"
+ : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
+ ? "History" : "ApplicationMaster")
+ ._(
+ "Diagnostics Info:",
+ appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt
+ .getDiagnosticsInfo());
+ }
+
+ protected boolean hasAMContainer(ContainerId containerId,
Collection<ContainerReport> containers) {
for (ContainerReport container : containers) {
if (containerId.equals(container.getContainerId())) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 93b24a3..c310f92 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
+
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -89,7 +90,7 @@ public class AppBlock extends HtmlBlock {
}
UserGroupInformation callerUGI = getCallerUGI();
- ApplicationReport appReport = null;
+ ApplicationReport appReport;
try {
final GetApplicationReportRequest request =
GetApplicationReportRequest.newInstance(appID);
@@ -161,7 +162,8 @@ public class AppBlock extends HtmlBlock {
._("Application Type:", app.getType())
._("Application Tags:",
app.getApplicationTags() == null ? "" : app.getApplicationTags())
- ._("YarnApplicationState:",
+ ._(
+ "YarnApplicationState:",
app.getAppState() == null ? UNAVAILABLE : clarifyAppState(app
.getAppState()))
._("FinalStatus Reported by AM:",
@@ -171,16 +173,19 @@ public class AppBlock extends HtmlBlock {
"Elapsed:",
StringUtils.formatTime(Times.elapsed(app.getStartedTime(),
app.getFinishedTime())))
- ._("Tracking URL:",
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? null : root_url(app.getTrackingUrl()),
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? "Unassigned" : app.getAppState() == YarnApplicationState.FINISHED
- || app.getAppState() == YarnApplicationState.FAILED
- || app.getAppState() == YarnApplicationState.KILLED ? "History"
- : "ApplicationMaster")
+ ._(
+ "Tracking URL:",
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : root_url(app
+ .getTrackingUrl()),
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
+ .getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED ? "History"
+ : "ApplicationMaster")
._("Diagnostics:",
- app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
+ app.getDiagnosticsInfo() == null ? "" : app.getDiagnosticsInfo());
Collection<ApplicationAttemptReport> attempts;
try {
@@ -211,6 +216,13 @@ public class AppBlock extends HtmlBlock {
html._(InfoBlock.class);
+ generateApplicationTable(html, callerUGI, attempts);
+
+ }
+
+ protected void generateApplicationTable(Block html,
+ UserGroupInformation callerUGI,
+ Collection<ApplicationAttemptReport> attempts) {
// Application Attempt Table
TBODY<TABLE<Hamlet>> tbody =
html.table("#attempts").thead().tr().th(".id", "Attempt ID")
@@ -220,7 +232,7 @@ public class AppBlock extends HtmlBlock {
StringBuilder attemptsTableData = new StringBuilder("[\n");
for (final ApplicationAttemptReport appAttemptReport : attempts) {
AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
- ContainerReport containerReport = null;
+ ContainerReport containerReport;
try {
// AM container is always the first container of the attempt
final GetContainerReportRequest request =
@@ -231,7 +243,7 @@ public class AppBlock extends HtmlBlock {
appBaseProt.getContainerReport(request).getContainerReport();
} else {
containerReport = callerUGI.doAs(
- new PrivilegedExceptionAction<ContainerReport> () {
+ new PrivilegedExceptionAction<ContainerReport>() {
@Override
public ContainerReport run() throws Exception {
ContainerReport report = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
index 583a0ae..6f8e10b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppsBlock.java
@@ -24,6 +24,7 @@ import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPS_NUM;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.EnumSet;
@@ -36,6 +37,7 @@ import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationsRequest;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
@@ -48,26 +50,19 @@ public class AppsBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppsBlock.class);
protected ApplicationBaseProtocol appBaseProt;
+ protected EnumSet<YarnApplicationState> reqAppStates;
+ protected UserGroupInformation callerUGI;
+ protected Collection<ApplicationReport> appReports;
@Inject
- AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
+ protected AppsBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
super(ctx);
this.appBaseProt = appBaseProt;
}
- @Override
- public void render(Block html) {
- setTitle("Applications");
-
- TBODY<TABLE<Hamlet>> tbody =
- html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
- .th(".name", "Name").th(".type", "Application Type")
- .th(".queue", "Queue").th(".starttime", "StartTime")
- .th(".finishtime", "FinishTime").th(".state", "State")
- .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
- .th(".ui", "Tracking UI")._()._().tbody();
- EnumSet<YarnApplicationState> reqAppStates =
- EnumSet.noneOf(YarnApplicationState.class);
+ protected void fetchData() throws YarnException, IOException,
+ InterruptedException {
+ reqAppStates = EnumSet.noneOf(YarnApplicationState.class);
String reqStateString = $(APP_STATE);
if (reqStateString != null && !reqStateString.isEmpty()) {
String[] appStateStrings = reqStateString.split(",");
@@ -76,33 +71,54 @@ public class AppsBlock extends HtmlBlock {
}
}
- UserGroupInformation callerUGI = getCallerUGI();
- Collection<ApplicationReport> appReports = null;
+ callerUGI = getCallerUGI();
+ final GetApplicationsRequest request =
+ GetApplicationsRequest.newInstance(reqAppStates);
+ String appsNumStr = $(APPS_NUM);
+ if (appsNumStr != null && !appsNumStr.isEmpty()) {
+ long appsNum = Long.parseLong(appsNumStr);
+ request.setLimit(appsNum);
+ }
+ if (callerUGI == null) {
+ appReports = appBaseProt.getApplications(request).getApplicationList();
+ } else {
+ appReports =
+ callerUGI
+ .doAs(new PrivilegedExceptionAction<Collection<ApplicationReport>>() {
+ @Override
+ public Collection<ApplicationReport> run() throws Exception {
+ return appBaseProt.getApplications(request)
+ .getApplicationList();
+ }
+ });
+ }
+ }
+
+ @Override
+ public void render(Block html) {
+ setTitle("Applications");
+
try {
- final GetApplicationsRequest request =
- GetApplicationsRequest.newInstance(reqAppStates);
- String appsNumStr = $(APPS_NUM);
- if (appsNumStr != null && !appsNumStr.isEmpty()) {
- long appsNum = Long.parseLong(appsNumStr);
- request.setLimit(appsNum);
- }
- if (callerUGI == null) {
- appReports = appBaseProt.getApplications(request).getApplicationList();
- } else {
- appReports = callerUGI.doAs(
- new PrivilegedExceptionAction<Collection<ApplicationReport>> () {
- @Override
- public Collection<ApplicationReport> run() throws Exception {
- return appBaseProt.getApplications(request).getApplicationList();
- }
- });
- }
- } catch (Exception e) {
+ fetchData();
+ }
+ catch( Exception e) {
String message = "Failed to read the applications.";
LOG.error(message, e);
html.p()._(message)._();
return;
}
+ renderData(html);
+ }
+
+ protected void renderData(Block html) {
+ TBODY<TABLE<Hamlet>> tbody =
+ html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
+ .th(".name", "Name").th(".type", "Application Type")
+ .th(".queue", "Queue").th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime").th(".state", "State")
+ .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+ .th(".ui", "Tracking UI")._()._().tbody();
+
StringBuilder appsTableData = new StringBuilder("[\n");
for (ApplicationReport appReport : appReports) {
// TODO: remove the following condition. It is still here because
@@ -123,7 +139,7 @@ public class AppsBlock extends HtmlBlock {
.append("</a>\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
- .getUser())))
+ .getUser())))
.append("\",\"")
.append(
StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
@@ -150,11 +166,12 @@ public class AppsBlock extends HtmlBlock {
.append("'> </div> </div>").append("\",\"<a ");
String trackingURL =
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
- ? null : app.getTrackingUrl();
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : app
+ .getTrackingUrl();
String trackingUI =
- app.getTrackingUrl() == null || app.getTrackingUrl() == UNAVAILABLE
+ app.getTrackingUrl() == null || app.getTrackingUrl().equals(UNAVAILABLE)
? "Unassigned"
: app.getAppState() == YarnApplicationState.FINISHED
|| app.getAppState() == YarnApplicationState.FAILED
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 331cee9..b8d552b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -437,6 +437,10 @@ public class AppSchedulingInfo {
return this.blacklist;
}
+ public synchronized Set<String> getBlackListCopy() {
+ return new HashSet<String>(this.blacklist);
+ }
+
public synchronized void transferStateFromPreviousAppSchedulingInfo(
AppSchedulingInfo appInfo) {
// this.priorities = appInfo.getPriorities();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 84975b6..510276c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -615,4 +615,8 @@ public class SchedulerApplicationAttempt {
// schedulingOpportunities
// lastScheduledContainer
}
+
+ public Set<String> getBlacklistedNodes() {
+ return this.appSchedulingInfo.getBlackListCopy();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
index cb0836a..29889ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppsBlockWithMetrics.java
@@ -27,6 +27,6 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
class AppsBlockWithMetrics extends HtmlBlock {
@Override public void render(Block html) {
html._(MetricsOverviewTable.class);
- html._(AppsBlock.class);
+ html._(RMAppsBlock.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index 0be361a..b1d0eba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -230,7 +230,7 @@ class CapacitySchedulerPage extends RmView {
ul._()._().
script().$type("text/javascript").
_("$('#cs').hide();")._()._().
- _(AppsBlock.class);
+ _(RMAppsBlock.class);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
index 7dc7030..c94d59a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -18,12 +18,23 @@
package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import com.google.inject.Inject;
@@ -53,4 +64,61 @@ public class RMAppAttemptBlock extends AppAttemptBlock{
}
return attempt;
}
+
+ protected void generateOverview(ApplicationAttemptReport appAttemptReport,
+ Collection<ContainerReport> containers, AppAttemptInfo appAttempt,
+ String node) {
+
+ String blacklistedNodes = "-";
+ Set<String> nodes =
+ getBlacklistedNodes(rm, getRMAppAttempt().getAppAttemptId());
+ if (nodes != null) {
+ if (!nodes.isEmpty()) {
+ blacklistedNodes = StringUtils.join(nodes, ", ");
+ }
+ }
+
+ info("Application Attempt Overview")
+ ._(
+ "Application Attempt State:",
+ appAttempt.getAppAttemptState() == null ? UNAVAILABLE : appAttempt
+ .getAppAttemptState())
+ ._(
+ "AM Container:",
+ appAttempt.getAmContainerId() == null || containers == null
+ || !hasAMContainer(appAttemptReport.getAMContainerId(), containers)
+ ? null : root_url("container", appAttempt.getAmContainerId()),
+ String.valueOf(appAttempt.getAmContainerId()))
+ ._("Node:", node)
+ ._(
+ "Tracking URL:",
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE) ? null
+ : root_url(appAttempt.getTrackingUrl()),
+ appAttempt.getTrackingUrl() == null
+ || appAttempt.getTrackingUrl().equals(UNAVAILABLE)
+ ? "Unassigned"
+ : appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FINISHED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.FAILED
+ || appAttempt.getAppAttemptState() == YarnApplicationAttemptState.KILLED
+ ? "History" : "ApplicationMaster")
+ ._(
+ "Diagnostics Info:",
+ appAttempt.getDiagnosticsInfo() == null ? "" : appAttempt
+ .getDiagnosticsInfo())._("Blacklisted Nodes:", blacklistedNodes);
+ }
+
+ public static Set<String> getBlacklistedNodes(ResourceManager rm,
+ ApplicationAttemptId appid) {
+ if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
+ AbstractYarnScheduler ayScheduler =
+ (AbstractYarnScheduler) rm.getResourceScheduler();
+ SchedulerApplicationAttempt attempt =
+ ayScheduler.getApplicationAttempt(appid);
+ if (attempt != null) {
+ return attempt.getBlacklistedNodes();
+ }
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 64c5747..171c6c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -20,13 +20,29 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+import java.security.PrivilegedExceptionAction;
+import java.util.Collection;
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
+import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
+import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
@@ -36,8 +52,10 @@ import com.google.inject.Inject;
public class RMAppBlock extends AppBlock{
+ private static final Log LOG = LogFactory.getLog(RMAppBlock.class);
private final ResourceManager rm;
+
@Inject
RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
super(rm.getClientRMService(), ctx, conf);
@@ -91,4 +109,96 @@ public class RMAppBlock extends AppBlock{
appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
pdiv._();
}
+
+ @Override
+ protected void generateApplicationTable(Block html,
+ UserGroupInformation callerUGI,
+ Collection<ApplicationAttemptReport> attempts) {
+ // Application Attempt Table
+ Hamlet.TBODY<Hamlet.TABLE<Hamlet>> tbody =
+ html.table("#attempts").thead().tr().th(".id", "Attempt ID")
+ .th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
+ .th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody();
+
+ StringBuilder attemptsTableData = new StringBuilder("[\n");
+ for (final ApplicationAttemptReport appAttemptReport : attempts) {
+ AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
+ ContainerReport containerReport = null;
+ try {
+ // AM container is always the first container of the attempt
+ final GetContainerReportRequest request =
+ GetContainerReportRequest.newInstance(ContainerId.newContainerId(
+ appAttemptReport.getApplicationAttemptId(), 1));
+ if (callerUGI == null) {
+ containerReport =
+ appBaseProt.getContainerReport(request).getContainerReport();
+ } else {
+ containerReport = callerUGI.doAs(
+ new PrivilegedExceptionAction<ContainerReport>() {
+ @Override
+ public ContainerReport run() throws Exception {
+ ContainerReport report = null;
+ try {
+ report = appBaseProt.getContainerReport(request)
+ .getContainerReport();
+ } catch (ContainerNotFoundException ex) {
+ LOG.warn(ex.getMessage());
+ }
+ return report;
+ }
+ });
+ }
+ } catch (Exception e) {
+ String message =
+ "Failed to read the AM container of the application attempt "
+ + appAttemptReport.getApplicationAttemptId() + ".";
+ LOG.error(message, e);
+ html.p()._(message)._();
+ return;
+ }
+ long startTime = 0L;
+ String logsLink = null;
+ String nodeLink = null;
+ if (containerReport != null) {
+ ContainerInfo container = new ContainerInfo(containerReport);
+ startTime = container.getStartedTime();
+ logsLink = containerReport.getLogUrl();
+ nodeLink = containerReport.getNodeHttpAddress();
+ }
+ String blacklistedNodesCount = "N/A";
+ Set<String> nodes = RMAppAttemptBlock.getBlacklistedNodes(rm,
+ ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId()));
+ if(nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
+
+ // AppAttemptID numerical value parsed by parseHadoopID in
+ // yarn.dt.plugins.js
+ attemptsTableData
+ .append("[\"<a href='")
+ .append(url("appattempt", appAttempt.getAppAttemptId()))
+ .append("'>")
+ .append(appAttempt.getAppAttemptId())
+ .append("</a>\",\"")
+ .append(startTime)
+ .append("\",\"<a ")
+ .append(nodeLink == null ? "#" : "href='" + nodeLink)
+ .append("'>")
+ .append(nodeLink == null ? "N/A" : StringEscapeUtils
+ .escapeJavaScript(StringEscapeUtils.escapeHtml(nodeLink)))
+ .append("</a>\",\"<a ")
+ .append(logsLink == null ? "#" : "href='" + logsLink).append("'>")
+ .append(logsLink == null ? "N/A" : "Logs").append("</a>\",").append(
+ "\"").append(blacklistedNodesCount).append("\"],\n");
+ }
+ if (attemptsTableData.charAt(attemptsTableData.length() - 2) == ',') {
+ attemptsTableData.delete(attemptsTableData.length() - 2,
+ attemptsTableData.length() - 1);
+ }
+ attemptsTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var attemptsTableData=" + attemptsTableData)._();
+
+ tbody._()._();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
new file mode 100644
index 0000000..94f3afa
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppsBlock.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.util.StringHelper.join;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR;
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI.C_PROGRESSBAR_VALUE;
+
+import java.util.Set;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.hadoop.yarn.api.ApplicationBaseProtocol;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.webapp.AppsBlock;
+import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.webapp.View;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
+
+import com.google.inject.Inject;
+
+public class RMAppsBlock extends AppsBlock {
+
+ private ResourceManager rm;
+
+ @Inject
+ RMAppsBlock(ResourceManager rm, ApplicationBaseProtocol appBaseProt,
+ View.ViewContext ctx) {
+ super(appBaseProt, ctx);
+ this.rm = rm;
+ }
+
+ @Override
+ protected void renderData(Block html) {
+ TBODY<TABLE<Hamlet>> tbody =
+ html.table("#apps").thead().tr().th(".id", "ID").th(".user", "User")
+ .th(".name", "Name").th(".type", "Application Type")
+ .th(".queue", "Queue").th(".starttime", "StartTime")
+ .th(".finishtime", "FinishTime").th(".state", "State")
+ .th(".finalstatus", "FinalStatus").th(".progress", "Progress")
+ .th(".ui", "Tracking UI").th(".blacklisted", "Blacklisted Nodes")._()
+ ._().tbody();
+
+ StringBuilder appsTableData = new StringBuilder("[\n");
+ for (ApplicationReport appReport : appReports) {
+ // TODO: remove the following condition. It is still here because
+ // the history side implementation of ApplicationBaseProtocol
+ // hasn't filtering capability (YARN-1819).
+ if (!reqAppStates.isEmpty()
+ && !reqAppStates.contains(appReport.getYarnApplicationState())) {
+ continue;
+ }
+
+ AppInfo app = new AppInfo(appReport);
+ String blacklistedNodesCount = "N/A";
+ Set<String> nodes =
+ RMAppAttemptBlock
+ .getBlacklistedNodes(rm, ConverterUtils.toApplicationAttemptId(app
+ .getCurrentAppAttemptId()));
+ if (nodes != null) {
+ blacklistedNodesCount = String.valueOf(nodes.size());
+ }
+ String percent = String.format("%.1f", app.getProgress());
+ // AppID numerical value parsed by parseHadoopID in yarn.dt.plugins.js
+ appsTableData
+ .append("[\"<a href='")
+ .append(url("app", app.getAppId()))
+ .append("'>")
+ .append(app.getAppId())
+ .append("</a>\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getUser())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getName())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getType())))
+ .append("\",\"")
+ .append(
+ StringEscapeUtils.escapeJavaScript(StringEscapeUtils.escapeHtml(app
+ .getQueue()))).append("\",\"").append(app.getStartedTime())
+ .append("\",\"").append(app.getFinishedTime())
+ .append("\",\"")
+ .append(app.getAppState() == null ? UNAVAILABLE : app.getAppState())
+ .append("\",\"")
+ .append(app.getFinalAppStatus())
+ .append("\",\"")
+ // Progress bar
+ .append("<br title='").append(percent).append("'> <div class='")
+ .append(C_PROGRESSBAR).append("' title='").append(join(percent, '%'))
+ .append("'> ").append("<div class='").append(C_PROGRESSBAR_VALUE)
+ .append("' style='").append(join("width:", percent, '%'))
+ .append("'> </div> </div>").append("\",\"<a ");
+
+ String trackingURL =
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? null : app
+ .getTrackingUrl();
+
+ String trackingUI =
+ app.getTrackingUrl() == null
+ || app.getTrackingUrl().equals(UNAVAILABLE) ? "Unassigned" : app
+ .getAppState() == YarnApplicationState.FINISHED
+ || app.getAppState() == YarnApplicationState.FAILED
+ || app.getAppState() == YarnApplicationState.KILLED ? "History"
+ : "ApplicationMaster";
+ appsTableData.append(trackingURL == null ? "#" : "href='" + trackingURL)
+ .append("'>").append(trackingUI).append("</a>\",").append("\"")
+ .append(blacklistedNodesCount).append("\"],\n");
+
+ }
+ if (appsTableData.charAt(appsTableData.length() - 2) == ',') {
+ appsTableData.delete(appsTableData.length() - 2,
+ appsTableData.length() - 1);
+ }
+ appsTableData.append("]");
+ html.script().$type("text/javascript")
+ ._("var appsTableData=" + appsTableData)._();
+
+ tbody._()._();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index 27417a9..7746922 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -631,7 +631,8 @@ public class RMWebServices {
AppAttemptsInfo appAttemptsInfo = new AppAttemptsInfo();
for (RMAppAttempt attempt : app.getAppAttempts().values()) {
- AppAttemptInfo attemptInfo = new AppAttemptInfo(attempt, app.getUser());
+ AppAttemptInfo attemptInfo =
+ new AppAttemptInfo(rm, attempt, app.getUser());
appAttemptsInfo.add(attemptInfo);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
index 0bb9cf7..2c7bbd0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/AppAttemptInfo.java
@@ -21,8 +21,13 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMAppAttemptBlock;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
@@ -36,16 +41,18 @@ public class AppAttemptInfo {
protected String nodeHttpAddress;
protected String nodeId;
protected String logsLink;
+ protected String blacklistedNodes;
public AppAttemptInfo() {
}
- public AppAttemptInfo(RMAppAttempt attempt, String user) {
+ public AppAttemptInfo(ResourceManager rm, RMAppAttempt attempt, String user) {
this.startTime = 0;
this.containerId = "";
this.nodeHttpAddress = "";
this.nodeId = "";
this.logsLink = "";
+ this.blacklistedNodes = "";
if (attempt != null) {
this.id = attempt.getAppAttemptId().getAttemptId();
this.startTime = attempt.getStartTime();
@@ -57,6 +64,16 @@ public class AppAttemptInfo {
this.logsLink =
WebAppUtils.getRunningLogURL("//" + masterContainer.getNodeHttpAddress(),
ConverterUtils.toString(masterContainer.getId()), user);
+ if (rm.getResourceScheduler() instanceof AbstractYarnScheduler) {
+ AbstractYarnScheduler ayScheduler =
+ (AbstractYarnScheduler) rm.getResourceScheduler();
+ SchedulerApplicationAttempt sattempt =
+ ayScheduler.getApplicationAttempt(attempt.getAppAttemptId());
+ if (sattempt != null) {
+ blacklistedNodes =
+ StringUtils.join(sattempt.getBlacklistedNodes(), ", ");
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7af5d6b4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index 23ea22e..3258d7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1599,7 +1599,7 @@ public class TestRMWebServicesApps extends JerseyTest {
String user)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 6, info.length());
+ assertEquals("incorrect number of elements", 7, info.length());
verifyAppAttemptInfoGeneric(appAttempt, info.getInt("id"),
info.getLong("startTime"), info.getString("containerId"),
[42/50] [abbrv] hadoop git commit: YARN-3248. Moving it to
improvements section in CHANGES.txt.
Posted by sj...@apache.org.
YARN-3248. Moving it to improvements section in CHANGES.txt.
(cherry picked from commit df52ff92ae42977bc743630191d1cbe1e17d00cf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e8c4544d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e8c4544d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e8c4544d
Branch: refs/heads/branch-2.6
Commit: e8c4544d8ede7196cabeec4b696e2dfa68a68874
Parents: 4697d94
Author: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Authored: Wed Sep 23 15:01:29 2015 -0700
Committer: Vinod Kumar Vavilapalli (I am also known as @tshooter.) <vi...@apache.org>
Committed: Wed Sep 23 15:04:39 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e8c4544d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c0f630c..ae4f089 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -27,6 +27,9 @@ Release 2.6.1 - 2015-09-16
YARN-3978. Configurably turn off the saving of container info in Generic AHS
(Eric Payne via jeagles)
+ YARN-3248. Display count of nodes blacklisted by apps in the web UI.
+ (Varun Vasudev via xgong)
+
OPTIMIZATIONS
BUG FIXES
@@ -211,9 +214,6 @@ Release 2.6.1 - 2015-09-16
YARN-3740. Fixed the typo in the configuration name:
APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
- YARN-3248. Display count of nodes blacklisted by apps in the web UI.
- (Varun Vasudev via xgong)
-
YARN-3544. Got back AM logs link on the RM web UI for a completed app.
(Xuan Gong via zjshen)
[29/50] [abbrv] hadoop git commit: Preparing for release 2.6.1:
Posting release notes generated.
Posted by sj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c86d9bd1/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
index acd0c56..29da383 100644
--- a/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
+++ b/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
@@ -1,5 +1,5 @@
<META http-equiv="Content-Type" content="text/html; charset=UTF-8">
-<title>Hadoop 2.6.0 Release Notes</title>
+<title>Hadoop 2.6.1 Release Notes</title>
<STYLE type="text/css">
H1 {font-family: sans-serif}
H2 {font-family: sans-serif; margin-left: 7mm}
@@ -7,29168 +7,632 @@
</STYLE>
</head>
<body>
-<h1>Hadoop 2.6.0 Release Notes</h1>
+<h1>Hadoop 2.6.1 Release Notes</h1>
These release notes include new developer and user-facing incompatibilities, features, and major improvements.
<a name="changes"/>
-<h2>Changes since Hadoop 2.5.1</h2>
+<h2>Changes since Hadoop 2.6.0</h2>
<ul>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2853">YARN-2853</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Killing app may hang while AM is unregistering</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2846">YARN-2846</a>.
- Blocker bug reported by Junping Du and fixed by Junping Du (nodemanager)<br>
- <b>Incorrect persist exit code for running containers in reacquireContainer() that interrupted by NodeManager restart.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2843">YARN-2843</a>.
- Major sub-task reported by Sushmitha Sreenivasan and fixed by Wangda Tan (resourcemanager)<br>
- <b>NodeLabels manager should trim all inputs for hosts and labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2841">YARN-2841</a>.
- Critical sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>RMProxy should retry EOFException </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2834">YARN-2834</a>.
- Blocker bug reported by Yesha Vora and fixed by Jian He <br>
- <b>Resource manager crashed with Null Pointer Exception</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2830">YARN-2830</a>.
- Blocker bug reported by Jonathan Eagles and fixed by Jonathan Eagles <br>
- <b>Add backwords compatible ContainerId.newInstance constructor for use within Tez Local Mode</b><br>
- <blockquote>I just committed this. Thanks [~jeagles] for the patch and [~ozawa] for the reviews!</blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2827">YARN-2827</a>.
- Critical bug reported by Wangda Tan and fixed by Wangda Tan (client)<br>
- <b>Fix bugs of yarn queue CLI</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2826">YARN-2826</a>.
- Critical bug reported by Sidharta Seethana and fixed by Wangda Tan <br>
- <b>User-Group mappings not updated by RM when a user is removed from a group.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2825">YARN-2825</a>.
- Critical bug reported by Jian He and fixed by Jian He <br>
- <b>Container leak on NM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2824">YARN-2824</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Capacity of labels should be zero by default</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2823">YARN-2823</a>.
- Critical bug reported by Gour Saha and fixed by Jian He (resourcemanager)<br>
- <b>NullPointerException in RM HA enabled 3-node cluster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2819">YARN-2819</a>.
- Critical bug reported by Gopal V and fixed by Zhijie Shen (timelineserver)<br>
- <b>NPE in ATS Timeline Domains when upgrading from 2.4 to 2.6</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2818">YARN-2818</a>.
- Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Remove the logic to inject entity owner as the primary filter</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2813">YARN-2813</a>.
- Major bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>NPE from MemoryTimelineStore.getDomains</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2812">YARN-2812</a>.
- Major test reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>TestApplicationHistoryServer is likely to fail on less powerful machine</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2810">YARN-2810</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev (resourcemanager)<br>
- <b>TestRMProxyUsersConf fails on Windows VMs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2805">YARN-2805</a>.
- Blocker bug reported by Arpit Gupta and fixed by Wangda Tan (resourcemanager)<br>
- <b>RM2 in HA setup tries to login using the RM1's kerberos principal</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2804">YARN-2804</a>.
- Critical bug reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Timeline server .out log have JAXB binding exceptions and warnings.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2803">YARN-2803</a>.
- Critical bug reported by Chris Nauroth and fixed by Craig Welch (nodemanager)<br>
- <b>MR distributed cache not working correctly on Windows after NodeManager privileged account changes.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2798">YARN-2798</a>.
- Blocker bug reported by Arpit Gupta and fixed by Zhijie Shen (timelineserver)<br>
- <b>YarnClient doesn't need to translate Kerberos name of timeline DT renewer</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2795">YARN-2795</a>.
- Major sub-task reported by Phil D'Amore and fixed by Wangda Tan (resourcemanager)<br>
- <b>Resource Manager fails startup with HDFS label storage and secure cluster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2794">YARN-2794</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Fix log msgs about distributing system-credentials </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2790">YARN-2790</a>.
- Critical bug reported by Tassapol Athiapinya and fixed by Jian He (nodemanager)<br>
- <b>NM can't aggregate logs past HDFS delegation token expiry.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2789">YARN-2789</a>.
- Critical task reported by Siddharth Seth and fixed by Wangda Tan <br>
- <b>Re-instate the NodeReport.newInstance API modified in YARN-2698</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2788">YARN-2788</a>.
- Blocker bug reported by Gopal V and fixed by Xuan Gong (log-aggregation)<br>
- <b>yarn logs -applicationId on 2.6.0 should support logs written by 2.4.0</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2785">YARN-2785</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>TestContainerResourceUsage fails intermittently</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2779">YARN-2779</a>.
- Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
- <b>SystemMetricsPublisher can use Kerberos directly instead of timeline DT</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2778">YARN-2778</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (client)<br>
- <b>YARN node CLI should display labels on returned node reports</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2770">YARN-2770</a>.
- Critical sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Timeline delegation tokens need to be automatically renewed by the RM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2769">YARN-2769</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev (applications/distributed-shell)<br>
- <b>Timeline server domain not set correctly when using shell_command on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2767">YARN-2767</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev (resourcemanager)<br>
- <b>RM web services - add test case to ensure the http static user cannot kill or submit apps in secure mode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2760">YARN-2760</a>.
- Trivial bug reported by Harsh J and fixed by Harsh J (documentation)<br>
- <b>Completely remove word 'experimental' from FairScheduler docs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2758">YARN-2758</a>.
- Major test reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Update TestApplicationHistoryClientService to use the new generic history store</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2755">YARN-2755</a>.
- Critical bug reported by Siqi Li and fixed by Siqi Li <br>
- <b>NM fails to clean up usercache_DEL_<timestamp> dirs after YARN-661</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2753">YARN-2753</a>.
- Major sub-task reported by zhihai xu and fixed by zhihai xu <br>
- <b>Fix potential issues and code clean up for *NodeLabelsManager</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2752">YARN-2752</a>.
- Critical bug reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>ContainerExecutor always append "nice -n" in command on branch-2</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2747">YARN-2747</a>.
- Major test reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>TestAggregatedLogFormat fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2744">YARN-2744</a>.
- Critical sub-task reported by Sumit Mohanty and fixed by Wangda Tan (capacityscheduler)<br>
- <b>Under some scenario, it is possible to end up with capacity scheduler configuration that uses labels that no longer exist</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2743">YARN-2743</a>.
- Blocker bug reported by Arpit Gupta and fixed by Jian He (resourcemanager)<br>
- <b>Yarn jobs via oozie fail with failed to renew token (secure) or digest mismatch (unsecure) errors when RM is being killed</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2741">YARN-2741</a>.
- Major bug reported by Craig Welch and fixed by Craig Welch (nodemanager)<br>
- <b>Windows: Node manager cannot serve up log files via the web user interface when yarn.nodemanager.log-dirs to any drive letter other than C: (or, the drive that nodemanager is running on)</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2734">YARN-2734</a>.
- Major bug reported by Sumit Mohanty and fixed by Xuan Gong (log-aggregation)<br>
- <b>If a sub-folder is encountered by log aggregator it results in invalid aggregated file</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2732">YARN-2732</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Fix syntax error in SecureContainer.apt.vm</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2730">YARN-2730</a>.
- Critical bug reported by Siqi Li and fixed by Siqi Li <br>
- <b>DefaultContainerExecutor runs only one localizer at a time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2726">YARN-2726</a>.
- Minor sub-task reported by Phil D'Amore and fixed by Wangda Tan (capacityscheduler)<br>
- <b>CapacityScheduler should explicitly log when an accessible label has no capacity</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2724">YARN-2724</a>.
- Major bug reported by Sumit Mohanty and fixed by Xuan Gong (log-aggregation)<br>
- <b>If an unreadable file is encountered during log aggregation then aggregated file in HDFS badly formed</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2723">YARN-2723</a>.
- Major sub-task reported by Phil D'Amore and fixed by Naganarasimha G R (client)<br>
- <b>rmadmin -replaceLabelsOnNode does not correctly parse port</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2721">YARN-2721</a>.
- Blocker bug reported by Jian He and fixed by Jian He <br>
- <b>Race condition: ZKRMStateStore retry logic may throw NodeExist exception </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2720">YARN-2720</a>.
- Major bug reported by Craig Welch and fixed by Craig Welch (nodemanager)<br>
- <b>Windows: Wildcard classpath variables not expanded against resources contained in archives</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2717">YARN-2717</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (log-aggregation)<br>
- <b>containerLogNotFound log shows multiple time for the same container</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2715">YARN-2715</a>.
- Blocker bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager)<br>
- <b>Proxy user is problem for RPC interface if yarn.resourcemanager.webapp.proxyuser is not set.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2711">YARN-2711</a>.
- Major test reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>TestDefaultContainerExecutor#testContainerLaunchError fails on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2709">YARN-2709</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Add retry for timeline client getDelegationToken method</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2707">YARN-2707</a>.
- Minor bug reported by Ted Yu and fixed by Gera Shegalov <br>
- <b>Potential null dereference in FSDownload</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2705">YARN-2705</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Changes of RM node label manager default configuration</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2704">YARN-2704</a>.
- Critical sub-task reported by Jian He and fixed by Jian He <br>
- <b> Localization and log-aggregation will fail if hdfs delegation token expired after token-max-life-time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2703">YARN-2703</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Add logUploadedTime into LogValue for better display</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2701">YARN-2701</a>.
- Blocker bug reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>Potential race condition in startLocalizer when using LinuxContainerExecutor </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2700">YARN-2700</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>TestSecureRMRegistryOperations failing on windows: auth problems</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2699">YARN-2699</a>.
- Blocker sub-task reported by Wangda Tan and fixed by Wangda Tan (client)<br>
- <b>Fix test timeout in TestResourceTrackerOnHA#testResourceTrackerOnHA</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2698">YARN-2698</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Move getClusterNodeLabels and getNodeToLabels to YarnClient instead of AdminService</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2692">YARN-2692</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran <br>
- <b>ktutil test hanging on some machines/ktutil versions</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2689">YARN-2689</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>TestSecureRMRegistryOperations failing on windows: secure ZK won't start</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2685">YARN-2685</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Resource on each label not correct when multiple NMs in a same host and some has label some not</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2682">YARN-2682</a>.
- Minor bug reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
- <b>WindowsSecureContainerExecutor should not depend on DefaultContainerExecutor#getFirstApplicationDir. </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2678">YARN-2678</a>.
- Major sub-task reported by Gour Saha and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>Improved Yarn Registry service record structure</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2677">YARN-2677</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api , resourcemanager)<br>
- <b>registry punycoding of usernames doesn't fix all usernames to be DNS-valid</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2676">YARN-2676</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Timeline authentication filter should add support for proxy user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2673">YARN-2673</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Add retry for timeline client put APIs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2671">YARN-2671</a>.
- Blocker bug reported by Zhijie Shen and fixed by Wangda Tan (resourcemanager)<br>
- <b>ApplicationSubmissionContext change breaks the existing app submission</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2668">YARN-2668</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (client)<br>
- <b>yarn-registry JAR won't link against ZK 3.4.5</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2667">YARN-2667</a>.
- Minor bug reported by Yi Liu and fixed by Yi Liu <br>
- <b>Fix the release audit warning caused by hadoop-yarn-registry</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2662">YARN-2662</a>.
- Minor bug reported by Chris Nauroth and fixed by Chris Nauroth (test)<br>
- <b>TestCgroupsLCEResourcesHandler leaks file descriptors.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2656">YARN-2656</a>.
- Major bug reported by Varun Vasudev and fixed by Zhijie Shen (resourcemanager)<br>
- <b>RM web services authentication filter should add support for proxy user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2652">YARN-2652</a>.
- Major sub-task reported by Steve Loughran and fixed by Steve Loughran (api)<br>
- <b>add hadoop-yarn-registry package under hadoop-yarn</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2651">YARN-2651</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Spin off the LogRollingInterval from LogAggregationContext</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2649">YARN-2649</a>.
- Major bug reported by Ming Ma and fixed by Ming Ma <br>
- <b>Flaky test TestAMRMRPCNodeUpdates</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2647">YARN-2647</a>.
- Major sub-task reported by Wangda Tan and fixed by Sunil G (client)<br>
- <b>Add yarn queue CLI to get queue infos</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2644">YARN-2644</a>.
- Major sub-task reported by Craig Welch and fixed by Craig Welch <br>
- <b>Recalculate headroom more frequently to keep it accurate</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2635">YARN-2635</a>.
- Major bug reported by Wei Yan and fixed by Wei Yan <br>
- <b>TestRM, TestRMRestart, TestClientToAMTokens should run with both CS and FS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2632">YARN-2632</a>.
- Blocker sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
- <b>Document NM Restart feature</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-4047">YARN-4047</a>.
+ Major bug reported by Jason Lowe and fixed by Jason Lowe (resourcemanager)<br>
+ <b>ClientRMService getApplications has high scheduler lock contention</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2630">YARN-2630</a>.
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3999">YARN-3999</a>.
Major bug reported by Jian He and fixed by Jian He <br>
- <b>TestDistributedShell#testDSRestartWithPreviousRunningContainers fails</b><br>
+ <b>RM hangs on draining events</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2629">YARN-2629</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Make distributed shell use the domain-based timeline ACLs</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3990">YARN-3990</a>.
+ Critical bug reported by Rohith Sharma K S and fixed by Bibin A Chundatt (resourcemanager)<br>
+ <b>AsyncDispatcher may overloaded with RMAppNodeUpdateEvent when Node is connected/disconnected</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2628">YARN-2628</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev (capacityscheduler)<br>
- <b>Capacity scheduler with DominantResourceCalculator carries out reservation even though slots are free</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3978">YARN-3978</a>.
+ Major improvement reported by Eric Payne and fixed by Eric Payne (timelineserver , yarn)<br>
+ <b>Configurably turn off the saving of container info in Generic AHS</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2627">YARN-2627</a>.
- Major improvement reported by Xuan Gong and fixed by Xuan Gong <br>
- <b>Add logs when attemptFailuresValidityInterval is enabled</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3850">YARN-3850</a>.
+ Blocker bug reported by Varun Saxena and fixed by Varun Saxena (log-aggregation , nodemanager)<br>
+ <b>NM fails to read files from full disks which can lead to container logs being lost and other issues</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2624">YARN-2624</a>.
- Blocker bug reported by Anubhav Dhoot and fixed by Anubhav Dhoot (nodemanager)<br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3832">YARN-3832</a>.
+ Critical bug reported by Ranga Swamy and fixed by Brahma Reddy Battula (nodemanager)<br>
<b>Resource Localization fails on a cluster due to existing cache directories</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2621">YARN-2621</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Simplify the output when the user doesn't have the access for getDomain(s) </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2617">YARN-2617</a>.
- Major bug reported by Jun Gong and fixed by Jun Gong (nodemanager)<br>
- <b>NM does not need to send finished container whose APP is not running to RM</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2615">YARN-2615</a>.
- Blocker sub-task reported by Junping Du and fixed by Junping Du <br>
- <b>ClientToAMTokenIdentifier and DelegationTokenIdentifier should allow extended fields</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2611">YARN-2611</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , resourcemanager , scheduler)<br>
- <b>Fix jenkins findbugs warning and test case failures for trunk merge patch</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2610">YARN-2610</a>.
- Major bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Hamlet should close table tags</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2608">YARN-2608</a>.
- Major bug reported by Wei Yan and fixed by Wei Yan <br>
- <b>FairScheduler: Potential deadlocks in loading alloc files and clock access</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2607">YARN-2607</a>.
- Major test reported by Ted Yu and fixed by Wangda Tan <br>
- <b>TestDistributedShell fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2606">YARN-2606</a>.
- Major bug reported by Mit Desai and fixed by Mit Desai (timelineserver)<br>
- <b>Application History Server tries to access hdfs before doing secure login</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2602">YARN-2602</a>.
- Major bug reported by Karam Singh and fixed by Zhijie Shen (timelineserver)<br>
- <b>Generic History Service of TimelineServer sometimes not able to handle NPE</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2596">YARN-2596</a>.
- Major test reported by Junping Du and fixed by Karthik Kambatla <br>
- <b>TestWorkPreservingRMRestart fails with FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2594">YARN-2594</a>.
- Blocker bug reported by Karam Singh and fixed by Wangda Tan (resourcemanager)<br>
- <b>Potential deadlock in RM when querying ApplicationResourceUsageReport</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2591">YARN-2591</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>AHSWebServices should return FORBIDDEN(403) if the request user doesn't have access to the history data</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2588">YARN-2588</a>.
- Major bug reported by Rohith and fixed by Rohith (resourcemanager)<br>
- <b>Standby RM does not transitionToActive if previous transitionToActive is failed with ZK exception.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2584">YARN-2584</a>.
- Major test reported by Zhijie Shen and fixed by Jian He <br>
- <b>TestContainerManagerSecurity fails on trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2583">YARN-2583</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Modify the LogDeletionService to support Log aggregation for LRS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2582">YARN-2582</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>Log related CLI and Web UI changes for Aggregated Logs in LRS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2581">YARN-2581</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>NMs need to find a way to get LogAggregationContext</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2579">YARN-2579</a>.
- Blocker bug reported by Rohith and fixed by Rohith (resourcemanager)<br>
- <b>Deadlock when EmbeddedElectorService and FatalEventDispatcher try to transition RM to StandBy at the same time</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2577">YARN-2577</a>.
- Trivial improvement reported by Miklos Christine and fixed by Miklos Christine (documentation , fairscheduler)<br>
- <b>Clarify ACL delimiter and how to configure ACL groups only</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2576">YARN-2576</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , resourcemanager , scheduler)<br>
- <b>Prepare yarn-1051 branch for merging with trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2569">YARN-2569</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (nodemanager , resourcemanager)<br>
- <b>API changes for handling logs of long-running services</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2568">YARN-2568</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>TestAMRMClientOnRMRestart test fails</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2566">YARN-2566</a>.
- Critical sub-task reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
- <b>DefaultContainerExecutor should pick a working directory randomly</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2565">YARN-2565</a>.
- Major bug reported by Karam Singh and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
- <b>RM shouldn't use the old RMApplicationHistoryWriter unless explicitly setting FileSystemApplicationHistoryStore</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2563">YARN-2563</a>.
- Blocker bug reported by Arpit Gupta and fixed by Zhijie Shen (timelineserver)<br>
- <b>On secure clusters call to timeline server fails with authentication errors when running a job via oozie</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2562">YARN-2562</a>.
- Critical bug reported by Vinod Kumar Vavilapalli and fixed by Tsuyoshi OZAWA <br>
- <b>ContainerId@toString() is unreadable for epoch >0 after YARN-2182</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2561">YARN-2561</a>.
- Blocker sub-task reported by Tassapol Athiapinya and fixed by Junping Du <br>
- <b>MR job client cannot reconnect to AM after NM restart.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2559">YARN-2559</a>.
- Major bug reported by Karam Singh and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
- <b>ResourceManager sometime become un-responsive due to NPE in SystemMetricsPublisher</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2558">YARN-2558</a>.
- Blocker sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>Updating ContainerTokenIdentifier#read/write to use ContainerId#getContainerId</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2557">YARN-2557</a>.
- Major bug reported by Xuan Gong and fixed by Xuan Gong (applications/distributed-shell)<br>
- <b>Add a parameter "attempt_Failures_Validity_Interval" in DistributedShell </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2549">YARN-2549</a>.
- Minor test reported by Chris Nauroth and fixed by Chris Nauroth (nodemanager , test)<br>
- <b>TestContainerLaunch fails due to classpath problem with hamcrest classes.</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2547">YARN-2547</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
- <b>Cross Origin Filter throws UnsupportedOperationException upon destroy</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2546">YARN-2546</a>.
- Major bug reported by Doug Haigh and fixed by Varun Vasudev (api)<br>
- <b>REST API for application creation/submission is using strings for numeric & boolean values</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2544">YARN-2544</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
- <b>Common server side PB changes (not include user API PB changes)</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2542">YARN-2542</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>"yarn application -status <appId>" throws NPE when retrieving the app from the timelineserver</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2541">YARN-2541</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>Fix ResourceManagerRest.apt.vm syntax error</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2540">YARN-2540</a>.
- Major bug reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
- <b>FairScheduler: Queue filters not working on scheduler page in RM UI</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2539">YARN-2539</a>.
- Minor improvement reported by Wei Yan and fixed by Wei Yan <br>
- <b>FairScheduler: Set the default value for maxAMShare to 0.5</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3733">YARN-3733</a>.
+ Blocker bug reported by Bibin A Chundatt and fixed by Rohith Sharma K S (resourcemanager)<br>
+ <b>Fix DominantRC#compare() does not work as expected if cluster resource is empty</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2538">YARN-2538</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>Add logs when RM send new AMRMToken to ApplicationMaster</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3725">YARN-3725</a>.
+ Blocker bug reported by Zhijie Shen and fixed by Zhijie Shen (resourcemanager , timelineserver)<br>
+ <b>App submission via REST API is broken in secure mode due to Timeline DT service address is empty</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2534">YARN-2534</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu (scheduler)<br>
- <b>FairScheduler: Potential integer overflow calculating totalMaxShare</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3700">YARN-3700</a>.
+ Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager , webapp , yarn)<br>
+ <b>ATS Web Performance issue at load time when large number of jobs</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2531">YARN-2531</a>.
- Major improvement reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>CGroups - Admins should be allowed to enforce strict cpu limits</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3585">YARN-3585</a>.
+ Critical bug reported by Peng Zhang and fixed by Rohith Sharma K S <br>
+ <b>NodeManager cannot exit on SHUTDOWN event triggered and NM recovery is enabled</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2529">YARN-2529</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Generic history service RPC interface doesn't work when service authorization is enabled</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3526">YARN-3526</a>.
+ Major bug reported by Weiwei Yang and fixed by Weiwei Yang (resourcemanager , webapp)<br>
+ <b>ApplicationMaster tracking URL is incorrectly redirected on a QJM cluster</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2528">YARN-2528</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Cross Origin Filter Http response split vulnerability protection rejects valid origins</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3493">YARN-3493</a>.
+ Critical bug reported by Sumana Sathish and fixed by Jian He (yarn)<br>
+ <b>RM fails to come up with error "Failed to load/recover state" when mem settings are changed</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2527">YARN-2527</a>.
- Major bug reported by Benoy Antony and fixed by Benoy Antony (resourcemanager)<br>
- <b>NPE in ApplicationACLsManager</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3487">YARN-3487</a>.
+ Critical sub-task reported by Jason Lowe and fixed by Jason Lowe (capacityscheduler)<br>
+ <b>CapacityScheduler scheduler lock obtained unnecessarily when calling getQueue</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2526">YARN-2526</a>.
- Critical bug reported by Wei Yan and fixed by Wei Yan (scheduler-load-simulator)<br>
- <b>SLS can deadlock when all the threads are taken by AMSimulators</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3464">YARN-3464</a>.
+ Critical bug reported by zhihai xu and fixed by zhihai xu (nodemanager)<br>
+ <b>Race condition in LocalizerRunner kills localizer before localizing all resources</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2523">YARN-2523</a>.
- Major bug reported by Nishan Shetty and fixed by Rohith (resourcemanager , webapp)<br>
- <b>ResourceManager UI showing negative value for "Decommissioned Nodes" field</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2519">YARN-2519</a>.
- Major test reported by Xiaoyu Yao and fixed by Xiaoyu Yao (webapp)<br>
- <b>Credential Provider related unit tests failed on Windows</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2515">YARN-2515</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Update ConverterUtils#toContainerId to parse epoch</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2512">YARN-2512</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Allow for origin pattern matching in cross origin filter</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2511">YARN-2511</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Allow All Origins by default when Cross Origin Filter is enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2509">YARN-2509</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
- <b>Enable Cross Origin Filter for timeline server only and not all Yarn servers</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2508">YARN-2508</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
- <b>Cross Origin configuration parameters prefix are not honored</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2507">YARN-2507</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (documentation , timelineserver)<br>
- <b>Document Cross Origin Filter Configuration for ATS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2505">YARN-2505</a>.
- Major sub-task reported by Wangda Tan and fixed by Craig Welch (resourcemanager)<br>
- <b>Support get/add/remove/change labels in RM REST API</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2504">YARN-2504</a>.
- Critical sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Support get/add/remove/change labels in RM admin CLI </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2503">YARN-2503</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes in RM Web UI to better show labels to end users</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2502">YARN-2502</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes in distributed shell to support specify labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2501">YARN-2501</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes in AMRMClient to support labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2500">YARN-2500</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Miscellaneous changes in ResourceManager to support labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2496">YARN-2496</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Changes for capacity scheduler to support allocate resource respect labels</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2494">YARN-2494</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (resourcemanager)<br>
- <b>Node label manager API and storage implementations</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2493">YARN-2493</a>.
- Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api)<br>
- <b>API changes for users</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2484">YARN-2484</a>.
- Trivial bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>FileSystemRMStateStore#readFile/writeFile should close FSData(In|Out)putStream in final block</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2475">YARN-2475</a>.
- Major sub-task reported by Carlo Curino and fixed by Carlo Curino (resourcemanager)<br>
- <b>ReservationSystem: replan upon capacity reduction</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2468">YARN-2468</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (log-aggregation , nodemanager , resourcemanager)<br>
- <b>Log handling for LRS</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2462">YARN-2462</a>.
- Major bug reported by Jason Lowe and fixed by Eric Payne <br>
- <b>TestNodeManagerResync#testBlockNewContainerRequestsOnStartAndResync should have a test timeout</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2460">YARN-2460</a>.
- Minor bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Remove obsolete entries from yarn-default.xml</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2459">YARN-2459</a>.
- Major bug reported by Mayank Bansal and fixed by Mayank Bansal (resourcemanager)<br>
- <b>RM crashes if App gets rejected for any reason and HA is enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2456">YARN-2456</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Possible livelock in CapacityScheduler when RM is recovering apps</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2453">YARN-2453</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu <br>
- <b>TestProportionalCapacityPreemptionPolicy fails with FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2452">YARN-2452</a>.
- Major bug reported by zhihai xu and fixed by zhihai xu <br>
- <b>TestRMApplicationHistoryWriter fails with FairScheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2450">YARN-2450</a>.
- Trivial bug reported by Ray Chiang and fixed by Ray Chiang <br>
- <b>Fix typos in log messages</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2449">YARN-2449</a>.
- Critical bug reported by Karam Singh and fixed by Varun Vasudev (timelineserver)<br>
- <b>Timelineserver returns invalid Delegation token in secure kerberos enabled cluster when hadoop.http.filter.initializers are not set</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2448">YARN-2448</a>.
- Major improvement reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>RM should expose the resource types considered during scheduling when AMs register</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2447">YARN-2447</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>RM web services app submission doesn't pass secrets correctly</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2446">YARN-2446</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Using TimelineNamespace to shield the entities of a user</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2440">YARN-2440</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>Cgroups should allow YARN containers to be limited to allocated cores</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2434">YARN-2434</a>.
- Major sub-task reported by Jian He and fixed by Jian He <br>
- <b>RM should not recover containers from previously failed attempt when AM restart is not enabled</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2431">YARN-2431</a>.
- Major sub-task reported by Jason Lowe and fixed by Jason Lowe (nodemanager)<br>
- <b>NM restart: cgroup is not removed for reacquired containers</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2424">YARN-2424</a>.
- Blocker bug reported by Allen Wittenauer and fixed by Allen Wittenauer (nodemanager)<br>
- <b>LCE should support non-cgroups, non-secure mode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2411">YARN-2411</a>.
- Major improvement reported by Ram Venkatesh and fixed by Ram Venkatesh (capacityscheduler)<br>
- <b>[Capacity Scheduler] support simple user and group mappings to queues</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2409">YARN-2409</a>.
- Critical bug reported by Nishan Shetty and fixed by Rohith (resourcemanager)<br>
- <b>Active to StandBy transition does not stop rmDispatcher that causes 1 AsyncDispatcher thread leak. </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2406">YARN-2406</a>.
- Major sub-task reported by Jian He and fixed by Tsuyoshi OZAWA <br>
- <b>Move RM recovery related proto to yarn_server_resourcemanager_recovery.proto</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2405">YARN-2405</a>.
- Major bug reported by Maysam Yabandeh and fixed by Tsuyoshi OZAWA <br>
- <b>NPE in FairSchedulerAppsBlock</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2400">YARN-2400</a>.
- Major bug reported by Jian He and fixed by Jian He <br>
- <b>TestAMRestart fails intermittently</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2399">YARN-2399</a>.
- Major improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (fairscheduler)<br>
- <b>FairScheduler: Merge AppSchedulable and FSSchedulerApp into FSAppAttempt</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2397">YARN-2397</a>.
- Critical bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>RM and TS web interfaces sometimes return request is a replay error in secure mode</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2395">YARN-2395</a>.
- Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
- <b>FairScheduler: Preemption timeout should be configurable per queue</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2394">YARN-2394</a>.
- Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
- <b>FairScheduler: Configure fairSharePreemptionThreshold per queue</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2393">YARN-2393</a>.
- Major new feature reported by Ashwin Shankar and fixed by Wei Yan (fairscheduler)<br>
- <b>FairScheduler: Add the notion of steady fair share</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2389">YARN-2389</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler , fairscheduler)<br>
- <b>Adding support for drainig a queue, ie killing all apps in the queue</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2388">YARN-2388</a>.
- Major test reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>TestTimelineWebServices fails on trunk after HADOOP-10791</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2387">YARN-2387</a>.
- Blocker bug reported by Mit Desai and fixed by Mit Desai <br>
- <b>Resource Manager crashes with NPE due to lack of synchronization</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2378">YARN-2378</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (capacityscheduler)<br>
- <b>Adding support for moving apps between queues in Capacity Scheduler</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2377">YARN-2377</a>.
- Major improvement reported by Gera Shegalov and fixed by Gera Shegalov (nodemanager)<br>
- <b>Localization exception stack traces are not passed as diagnostic info</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2374">YARN-2374</a>.
- Major bug reported by Varun Vasudev and fixed by Varun Vasudev <br>
- <b>YARN trunk build failing TestDistributedShell.testDSShell</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2373">YARN-2373</a>.
- Major bug reported by Larry McCay and fixed by Larry McCay <br>
- <b>WebAppUtils Should Use configuration.getPassword for Accessing SSL Passwords</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3393">YARN-3393</a>.
+ Critical bug reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
+ <b>Getting application(s) goes wrong when app finishes before starting the attempt</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2372">YARN-2372</a>.
- Minor improvement reported by Fengdong Yu and fixed by Fengdong Yu (documentation)<br>
- <b>There are Chinese Characters in the FairScheduler's document</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3369">YARN-3369</a>.
+ Blocker bug reported by Giovanni Matteo Fumarola and fixed by Brahma Reddy Battula (resourcemanager)<br>
+ <b>Missing NullPointer check in AppSchedulingInfo causes RM to die </b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2370">YARN-2370</a>.
- Trivial bug reported by Wenwu Peng and fixed by Wenwu Peng (resourcemanager)<br>
- <b>Fix comment in o.a.h.y.server.resourcemanager.schedulerAppSchedulingInfo</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3287">YARN-3287</a>.
+ Major bug reported by Jonathan Eagles and fixed by Daryn Sharp <br>
+ <b>TimelineClient kerberos authentication failure uses wrong login context.</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2363">YARN-2363</a>.
- Major bug reported by Jason Lowe and fixed by Jason Lowe (resourcemanager)<br>
- <b>Submitted applications occasionally lack a tracking URL</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3267">YARN-3267</a>.
+ Major bug reported by Prakash Ramachandran and fixed by Chang Li <br>
+ <b>Timelineserver applies the ACL rules after applying the limit on the number of records</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2361">YARN-2361</a>.
- Trivial improvement reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
- <b>RMAppAttempt state machine entries for KILLED state has duplicate event entries</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3251">YARN-3251</a>.
+ Blocker bug reported by Jason Lowe and fixed by Craig Welch <br>
+ <b>Fix CapacityScheduler deadlock when computing absolute max avail capacity (short term fix for 2.6.1)</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2360">YARN-2360</a>.
- Major new feature reported by Ashwin Shankar and fixed by Ashwin Shankar (fairscheduler)<br>
- <b>Fair Scheduler: Display dynamic fair share for queues on the scheduler page</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3249">YARN-3249</a>.
+ Minor improvement reported by Ryu Kobayashi and fixed by Ryu Kobayashi (resourcemanager)<br>
+ <b>Add a "kill application" button to Resource Manager's Web UI</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2359">YARN-2359</a>.
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3242">YARN-3242</a>.
Critical bug reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
- <b>Application hangs when it fails to launch AM container </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2354">YARN-2354</a>.
- Major sub-task reported by Jian He and fixed by Li Lu <br>
- <b>DistributedShell may allocate more containers than client specified after it restarts</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2352">YARN-2352</a>.
- Major improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (scheduler)<br>
- <b>FairScheduler: Collect metrics on duration of critical methods that affect performance</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2347">YARN-2347</a>.
- Major sub-task reported by Junping Du and fixed by Junping Du <br>
- <b>Consolidate RMStateVersion and NMDBSchemaVersion into StateVersion in yarn-server-common</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2343">YARN-2343</a>.
- Trivial improvement reported by Li Lu and fixed by Li Lu <br>
- <b>Improve error message on token expire exception</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2337">YARN-2337</a>.
- Trivial improvement reported by zhihai xu and fixed by zhihai xu (resourcemanager)<br>
- <b>ResourceManager sets ClientRMService in RMContext multiple times</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2323">YARN-2323</a>.
- Minor improvement reported by Hong Zhiguo and fixed by Hong Zhiguo (fairscheduler)<br>
- <b>FairShareComparator creates too many Resource objects</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2321">YARN-2321</a>.
- Major bug reported by Leitao Guo and fixed by Leitao Guo (nodemanager)<br>
- <b>NodeManager web UI can incorrectly report Pmem enforcement</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2317">YARN-2317</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu (documentation)<br>
- <b>Update documentation about how to write YARN applications</b><br>
+ <b>Asynchrony in ZK-close can lead to ZKRMStateStore watcher receiving events for old client</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2314">YARN-2314</a>.
- Critical bug reported by Jason Lowe and fixed by Jason Lowe (client)<br>
- <b>ContainerManagementProtocolProxy can create thousands of threads for a large cluster</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2313">YARN-2313</a>.
- Major bug reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (fairscheduler)<br>
- <b>Livelock can occur in FairScheduler when there are lots of running apps</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2312">YARN-2312</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Marking ContainerId#getId as deprecated</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2308">YARN-2308</a>.
- Critical bug reported by Wangda Tan and fixed by chang li (resourcemanager , scheduler)<br>
- <b>NPE happened when RM restart after CapacityScheduler queue configuration changed </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2302">YARN-2302</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Refactor TimelineWebServices</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2298">YARN-2298</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (client)<br>
- <b>Move TimelineClient to yarn-common project</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2295">YARN-2295</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Refactor YARN distributed shell with existing public stable API</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2288">YARN-2288</a>.
- Major sub-task reported by Junping Du and fixed by Junping Du (timelineserver)<br>
- <b>Data persistent in timelinestore should be versioned</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2279">YARN-2279</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen (timelineserver)<br>
- <b>Add UTs to cover timeline server authentication</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2277">YARN-2277</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles (timelineserver)<br>
- <b>Add Cross-Origin support to the ATS REST API</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2274">YARN-2274</a>.
- Trivial improvement reported by Karthik Kambatla and fixed by Karthik Kambatla (fairscheduler)<br>
- <b>FairScheduler: Add debug information about cluster capacity, availability and reservations</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2273">YARN-2273</a>.
- Major bug reported by Andy Skelton and fixed by Wei Yan (fairscheduler , resourcemanager)<br>
- <b>NPE in ContinuousScheduling thread when we lose a node</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2269">YARN-2269</a>.
- Major bug reported by Yesha Vora and fixed by Craig Welch <br>
- <b>External links need to be removed from YARN UI</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2264">YARN-2264</a>.
- Major bug reported by Siddharth Seth and fixed by Li Lu <br>
- <b>Race in DrainDispatcher can cause random test failures</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2260">YARN-2260</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Add containers to launchedContainers list in RMNode on container recovery</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2252">YARN-2252</a>.
- Major bug reported by Ratandeep Ratti and fixed by (scheduler)<br>
- <b>Intermittent failure of TestFairScheduler.testContinuousScheduling</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2251">YARN-2251</a>.
- Major bug reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Avoid negative elapsed time in JHS/MRAM web UI and services</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2249">YARN-2249</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>AM release request may be lost on RM restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2248">YARN-2248</a>.
- Major sub-task reported by Janos Matyas and fixed by Janos Matyas (capacityscheduler)<br>
- <b>Capacity Scheduler changes for moving apps between queues</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2244">YARN-2244</a>.
- Critical bug reported by Anubhav Dhoot and fixed by Anubhav Dhoot (fairscheduler)<br>
- <b>FairScheduler missing handling of containers for unknown application attempts </b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2242">YARN-2242</a>.
- Major sub-task reported by Li Lu and fixed by Li Lu <br>
- <b>Improve exception information on AM launch crashes</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2237">YARN-2237</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>MRAppMaster changes for AMRMToken roll-up</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2229">YARN-2229</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>ContainerId can overflow with RM restart</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2228">YARN-2228</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>TimelineServer should load pseudo authentication filter when authentication = simple</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2219">YARN-2219</a>.
- Major bug reported by Ashwin Shankar and fixed by Jian He (resourcemanager)<br>
- <b>AMs and NMs can get exceptions after recovery but before scheduler knowns apps and app-attempts</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2214">YARN-2214</a>.
- Major improvement reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
- <b>FairScheduler: preemptContainerPreCheck() in FSParentQueue delays convergence towards fairness</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3239">YARN-3239</a>.
+ Major bug reported by Hitesh Shah and fixed by Jian He <br>
+ <b>WebAppProxy does not support a final tracking url which has query fragments and params </b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2212">YARN-2212</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>ApplicationMaster needs to find a way to update the AMRMToken periodically</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3238">YARN-3238</a>.
+ Blocker bug reported by Jason Lowe and fixed by Jason Lowe <br>
+ <b>Connection timeouts to nodemanagers are retried at multiple levels</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2211">YARN-2211</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>RMStateStore needs to save AMRMToken master key for recovery when RM restart/failover happens </b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3231">YARN-3231</a>.
+ Critical bug reported by Siqi Li and fixed by Siqi Li <br>
+ <b>FairScheduler: Changing queueMaxRunningApps interferes with pending jobs</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2209">YARN-2209</a>.
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3230">YARN-3230</a>.
Major improvement reported by Jian He and fixed by Jian He <br>
- <b>Replace AM resync/shutdown command with corresponding exceptions</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2208">YARN-2208</a>.
- Major sub-task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>AMRMTokenManager need to have a way to roll over AMRMToken</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2207">YARN-2207</a>.
- Major task reported by Xuan Gong and fixed by Xuan Gong (resourcemanager)<br>
- <b>Add ability to roll over AMRMToken</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2198">YARN-2198</a>.
- Major improvement reported by Remus Rusanu and fixed by Remus Rusanu <br>
- <b>Remove the need to run NodeManager as privileged account for Windows Secure Container Executor</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2197">YARN-2197</a>.
- Minor improvement reported by Akira AJISAKA and fixed by Akira AJISAKA (documentation)<br>
- <b>Add a link to YARN CHANGES.txt in the left side of doc</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2182">YARN-2182</a>.
- Major sub-task reported by Tsuyoshi OZAWA and fixed by Tsuyoshi OZAWA <br>
- <b>Update ContainerId#toString() to avoid conflicts before and after RM restart</b><br>
+ <b>Clarify application states on the web UI</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2181">YARN-2181</a>.
- Major bug reported by Wangda Tan and fixed by Wangda Tan (resourcemanager , webapp)<br>
- <b>Add preemption info to RM Web UI and add logs when preemption occurs</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3227">YARN-3227</a>.
+ Critical bug reported by Jonathan Eagles and fixed by Zhijie Shen <br>
+ <b>Timeline renew delegation token fails when RM user's TGT is expired</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2174">YARN-2174</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>Enabling HTTPs for the writer REST API of TimelineServer</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3222">YARN-3222</a>.
+ Critical bug reported by Rohith Sharma K S and fixed by Rohith Sharma K S (resourcemanager)<br>
+ <b>RMNodeImpl#ReconnectNodeTransition should send scheduler events in sequential order</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2161">YARN-2161</a>.
- Major bug reported by Binglin Chang and fixed by Binglin Chang <br>
- <b>Fix build on macosx: YARN parts</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3207">YARN-3207</a>.
+ Major bug reported by Prakash Ramachandran and fixed by Zhijie Shen (timelineserver)<br>
+ <b>secondary filter matches entites which do not have the key being filtered for.</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2158">YARN-2158</a>.
- Minor test reported by Ted Yu and fixed by Varun Vasudev <br>
- <b>TestRMWebServicesAppsModification sometimes fails in trunk</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2153">YARN-2153</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Ensure distributed shell work with RM work-preserving recovery</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2147">YARN-2147</a>.
- Minor bug reported by Jason Lowe and fixed by Chen He (resourcemanager)<br>
- <b>client lacks delegation token exception details when application submit fails</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2138">YARN-2138</a>.
- Major bug reported by Jian He and fixed by Varun Saxena <br>
- <b>Cleanup notifyDone* methods in RMStateStore</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2131">YARN-2131</a>.
- Major new feature reported by Karthik Kambatla and fixed by Robert Kanter (resourcemanager)<br>
- <b>Add a way to format the RMStateStore</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2102">YARN-2102</a>.
- Major sub-task reported by Zhijie Shen and fixed by Zhijie Shen <br>
- <b>More generalized timeline ACLs</b><br>
- <blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2088">YARN-2088</a>.
- Major bug reported by Binglin Chang and fixed by Binglin Chang <br>
- <b>Fix code bug in GetApplicationsRequestPBImpl#mergeLocalToBuilder</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3124">YARN-3124</a>.
+ Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
+ <b>Capacity Scheduler LeafQueue/ParentQueue should use QueueCapacities to track capacities-by-label</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2080">YARN-2080</a>.
- Major sub-task reported by Subru Krishnan and fixed by Subru Krishnan (resourcemanager)<br>
- <b>Admission Control: Integrate Reservation subsystem with ResourceManager</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3103">YARN-3103</a>.
+ Blocker bug reported by Jason Lowe and fixed by Jason Lowe (client)<br>
+ <b>AMRMClientImpl does not update AMRM token properly</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2070">YARN-2070</a>.
- Minor sub-task reported by Zhijie Shen and fixed by Robert Kanter <br>
- <b>DistributedShell publishes unfriendly user information to the timeline server</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3099">YARN-3099</a>.
+ Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
+ <b>Capacity Scheduler LeafQueue/ParentQueue should use ResourceUsage to track used-resources-by-label.</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2045">YARN-2045</a>.
- Major sub-task reported by Junping Du and fixed by Junping Du (nodemanager)<br>
- <b>Data persisted in NM should be versioned</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3098">YARN-3098</a>.
+ Major sub-task reported by Wangda Tan and fixed by Wangda Tan (capacityscheduler)<br>
+ <b>Create common QueueCapacities class in Capacity Scheduler to track capacities-by-labels of queues</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2035">YARN-2035</a>.
- Major sub-task reported by Jonathan Eagles and fixed by Jonathan Eagles <br>
- <b>FileSystemApplicationHistoryStore blocks RM and AHS while NN is in safemode</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3094">YARN-3094</a>.
+ Major bug reported by Jun Gong and fixed by Jun Gong (resourcemanager)<br>
+ <b>reset timer for liveness monitors after RM recovery</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2034">YARN-2034</a>.
- Minor bug reported by Jason Lowe and fixed by Chen He (nodemanager)<br>
- <b>Description for yarn.nodemanager.localizer.cache.target-size-mb is incorrect</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3092">YARN-3092</a>.
+ Major sub-task reported by Wangda Tan and fixed by Wangda Tan (api , client , resourcemanager)<br>
+ <b>Create common ResourceUsage class to track labeled resource usages in Capacity Scheduler</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2033">YARN-2033</a>.
- Major sub-task reported by Vinod Kumar Vavilapalli and fixed by Zhijie Shen <br>
- <b>Merging generic-history into the Timeline Store</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3055">YARN-3055</a>.
+ Blocker bug reported by Yi Liu and fixed by Daryn Sharp (security)<br>
+ <b>The token is not renewed properly if it's shared by jobs (oozie) in DelegationTokenRenewer</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2026">YARN-2026</a>.
- Major bug reported by Ashwin Shankar and fixed by Ashwin Shankar (scheduler)<br>
- <b>Fair scheduler: Consider only active queues for computing fairshare</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3024">YARN-3024</a>.
+ Major bug reported by Chengbing Liu and fixed by Chengbing Liu (nodemanager)<br>
+ <b>LocalizerRunner should give DIE action when all resources are localized</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2013">YARN-2013</a>.
- Major sub-task reported by Zhijie Shen and fixed by Tsuyoshi OZAWA (nodemanager)<br>
- <b>The diagnostics is always the ExitCodeException stack when the container crashes</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-3011">YARN-3011</a>.
+ Major sub-task reported by Wang Hao and fixed by Varun Saxena (nodemanager)<br>
+ <b>NM dies because of the failure of resource localization</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2010">YARN-2010</a>.
- Blocker bug reported by bc Wong and fixed by Karthik Kambatla (resourcemanager)<br>
- <b>Handle app-recovery failures gracefully</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2997">YARN-2997</a>.
+ Major bug reported by Chengbing Liu and fixed by Chengbing Liu (nodemanager)<br>
+ <b>NM keeps sending already-sent completed containers to RM until containers are removed from context</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2008">YARN-2008</a>.
- Major sub-task reported by Chen He and fixed by Craig Welch <br>
- <b>CapacityScheduler may report incorrect queueMaxCap if there is hierarchy queue structure </b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2992">YARN-2992</a>.
+ Blocker bug reported by Karthik Kambatla and fixed by Karthik Kambatla (resourcemanager)<br>
+ <b>ZKRMStateStore crashes due to session expiry</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-2001">YARN-2001</a>.
- Major sub-task reported by Jian He and fixed by Jian He (resourcemanager)<br>
- <b>Threshold for RM to accept requests from AM after failover</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2978">YARN-2978</a>.
+ Critical bug reported by Jason Tufo and fixed by Varun Saxena <br>
+ <b>ResourceManager crashes with NPE while getting queue info</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1994">YARN-1994</a>.
- Major improvement reported by Arpit Agarwal and fixed by Craig Welch (nodemanager , resourcemanager , webapp)<br>
- <b>Expose YARN/MR endpoints on multiple interfaces</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2964">YARN-2964</a>.
+ Blocker bug reported by Daryn Sharp and fixed by Jian He (resourcemanager)<br>
+ <b>RM prematurely cancels tokens for jobs that submit jobs (oozie)</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1972">YARN-1972</a>.
- Major sub-task reported by Remus Rusanu and fixed by Remus Rusanu (nodemanager)<br>
- <b>Implement secure Windows Container Executor</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2952">YARN-2952</a>.
+ Major bug reported by Jian He and fixed by Rohith Sharma K S <br>
+ <b>Incorrect version check in RMStateStore</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1964">YARN-1964</a>.
- Major new feature reported by Arun C Murthy and fixed by Abin Shahab <br>
- <b>Create Docker analog of the LinuxContainerExecutor in YARN</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2922">YARN-2922</a>.
+ Major bug reported by Jason Tufo and fixed by Rohith Sharma K S (capacityscheduler , resourcemanager , scheduler)<br>
+ <b>ConcurrentModificationException in CapacityScheduler's LeafQueue</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1959">YARN-1959</a>.
- Major bug reported by Sandy Ryza and fixed by Anubhav Dhoot <br>
- <b>Fix headroom calculation in FairScheduler</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2920">YARN-2920</a>.
+ Major sub-task reported by Wangda Tan and fixed by Wangda Tan <br>
+ <b>CapacityScheduler should be notified when labels on nodes changed</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1954">YARN-1954</a>.
- Major improvement reported by Zhijie Shen and fixed by Tsuyoshi OZAWA (client)<br>
- <b>Add waitFor to AMRMClient(Async)</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2918">YARN-2918</a>.
+ Major sub-task reported by Rohith Sharma K S and fixed by Wangda Tan (resourcemanager)<br>
+ <b>Don't fail RM if queue's configured labels are not existed in cluster-node-labels</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1922">YARN-1922</a>.
- Major bug reported by Billie Rinaldi and fixed by Billie Rinaldi (nodemanager)<br>
- <b>Process group remains alive after container process is killed externally</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2917">YARN-2917</a>.
+ Critical bug reported by Rohith Sharma K S and fixed by Rohith Sharma K S (resourcemanager)<br>
+ <b>Potential deadlock in AsyncDispatcher when system.exit called in AsyncDispatcher#dispatch and AsyscDispatcher#serviceStop from shutdown hook</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1919">YARN-1919</a>.
- Minor bug reported by Devaraj K and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Potential NPE in EmbeddedElectorService#stop</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2910">YARN-2910</a>.
+ Major bug reported by Wilfred Spiegelenburg and fixed by Wilfred Spiegelenburg (fairscheduler)<br>
+ <b>FSLeafQueue can throw ConcurrentModificationException</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1918">YARN-1918</a>.
- Trivial improvement reported by Devaraj K and fixed by Anandha L Ranganathan <br>
- <b>Typo in description and error message for 'yarn.resourcemanager.cluster-id'</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2906">YARN-2906</a>.
+ Major bug reported by Jason Lowe and fixed by Jason Lowe (capacityscheduler)<br>
+ <b>CapacitySchedulerPage shows HTML tags for a queue's Active Users</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1915">YARN-1915</a>.
- Blocker sub-task reported by Hitesh Shah and fixed by Jason Lowe <br>
- <b>ClientToAMTokenMasterKey should be provided to AM at launch time</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2905">YARN-2905</a>.
+ Blocker bug reported by Jason Lowe and fixed by Varun Saxena <br>
+ <b>AggregatedLogsBlock page can infinitely loop if the aggregated log file is corrupted</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1879">YARN-1879</a>.
- Critical sub-task reported by Jian He and fixed by Tsuyoshi OZAWA (resourcemanager)<br>
- <b>Mark Idempotent/AtMostOnce annotations to ApplicationMasterProtocol for RM fail over</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2900">YARN-2900</a>.
+ Major sub-task reported by Jonathan Eagles and fixed by Mit Desai (timelineserver)<br>
+ <b>Application (Attempt and Container) Not Found in AHS results in Internal Server Error (500)</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1857">YARN-1857</a>.
- Critical sub-task reported by Thomas Graves and fixed by Chen He (capacityscheduler)<br>
- <b>CapacityScheduler headroom doesn't account for other AM's running</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2894">YARN-2894</a>.
+ Major bug reported by Rohith Sharma K S and fixed by Rohith Sharma K S (resourcemanager)<br>
+ <b>When ACL's are enabled, if RM switches then application can not be viewed from web.</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1796">YARN-1796</a>.
- Minor bug reported by Aaron T. Myers and fixed by Aaron T. Myers (nodemanager)<br>
- <b>container-executor shouldn't require o-r permissions</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2890">YARN-2890</a>.
+ Major bug reported by Mit Desai and fixed by Mit Desai <br>
+ <b>MiniYarnCluster should turn on timeline service if configured to do so</b><br>
<blockquote></blockquote></li>
-<li> <a href="https://issues.apache.org/jira/browse/YARN-1779">YARN-1779</a>.
- Blocker sub-task reported by Karthik Kambatla and fixed by Jian He (resourcemanager)<br>
- <b>Handle AMRMTokens across RM failover</b><br>
+<li> <a href="https://issues.apache.org/jira/browse/YARN-2874">YARN-2874</a>.
+ Blocker bug reported by Naganarasimha G R and fixed by Naganarasimha G R (resourcemanager)<br>
+ <b>Dead lock in "Delega
<TRUNCATED>
[37/50] [abbrv] hadoop git commit: YARN-3544. Got back AM logs link
on the RM web UI for a completed app. Contributed by Xuan Gong.
Posted by sj...@apache.org.
YARN-3544. Got back AM logs link on the RM web UI for a completed app. Contributed by Xuan Gong.
(cherry picked from commit 21bf2cdcb77f69abc906e6cd401a8fb221f250e9)
(cherry picked from commit c9ee316045b83b18cb068aa4de739a1f4b50f02a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d57c3f0a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d57c3f0a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d57c3f0a
Branch: refs/heads/branch-2.6
Commit: d57c3f0a2628f6f540a128e163ffdad4cccb1301
Parents: 7af5d6b
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Apr 29 17:12:52 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 15 17:30:15 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../resourcemanager/webapp/RMAppBlock.java | 82 +++++++-------------
2 files changed, 30 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57c3f0a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c2c2bfa..17d888d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -220,6 +220,9 @@ Release 2.6.1 - 2015-09-09
YARN-3248. Display count of nodes blacklisted by apps in the web UI.
(Varun Vasudev via xgong)
+ YARN-3544. Got back AM logs link on the RM web UI for a completed app.
+ (Xuan Gong via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d57c3f0a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
index 171c6c9..e7f2d1a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
-import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.Set;
@@ -29,36 +28,35 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
-import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.AppBlock;
-import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
-import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
import com.google.inject.Inject;
+//import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
public class RMAppBlock extends AppBlock{
private static final Log LOG = LogFactory.getLog(RMAppBlock.class);
private final ResourceManager rm;
+ private final Configuration conf;
@Inject
RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
super(rm.getClientRMService(), ctx, conf);
+ this.conf = conf;
this.rm = rm;
}
@@ -120,67 +118,41 @@ public class RMAppBlock extends AppBlock{
.th(".started", "Started").th(".node", "Node").th(".logs", "Logs")
.th(".blacklistednodes", "Blacklisted Nodes")._()._().tbody();
+ RMApp rmApp = this.rm.getRMContext().getRMApps().get(this.appID);
+ if (rmApp == null) {
+ return;
+ }
StringBuilder attemptsTableData = new StringBuilder("[\n");
for (final ApplicationAttemptReport appAttemptReport : attempts) {
- AppAttemptInfo appAttempt = new AppAttemptInfo(appAttemptReport);
- ContainerReport containerReport = null;
- try {
- // AM container is always the first container of the attempt
- final GetContainerReportRequest request =
- GetContainerReportRequest.newInstance(ContainerId.newContainerId(
- appAttemptReport.getApplicationAttemptId(), 1));
- if (callerUGI == null) {
- containerReport =
- appBaseProt.getContainerReport(request).getContainerReport();
- } else {
- containerReport = callerUGI.doAs(
- new PrivilegedExceptionAction<ContainerReport>() {
- @Override
- public ContainerReport run() throws Exception {
- ContainerReport report = null;
- try {
- report = appBaseProt.getContainerReport(request)
- .getContainerReport();
- } catch (ContainerNotFoundException ex) {
- LOG.warn(ex.getMessage());
- }
- return report;
- }
- });
- }
- } catch (Exception e) {
- String message =
- "Failed to read the AM container of the application attempt "
- + appAttemptReport.getApplicationAttemptId() + ".";
- LOG.error(message, e);
- html.p()._(message)._();
- return;
- }
- long startTime = 0L;
- String logsLink = null;
- String nodeLink = null;
- if (containerReport != null) {
- ContainerInfo container = new ContainerInfo(containerReport);
- startTime = container.getStartedTime();
- logsLink = containerReport.getLogUrl();
- nodeLink = containerReport.getNodeHttpAddress();
+ RMAppAttempt rmAppAttempt =
+ rmApp.getRMAppAttempt(appAttemptReport.getApplicationAttemptId());
+ if (rmAppAttempt == null) {
+ continue;
}
+ AppAttemptInfo attemptInfo =
+ new AppAttemptInfo(this.rm, rmAppAttempt, rmApp.getUser());
String blacklistedNodesCount = "N/A";
- Set<String> nodes = RMAppAttemptBlock.getBlacklistedNodes(rm,
- ConverterUtils.toApplicationAttemptId(appAttempt.getAppAttemptId()));
+ Set<String> nodes =
+ RMAppAttemptBlock.getBlacklistedNodes(rm,
+ rmAppAttempt.getAppAttemptId());
if(nodes != null) {
blacklistedNodesCount = String.valueOf(nodes.size());
}
// AppAttemptID numerical value parsed by parseHadoopID in
// yarn.dt.plugins.js
+ String nodeLink = attemptInfo.getNodeHttpAddress();
+ if (nodeLink != null) {
+ nodeLink = WebAppUtils.getHttpSchemePrefix(conf) + nodeLink;
+ }
+ String logsLink = attemptInfo.getLogsLink();
attemptsTableData
.append("[\"<a href='")
- .append(url("appattempt", appAttempt.getAppAttemptId()))
+ .append(url("appattempt", rmAppAttempt.getAppAttemptId().toString()))
.append("'>")
- .append(appAttempt.getAppAttemptId())
+ .append(String.valueOf(rmAppAttempt.getAppAttemptId()))
.append("</a>\",\"")
- .append(startTime)
+ .append(attemptInfo.getStartTime())
.append("\",\"<a ")
.append(nodeLink == null ? "#" : "href='" + nodeLink)
.append("'>")
[47/50] [abbrv] hadoop git commit: YARN-4092. Fixed UI redirection to
print useful messages when both RMs are in standby mode. Contributed by Xuan
Gong
Posted by sj...@apache.org.
YARN-4092. Fixed UI redirection to print useful messages when both RMs are in standby mode. Contributed by Xuan Gong
(cherry picked from commit 6b3b487d3f4883a6e849c71886da52c4c4d9f0bf)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cc30002b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cc30002b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cc30002b
Branch: refs/heads/branch-2.6
Commit: cc30002bc838d746903e3a223fdcf29d26647ad9
Parents: ab4c2ce
Author: Jian He <ji...@apache.org>
Authored: Mon Aug 31 22:06:01 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 16:25:13 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/client/TestRMFailover.java | 25 ++++++
.../hadoop/yarn/webapp/YarnWebParams.java | 1 +
.../resourcemanager/webapp/RMWebAppFilter.java | 88 +++++++++++++++++++-
4 files changed, 114 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc30002b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fe89d45..a2e0653 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -8,6 +8,9 @@ Release 2.6.2 - UNRELEASED
IMPROVEMENTS
+ YARN-4092. Fixed UI redirection to print useful messages when both RMs are
+ in standby mode. (Xuan Gong via jianhe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc30002b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
index 294aaaf..37c836d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/TestRMFailover.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.AdminService;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -285,6 +286,7 @@ public class TestRMFailover extends ClientBaseWithFixes {
getAdminService(0).transitionToActive(req);
String rm1Url = "http://0.0.0.0:18088";
String rm2Url = "http://0.0.0.0:28088";
+
String header = getHeader("Refresh", rm2Url);
assertTrue(header.contains("; url=" + rm1Url));
@@ -326,6 +328,16 @@ public class TestRMFailover extends ClientBaseWithFixes {
// Due to the limitation of MiniYARNCluster and dispatcher is a singleton,
// we couldn't add the test case after explicitFailover();
+
+ // transit the active RM to standby
+ // Both of RMs are in standby mode
+ getAdminService(0).transitionToStandby(req);
+ // RM2 is expected to send the httpRequest to itself.
+ // The Header Field: Refresh is expected to be set.
+ String redirectURL = getRefreshURL(rm2Url);
+ assertTrue(redirectURL != null
+ && redirectURL.contains(YarnWebParams.NEXT_REFRESH_INTERVAL)
+ && redirectURL.contains(rm2Url));
}
static String getHeader(String field, String url) {
@@ -340,4 +352,17 @@ public class TestRMFailover extends ClientBaseWithFixes {
return fieldHeader;
}
+ static String getRefreshURL(String url) {
+ String redirectUrl = null;
+ try {
+ HttpURLConnection conn = (HttpURLConnection) new URL(url).openConnection();
+ // do not automatically follow the redirection
+ // otherwise we get too many redirections exception
+ conn.setInstanceFollowRedirects(false);
+ redirectUrl = conn.getHeaderField("Refresh");
+ } catch (Exception e) {
+ // throw new RuntimeException(e);
+ }
+ return redirectUrl;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc30002b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
index 74b2415..e0f9392 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnWebParams.java
@@ -37,4 +37,5 @@ public interface YarnWebParams {
String QUEUE_NAME = "queue.name";
String NODE_STATE = "node.state";
String WEB_UI_TYPE = "web.ui.type";
+ String NEXT_REFRESH_INTERVAL = "next.fresh.interval";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cc30002b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
index 9be07e1..61600dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebAppFilter.java
@@ -20,6 +20,10 @@ package org.apache.hadoop.yarn.server.resourcemanager.webapp;
import java.io.IOException;
import java.io.PrintWriter;
+import java.net.InetSocketAddress;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.Random;
import java.util.Set;
import javax.inject.Inject;
@@ -29,8 +33,11 @@ import javax.servlet.ServletException;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HtmlQuoting;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.webproxy.ProxyUriUtils;
+import org.apache.hadoop.yarn.webapp.YarnWebParams;
import com.google.common.collect.Sets;
import com.google.inject.Injector;
@@ -48,11 +55,26 @@ public class RMWebAppFilter extends GuiceContainer {
// define a set of URIs which do not need to do redirection
private static final Set<String> NON_REDIRECTED_URIS = Sets.newHashSet(
"/conf", "/stacks", "/logLevel", "/logs");
+ private String path;
+ private static final int BASIC_SLEEP_TIME = 5;
+ private static final int MAX_SLEEP_TIME = 5 * 60;
@Inject
- public RMWebAppFilter(Injector injector) {
+ public RMWebAppFilter(Injector injector, Configuration conf) {
super(injector);
this.injector=injector;
+ InetSocketAddress sock = YarnConfiguration.useHttps(conf)
+ ? conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_HTTPS_PORT)
+ : conf.getSocketAddr(YarnConfiguration.RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_RM_WEBAPP_PORT);
+
+ path = sock.getHostName() + ":" + Integer.toString(sock.getPort());
+ path = YarnConfiguration.useHttps(conf)
+ ? "https://" + path
+ : "http://" + path;
}
@Override
@@ -69,9 +91,11 @@ public class RMWebAppFilter extends GuiceContainer {
rmWebApp.checkIfStandbyRM();
if (rmWebApp.isStandby()
&& shouldRedirect(rmWebApp, uri)) {
- String redirectPath = rmWebApp.getRedirectPath() + uri;
+
+ String redirectPath = rmWebApp.getRedirectPath();
if (redirectPath != null && !redirectPath.isEmpty()) {
+ redirectPath += uri;
String redirectMsg =
"This is standby RM. Redirecting to the current active RM: "
+ redirectPath;
@@ -79,11 +103,40 @@ public class RMWebAppFilter extends GuiceContainer {
PrintWriter out = response.getWriter();
out.println(redirectMsg);
return;
+ } else {
+ boolean doRetry = true;
+ String retryIntervalStr =
+ request.getParameter(YarnWebParams.NEXT_REFRESH_INTERVAL);
+ int retryInterval = 0;
+ if (retryIntervalStr != null) {
+ try {
+ retryInterval = Integer.parseInt(retryIntervalStr.trim());
+ } catch (NumberFormatException ex) {
+ doRetry = false;
+ }
+ }
+ int next = calculateExponentialTime(retryInterval);
+
+ String redirectUrl =
+ appendOrReplaceParamter(path + uri,
+ YarnWebParams.NEXT_REFRESH_INTERVAL + "=" + (retryInterval + 1));
+ if (redirectUrl == null || next > MAX_SLEEP_TIME) {
+ doRetry = false;
+ }
+ String redirectMsg =
+ doRetry ? "Can not find any active RM. Will retry in next " + next
+ + " seconds." : "There is no active RM right now.";
+ PrintWriter out = response.getWriter();
+ out.println(redirectMsg);
+ if (doRetry) {
+ response.setHeader("Refresh", next + ";url=" + redirectUrl);
+ response.setStatus(HttpServletResponse.SC_TEMPORARY_REDIRECT);
+ }
}
+ return;
}
super.doFilter(request, response, chain);
-
}
private boolean shouldRedirect(RMWebApp rmWebApp, String uri) {
@@ -92,4 +145,33 @@ public class RMWebAppFilter extends GuiceContainer {
&& !uri.startsWith(ProxyUriUtils.PROXY_BASE)
&& !NON_REDIRECTED_URIS.contains(uri);
}
+
+ private String appendOrReplaceParamter(String uri, String newQuery) {
+ if (uri.contains(YarnWebParams.NEXT_REFRESH_INTERVAL + "=")) {
+ return uri.replaceAll(YarnWebParams.NEXT_REFRESH_INTERVAL + "=[^&]+",
+ newQuery);
+ }
+ try {
+ URI oldUri = new URI(uri);
+ String appendQuery = oldUri.getQuery();
+ if (appendQuery == null) {
+ appendQuery = newQuery;
+ } else {
+ appendQuery += "&" + newQuery;
+ }
+
+ URI newUri =
+ new URI(oldUri.getScheme(), oldUri.getAuthority(), oldUri.getPath(),
+ appendQuery, oldUri.getFragment());
+
+ return newUri.toString();
+ } catch (URISyntaxException e) {
+ return null;
+ }
+ }
+
+ private static int calculateExponentialTime(int retries) {
+ long baseTime = BASIC_SLEEP_TIME * (1L << retries);
+ return (int) (baseTime * ((new Random()).nextDouble() + 0.5));
+ }
}
[04/50] [abbrv] hadoop git commit: YARN-3733. Fix
DominantRC#compare() does not work as expected if cluster resource is empty.
(Rohith Sharmaks via wangda)
Posted by sj...@apache.org.
YARN-3733. Fix DominantRC#compare() does not work as expected if cluster resource is empty. (Rohith Sharmaks via wangda)
(cherry picked from commit ebd797c48fe236b404cf3a125ac9d1f7714e291e)
(cherry picked from commit 78d626fa892415023827e35ad549636e2a83275d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85d92721
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85d92721
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85d92721
Branch: refs/heads/branch-2.6
Commit: 85d92721a42fdc50cc828dfe28103ecee9831924
Parents: f1b35ff
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Jun 4 10:22:57 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 17:43:01 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../resource/DominantResourceCalculator.java | 15 +++++
.../capacity/TestCapacityScheduler.java | 58 +++++++++++++++++++-
3 files changed, 75 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d92721/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a5b270e..8244d61 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -168,6 +168,9 @@ Release 2.6.1 - UNRELEASED
YARN-2637. Fixed max-am-resource-percent calculation in CapacityScheduler
when activating applications. (Craig Welch via jianhe)
+ YARN-3733. Fix DominantRC#compare() does not work as expected if
+ cluster resource is empty. (Rohith Sharmaks via wangda)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d92721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
index 6f5b40e..2ee95ce 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
@@ -53,6 +53,21 @@ public class DominantResourceCalculator extends ResourceCalculator {
return 0;
}
+ if (isInvalidDivisor(clusterResource)) {
+ if ((lhs.getMemory() < rhs.getMemory() && lhs.getVirtualCores() > rhs
+ .getVirtualCores())
+ || (lhs.getMemory() > rhs.getMemory() && lhs.getVirtualCores() < rhs
+ .getVirtualCores())) {
+ return 0;
+ } else if (lhs.getMemory() > rhs.getMemory()
+ || lhs.getVirtualCores() > rhs.getVirtualCores()) {
+ return 1;
+ } else if (lhs.getMemory() < rhs.getMemory()
+ || lhs.getVirtualCores() < rhs.getVirtualCores()) {
+ return -1;
+ }
+ }
+
float l = getResourceAsValue(clusterResource, lhs, true);
float r = getResourceAsValue(clusterResource, rhs, true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85d92721/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
index 2aa57a0..edcd871 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
@@ -118,6 +118,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedule
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.DominantResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
@@ -1136,9 +1137,15 @@ public class TestCapacityScheduler {
private MockRM setUpMove() {
CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ return setUpMove(conf);
+ }
+
+ private MockRM setUpMove(Configuration config) {
+ CapacitySchedulerConfiguration conf =
+ new CapacitySchedulerConfiguration(config);
setupQueueConfiguration(conf);
conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
- ResourceScheduler.class);
+ ResourceScheduler.class);
MockRM rm = new MockRM(conf);
rm.start();
return rm;
@@ -2071,4 +2078,53 @@ public class TestCapacityScheduler {
Assert.assertEquals(0, report.getNumReservedContainers());
rm.stop();
}
+
+ @Test(timeout = 30000)
+ public void testAMLimitUsage() throws Exception {
+
+ CapacitySchedulerConfiguration config =
+ new CapacitySchedulerConfiguration();
+
+ config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DefaultResourceCalculator.class.getName());
+ verifyAMLimitForLeafQueue(config);
+
+ config.set(CapacitySchedulerConfiguration.RESOURCE_CALCULATOR_CLASS,
+ DominantResourceCalculator.class.getName());
+ verifyAMLimitForLeafQueue(config);
+
+ }
+
+ private void verifyAMLimitForLeafQueue(CapacitySchedulerConfiguration config)
+ throws Exception {
+ MockRM rm = setUpMove(config);
+
+ String queueName = "a1";
+ String userName = "user_0";
+ ResourceScheduler scheduler = rm.getRMContext().getScheduler();
+ LeafQueue queueA =
+ (LeafQueue) ((CapacityScheduler) scheduler).getQueue(queueName);
+ Resource amResourceLimit = queueA.getAMResourceLimit();
+
+ Resource amResource =
+ Resource.newInstance(amResourceLimit.getMemory() + 1,
+ amResourceLimit.getVirtualCores() + 1);
+
+ rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+
+ rm.submitApp(amResource.getMemory(), "app-1", userName, null, queueName);
+
+ // When AM limit is exceeded, 1 applications will be activated.Rest all
+ // applications will be in pending
+ Assert.assertEquals("PendingApplications should be 1", 1,
+ queueA.getNumPendingApplications());
+ Assert.assertEquals("Active applications should be 1", 1,
+ queueA.getNumActiveApplications());
+
+ Assert.assertEquals("User PendingApplications should be 1", 1, queueA
+ .getUser(userName).getPendingApplications());
+ Assert.assertEquals("User Active applications should be 1", 1, queueA
+ .getUser(userName).getActiveApplications());
+ rm.stop();
+ }
}
[44/50] [abbrv] hadoop git commit: Preparing for 2.6.2 development:
mvn versions:set -DnewVersion=2.6.2
Posted by sj...@apache.org.
Preparing for 2.6.2 development: mvn versions:set -DnewVersion=2.6.2
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4cb7dbae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4cb7dbae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4cb7dbae
Branch: refs/heads/branch-2.6
Commit: 4cb7dbaead0dcdbb2df7581d30c7895bf6b36707
Parents: 41d19f4
Author: Sangjin Lee <sj...@apache.org>
Authored: Fri Sep 25 15:51:13 2015 -0700
Committer: Sangjin Lee <sj...@apache.org>
Committed: Fri Sep 25 15:51:13 2015 -0700
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 4 ++--
hadoop-client/pom.xml | 4 ++--
hadoop-common-project/hadoop-annotations/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth-examples/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth/pom.xml | 4 ++--
hadoop-common-project/hadoop-common/pom.xml | 4 ++--
hadoop-common-project/hadoop-kms/pom.xml | 4 ++--
hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++--
hadoop-common-project/hadoop-nfs/pom.xml | 4 ++--
hadoop-common-project/pom.xml | 4 ++--
hadoop-dist/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml | 4 ++--
hadoop-hdfs-project/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++--
.../hadoop-mapreduce-client-common/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++--
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 4 ++--
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 ++--
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 4 ++--
hadoop-mapreduce-project/pom.xml | 4 ++--
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 ++--
hadoop-project-dist/pom.xml | 4 ++--
hadoop-project/pom.xml | 4 ++--
hadoop-tools/hadoop-ant/pom.xml | 4 ++--
hadoop-tools/hadoop-archives/pom.xml | 4 ++--
hadoop-tools/hadoop-aws/pom.xml | 4 ++--
hadoop-tools/hadoop-datajoin/pom.xml | 4 ++--
hadoop-tools/hadoop-distcp/pom.xml | 4 ++--
hadoop-tools/hadoop-extras/pom.xml | 4 ++--
hadoop-tools/hadoop-gridmix/pom.xml | 4 ++--
hadoop-tools/hadoop-openstack/pom.xml | 4 ++--
hadoop-tools/hadoop-pipes/pom.xml | 4 ++--
hadoop-tools/hadoop-rumen/pom.xml | 4 ++--
hadoop-tools/hadoop-sls/pom.xml | 4 ++--
hadoop-tools/hadoop-streaming/pom.xml | 4 ++--
hadoop-tools/hadoop-tools-dist/pom.xml | 4 ++--
hadoop-tools/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 4 ++--
.../hadoop-yarn-applications-distributedshell/pom.xml | 4 ++--
.../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 ++--
.../hadoop-yarn-server-applicationhistoryservice/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 4 ++--
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 ++--
hadoop-yarn-project/pom.xml | 4 ++--
pom.xml | 2 +-
62 files changed, 122 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-assemblies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 323ed12..8adbe2e 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -23,12 +23,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-assemblies</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>Apache Hadoop Assemblies</name>
<description>Apache Hadoop Assemblies</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index ad66680..d1ce6c1 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index e501c2b..e69e255 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Annotations</description>
<name>Apache Hadoop Annotations</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-auth-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index 40449e8..88c76ab 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth-examples</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop Auth Examples</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-auth/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index 9e4bea9..3008b09 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop Auth</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 59be1ab..436e27d 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Common</description>
<name>Apache Hadoop Common</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index a54ab7a..49b5a7f 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop KMS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-minikdc/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index 5dc0d2c..c638893 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -18,13 +18,13 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop MiniKDC</description>
<name>Apache Hadoop MiniKDC</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/hadoop-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 1924e5a..209c143 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-nfs</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>jar</packaging>
<name>Apache Hadoop NFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-common-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index 926cb0d..434578e 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Common Project</description>
<name>Apache Hadoop Common Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index d9b40e4..5bae7f0 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Distribution</description>
<name>Apache Hadoop Distribution</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index abdef37..e646fd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-httpfs</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>war</packaging>
<name>Apache Hadoop HttpFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index b377521..ad7046b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-nfs</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop HDFS-NFS</description>
<name>Apache Hadoop HDFS-NFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index 8bafaf1..870ba94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop HDFS</description>
<name>Apache Hadoop HDFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
index c8004c5..f978f52 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
@@ -20,13 +20,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../../../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop.contrib</groupId>
<artifactId>hadoop-hdfs-bkjournal</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop HDFS BookKeeper Journal</description>
<name>Apache Hadoop HDFS BookKeeper Journal</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 37d7702..7b35cf0 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop HDFS Project</description>
<name>Apache Hadoop HDFS Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index cf059e86..018b616 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-app</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 0c64fcb..b031fcc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 78156dc..b3522b5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-core</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
index de34ce0..c888c0e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-hs-plugins</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index 32d693d..93ede19 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-hs</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index 2f6bf3a..b00713f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-jobclient</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
index be7c522..9c1a014 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client-shuffle</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 8627c89..b63b828 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-mapreduce-client</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index fbae3c0..2395848 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Examples</description>
<name>Apache Hadoop MapReduce Examples</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 683256d..b44676a 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-mapreduce</name>
<url>http://hadoop.apache.org/mapreduce/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index 1bbdbfd..0099d58 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index dccb3d1..f71b580 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>jar</packaging>
<description>Apache Hadoop Mini-Cluster</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index e090c69..dd385b2 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Project Dist POM</description>
<name>Apache Hadoop Project Dist POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index da78853..3dec24b 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-ant/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ant/pom.xml b/hadoop-tools/hadoop-ant/pom.xml
index e693347..abccb1a 100644
--- a/hadoop-tools/hadoop-ant/pom.xml
+++ b/hadoop-tools/hadoop-ant/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ant</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Ant Tasks</description>
<name>Apache Hadoop Ant Tasks</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-archives/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index 2792efb..693cbd7 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-archives</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Archives</description>
<name>Apache Hadoop Archives</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 2420d30..0047502 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aws</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>Apache Hadoop Amazon Web Services support</name>
<description>
This module contains code to support integration with Amazon Web Services.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-datajoin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index d107993..9d99062 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-datajoin</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Data Join</description>
<name>Apache Hadoop Data Join</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-distcp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index 2e577fc..5e7c22f 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Distributed Copy</description>
<name>Apache Hadoop Distributed Copy</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-extras/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index d7df4b0..e346131 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-extras</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Extras</description>
<name>Apache Hadoop Extras</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-gridmix/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index 3202537..9110f2b 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-gridmix</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Gridmix</description>
<name>Apache Hadoop Gridmix</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-openstack/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index f2c76be..03ed317 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-openstack</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>Apache Hadoop OpenStack support</name>
<description>
This module contains code to support integration with OpenStack.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-pipes/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 380f1e3..9d819bf 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-pipes</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Pipes</description>
<name>Apache Hadoop Pipes</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-rumen/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index a045373..017a67b 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Rumen</description>
<name>Apache Hadoop Rumen</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index 4420234..2056291 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -19,12 +19,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Scheduler Load Simulator</description>
<name>Apache Hadoop Scheduler Load Simulator</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index 863df08..50388fd 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-streaming</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop MapReduce Streaming</description>
<name>Apache Hadoop MapReduce Streaming</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index a3d2c0f..f5bb27f 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools-dist</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Tools Dist</description>
<name>Apache Hadoop Tools Dist</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index c835c4e..87f11fb 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Tools</description>
<name>Apache Hadoop Tools</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index 5919d17..8ce51a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-api</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 36572cb..2083c5e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-distributedshell</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-applications-distributedshell</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index 76eef6c..0abd20d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-unmanaged-am-launcher</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-applications-unmanaged-am-launcher</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index ace6a23..99594ae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-applications</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 8dd7597..b0720a4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -17,11 +17,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-client</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 679942e..9d7187e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index ebaa1d3..a45c22a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-registry</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-registry</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 7cc1877..a26d5a8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server-applicationhistoryservice</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 5c4ec1f..a853f68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server-common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 2ed0fcc..1dc23bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server-nodemanager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 58842ed..177f4d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server-resourcemanager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index 623bdd7..f82398c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server-tests</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index 0c6d1f5..c3cf4b4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server-web-proxy</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index 27e5830..78e7cef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-server</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
index b156f9e..a7f2568 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-site</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<name>hadoop-yarn-site</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index 296f9be..b144a42 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -16,12 +16,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-yarn</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 8508bae..b505801 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-project</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<packaging>pom</packaging>
<name>hadoop-yarn-project</name>
<url>http://hadoop.apache.org/yarn/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4cb7dbae/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index ce20d86..d17112d 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>2.6.1</version>
+ <version>2.6.2-SNAPSHOT</version>
<description>Apache Hadoop Main</description>
<name>Apache Hadoop Main</name>
<packaging>pom</packaging>
[03/50] [abbrv] hadoop git commit: YARN-2637. Fixed
max-am-resource-percent calculation in CapacityScheduler when activating
applications. Contributed by Craig Welch (cherry picked from commit
c53420f58364b11fbda1dace7679d45534533382)
Posted by sj...@apache.org.
YARN-2637. Fixed max-am-resource-percent calculation in CapacityScheduler when activating applications. Contributed by Craig Welch
(cherry picked from commit c53420f58364b11fbda1dace7679d45534533382)
(cherry picked from commit 4931600030e13d9332d9a0e588487cb8684c667d)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f1b35ffd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f1b35ffd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f1b35ffd
Branch: refs/heads/branch-2.6
Commit: f1b35ffd4ca680d76bf82b541357b8e5748f129e
Parents: 3bd9b74
Author: Jian He <ji...@apache.org>
Authored: Tue Jan 13 17:32:07 2015 -0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Thu Sep 3 17:40:24 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 1 +
.../resourcemanager/RMActiveServiceContext.java | 4 +-
.../server/resourcemanager/RMContextImpl.java | 32 ++-
.../server/resourcemanager/rmapp/RMApp.java | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 5 +
.../scheduler/capacity/CSQueueUtils.java | 24 --
.../scheduler/capacity/LeafQueue.java | 252 ++++++++++++------
.../scheduler/common/fica/FiCaSchedulerApp.java | 15 ++
.../webapp/CapacitySchedulerPage.java | 4 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 24 +-
.../applicationsmanager/MockAsm.java | 7 +
.../TestAMRMRPCNodeUpdates.java | 9 +
.../TestCapacitySchedulerPlanFollower.java | 1 +
.../server/resourcemanager/rmapp/MockRMApp.java | 7 +
.../capacity/TestApplicationLimits.java | 256 +++++++++++++------
.../scheduler/capacity/TestLeafQueue.java | 79 +++---
.../scheduler/capacity/TestReservations.java | 34 +--
.../scheduler/fifo/TestFifoScheduler.java | 23 +-
.../webapp/TestRMWebServicesCapacitySched.java | 12 -
20 files changed, 519 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 60ae3d0..a5b270e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -165,6 +165,9 @@ Release 2.6.1 - UNRELEASED
YARN-3990. AsyncDispatcher may overloaded with RMAppNodeUpdateEvent when
Node is connected/disconnected (Bibin A Chundatt via jlowe)
+ YARN-2637. Fixed max-am-resource-percent calculation in CapacityScheduler
+ when activating applications. (Craig Welch via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 45d7294..971acea 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -173,6 +173,7 @@
<Field name="userLimit" />
<Field name="userLimitFactor" />
<Field name="maxAMResourcePerQueuePercent" />
+ <Field name="lastClusterResource" />
</Or>
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
index 3bc2e9b..03fc40e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMActiveServiceContext.java
@@ -117,7 +117,8 @@ public class RMActiveServiceContext {
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ RMApplicationHistoryWriter rmApplicationHistoryWriter,
+ ResourceScheduler scheduler) {
this();
this.setContainerAllocationExpirer(containerAllocationExpirer);
this.setAMLivelinessMonitor(amLivelinessMonitor);
@@ -128,6 +129,7 @@ public class RMActiveServiceContext {
this.setNMTokenSecretManager(nmTokenSecretManager);
this.setClientToAMTokenSecretManager(clientToAMTokenSecretManager);
this.setRMApplicationHistoryWriter(rmApplicationHistoryWriter);
+ this.setScheduler(scheduler);
RMStateStore nullStore = new NullRMStateStore();
nullStore.setRMDispatcher(rmDispatcher);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
index 55d7667..32216e5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
@@ -84,18 +84,46 @@ public class RMContextImpl implements RMContext {
RMContainerTokenSecretManager containerTokenSecretManager,
NMTokenSecretManagerInRM nmTokenSecretManager,
ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
- RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ RMApplicationHistoryWriter rmApplicationHistoryWriter,
+ ResourceScheduler scheduler) {
this();
this.setDispatcher(rmDispatcher);
setActiveServiceContext(new RMActiveServiceContext(rmDispatcher,
containerAllocationExpirer, amLivelinessMonitor, amFinishingMonitor,
delegationTokenRenewer, appTokenSecretManager,
containerTokenSecretManager, nmTokenSecretManager,
- clientToAMTokenSecretManager, rmApplicationHistoryWriter));
+ clientToAMTokenSecretManager, rmApplicationHistoryWriter,
+ scheduler));
ConfigurationProvider provider = new LocalConfigurationProvider();
setConfigurationProvider(provider);
}
+
+ @VisibleForTesting
+ // helper constructor for tests
+ public RMContextImpl(Dispatcher rmDispatcher,
+ ContainerAllocationExpirer containerAllocationExpirer,
+ AMLivelinessMonitor amLivelinessMonitor,
+ AMLivelinessMonitor amFinishingMonitor,
+ DelegationTokenRenewer delegationTokenRenewer,
+ AMRMTokenSecretManager appTokenSecretManager,
+ RMContainerTokenSecretManager containerTokenSecretManager,
+ NMTokenSecretManagerInRM nmTokenSecretManager,
+ ClientToAMTokenSecretManagerInRM clientToAMTokenSecretManager,
+ RMApplicationHistoryWriter rmApplicationHistoryWriter) {
+ this(
+ rmDispatcher,
+ containerAllocationExpirer,
+ amLivelinessMonitor,
+ amFinishingMonitor,
+ delegationTokenRenewer,
+ appTokenSecretManager,
+ containerTokenSecretManager,
+ nmTokenSecretManager,
+ clientToAMTokenSecretManager,
+ rmApplicationHistoryWriter,
+ null);
+ }
@Override
public Dispatcher getDispatcher() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
index 624aa18..fbcaab9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMApp.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ReservationId;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
@@ -239,4 +240,6 @@ public interface RMApp extends EventHandler<RMAppEvent> {
RMAppMetrics getRMAppMetrics();
ReservationId getReservationId();
+
+ ResourceRequest getAMResourceRequest();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 751dbe4..19f2193 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -1337,6 +1337,11 @@ public class RMAppImpl implements RMApp, Recoverable {
public ReservationId getReservationId() {
return submissionContext.getReservationID();
}
+
+ @Override
+ public ResourceRequest getAMResourceRequest() {
+ return this.amReq;
+ }
protected Credentials parseCredentials() throws IOException {
Credentials credentials = new Credentials();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
index 0a2fa3a..f458057 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueueUtils.java
@@ -109,30 +109,6 @@ class CSQueueUtils {
}
return absoluteMaxCapacityByNodeLabels;
}
-
- public static int computeMaxActiveApplications(
- ResourceCalculator calculator,
- Resource clusterResource, Resource minimumAllocation,
- float maxAMResourcePercent, float absoluteMaxCapacity) {
- return
- Math.max(
- (int)Math.ceil(
- Resources.ratio(
- calculator,
- clusterResource,
- minimumAllocation) *
- maxAMResourcePercent * absoluteMaxCapacity
- ),
- 1);
- }
-
- public static int computeMaxActiveApplicationsPerUser(
- int maxActiveApplications, int userLimit, float userLimitFactor) {
- return Math.max(
- (int)Math.ceil(
- maxActiveApplications * (userLimit / 100.0f) * userLimitFactor),
- 1);
- }
@Lock(CSQueue.class)
public static void updateQueueStatistics(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 6ffc61a..60b5a59 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -87,9 +87,6 @@ public class LeafQueue extends AbstractCSQueue {
protected int maxApplicationsPerUser;
private float maxAMResourcePerQueuePercent;
- private int maxActiveApplications; // Based on absolute max capacity
- private int maxActiveAppsUsingAbsCap; // Based on absolute capacity
- private int maxActiveApplicationsPerUser;
private int nodeLocalityDelay;
@@ -113,9 +110,16 @@ public class LeafQueue extends AbstractCSQueue {
// cache last cluster resource to compute actual capacity
private Resource lastClusterResource = Resources.none();
+ // absolute capacity as a resource (based on cluster resource)
+ private Resource absoluteCapacityResource = Resources.none();
+
private final QueueHeadroomInfo queueHeadroomInfo = new QueueHeadroomInfo();
private volatile float absoluteMaxAvailCapacity;
+
+ // sum of resources used by application masters for applications
+ // running in this queue
+ private final Resource usedAMResources = Resource.newInstance(0, 0);
public LeafQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
@@ -155,19 +159,6 @@ public class LeafQueue extends AbstractCSQueue {
float maxAMResourcePerQueuePercent = cs.getConfiguration()
.getMaximumApplicationMasterResourcePerQueuePercent(getQueuePath());
- int maxActiveApplications =
- CSQueueUtils.computeMaxActiveApplications(
- resourceCalculator,
- cs.getClusterResource(), this.minimumAllocation,
- maxAMResourcePerQueuePercent, absoluteMaxCapacity);
- this.maxActiveAppsUsingAbsCap =
- CSQueueUtils.computeMaxActiveApplications(
- resourceCalculator,
- cs.getClusterResource(), this.minimumAllocation,
- maxAMResourcePerQueuePercent, absoluteCapacity);
- int maxActiveApplicationsPerUser =
- CSQueueUtils.computeMaxActiveApplicationsPerUser(
- maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
this.queueInfo.setChildQueues(new ArrayList<QueueInfo>());
@@ -179,8 +170,7 @@ public class LeafQueue extends AbstractCSQueue {
setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
maximumCapacity, absoluteMaxCapacity, userLimit, userLimitFactor,
maxApplications, maxAMResourcePerQueuePercent, maxApplicationsPerUser,
- maxActiveApplications, maxActiveApplicationsPerUser, state, acls, cs
- .getConfiguration().getNodeLocalityDelay(), accessibleLabels,
+ state, acls, cs.getConfiguration().getNodeLocalityDelay(), accessibleLabels,
defaultLabelExpression, this.capacitiyByNodeLabels,
this.maxCapacityByNodeLabels,
cs.getConfiguration().getReservationContinueLook());
@@ -208,8 +198,7 @@ public class LeafQueue extends AbstractCSQueue {
float maximumCapacity, float absoluteMaxCapacity,
int userLimit, float userLimitFactor,
int maxApplications, float maxAMResourcePerQueuePercent,
- int maxApplicationsPerUser, int maxActiveApplications,
- int maxActiveApplicationsPerUser, QueueState state,
+ int maxApplicationsPerUser, QueueState state,
Map<QueueACL, AccessControlList> acls, int nodeLocalityDelay,
Set<String> labels, String defaultLabelExpression,
Map<String, Float> capacitieByLabel,
@@ -224,6 +213,16 @@ public class LeafQueue extends AbstractCSQueue {
float absCapacity = getParent().getAbsoluteCapacity() * capacity;
CSQueueUtils.checkAbsoluteCapacity(getQueueName(), absCapacity,
absoluteMaxCapacity);
+
+ this.lastClusterResource = clusterResource;
+ updateAbsoluteCapacityResource(clusterResource);
+
+ // Initialize headroom info, also used for calculating application
+ // master resource limits. Since this happens during queue initialization
+ // and all queues may not be realized yet, we'll use (optimistic)
+ // absoluteMaxCapacity (it will be replaced with the more accurate
+ // absoluteMaxAvailCapacity during headroom/userlimit/allocation events)
+ updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
this.absoluteCapacity = absCapacity;
@@ -234,9 +233,6 @@ public class LeafQueue extends AbstractCSQueue {
this.maxAMResourcePerQueuePercent = maxAMResourcePerQueuePercent;
this.maxApplicationsPerUser = maxApplicationsPerUser;
- this.maxActiveApplications = maxActiveApplications;
- this.maxActiveApplicationsPerUser = maxActiveApplicationsPerUser;
-
if (!SchedulerUtils.checkQueueLabelExpression(this.accessibleLabels,
this.defaultLabelExpression)) {
throw new IOException("Invalid default label expression of "
@@ -288,21 +284,6 @@ public class LeafQueue extends AbstractCSQueue {
"maxApplicationsPerUser = " + maxApplicationsPerUser +
" [= (int)(maxApplications * (userLimit / 100.0f) * " +
"userLimitFactor) ]" + "\n" +
- "maxActiveApplications = " + maxActiveApplications +
- " [= max(" +
- "(int)ceil((clusterResourceMemory / minimumAllocation) * " +
- "maxAMResourcePerQueuePercent * absoluteMaxCapacity)," +
- "1) ]" + "\n" +
- "maxActiveAppsUsingAbsCap = " + maxActiveAppsUsingAbsCap +
- " [= max(" +
- "(int)ceil((clusterResourceMemory / minimumAllocation) *" +
- "maxAMResourcePercent * absoluteCapacity)," +
- "1) ]" + "\n" +
- "maxActiveApplicationsPerUser = " + maxActiveApplicationsPerUser +
- " [= max(" +
- "(int)(maxActiveApplications * (userLimit / 100.0f) * " +
- "userLimitFactor)," +
- "1) ]" + "\n" +
"usedCapacity = " + usedCapacity +
" [= usedResourcesMemory / " +
"(clusterResourceMemory * absoluteCapacity)]" + "\n" +
@@ -355,14 +336,6 @@ public class LeafQueue extends AbstractCSQueue {
return maxApplicationsPerUser;
}
- public synchronized int getMaximumActiveApplications() {
- return maxActiveApplications;
- }
-
- public synchronized int getMaximumActiveApplicationsPerUser() {
- return maxActiveApplicationsPerUser;
- }
-
@Override
public ActiveUsersManager getActiveUsersManager() {
return activeUsersManager;
@@ -525,8 +498,6 @@ public class LeafQueue extends AbstractCSQueue {
newlyParsedLeafQueue.maxApplications,
newlyParsedLeafQueue.maxAMResourcePerQueuePercent,
newlyParsedLeafQueue.getMaxApplicationsPerUser(),
- newlyParsedLeafQueue.getMaximumActiveApplications(),
- newlyParsedLeafQueue.getMaximumActiveApplicationsPerUser(),
newlyParsedLeafQueue.state, newlyParsedLeafQueue.acls,
newlyParsedLeafQueue.getNodeLocalityDelay(),
newlyParsedLeafQueue.accessibleLabels,
@@ -612,27 +583,115 @@ public class LeafQueue extends AbstractCSQueue {
}
}
+
+ public synchronized Resource getAMResourceLimit() {
+ /*
+ * The limit to the amount of resources which can be consumed by
+ * application masters for applications running in the queue
+ * is calculated by taking the greater of the max resources currently
+ * available to the queue (see absoluteMaxAvailCapacity) and the absolute
+ * resources guaranteed for the queue and multiplying it by the am
+ * resource percent.
+ *
+ * This is to allow a queue to grow its (proportional) application
+ * master resource use up to its max capacity when other queues are
+ * idle but to scale back down to it's guaranteed capacity as they
+ * become busy.
+ *
+ */
+ Resource queueMaxCap;
+ synchronized (queueHeadroomInfo) {
+ queueMaxCap = queueHeadroomInfo.getQueueMaxCap();
+ }
+ Resource queueCap = Resources.max(resourceCalculator, lastClusterResource,
+ absoluteCapacityResource, queueMaxCap);
+ return Resources.multiplyAndNormalizeUp(
+ resourceCalculator,
+ queueCap,
+ maxAMResourcePerQueuePercent, minimumAllocation);
+ }
+
+ public synchronized Resource getUserAMResourceLimit() {
+ /*
+ * The user amresource limit is based on the same approach as the
+ * user limit (as it should represent a subset of that). This means that
+ * it uses the absolute queue capacity instead of the max and is modified
+ * by the userlimit and the userlimit factor as is the userlimit
+ *
+ */
+ float effectiveUserLimit = Math.max(userLimit / 100.0f, 1.0f /
+ Math.max(getActiveUsersManager().getNumActiveUsers(), 1));
+
+ return Resources.multiplyAndNormalizeUp(
+ resourceCalculator,
+ absoluteCapacityResource,
+ maxAMResourcePerQueuePercent * effectiveUserLimit *
+ userLimitFactor, minimumAllocation);
+ }
private synchronized void activateApplications() {
+ //limit of allowed resource usage for application masters
+ Resource amLimit = getAMResourceLimit();
+ Resource userAMLimit = getUserAMResourceLimit();
+
for (Iterator<FiCaSchedulerApp> i=pendingApplications.iterator();
i.hasNext(); ) {
FiCaSchedulerApp application = i.next();
- // Check queue limit
- if (getNumActiveApplications() >= getMaximumActiveApplications()) {
- break;
+ // Check am resource limit
+ Resource amIfStarted =
+ Resources.add(application.getAMResource(), usedAMResources);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("application AMResource " + application.getAMResource() +
+ " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent +
+ " amLimit " + amLimit +
+ " lastClusterResource " + lastClusterResource +
+ " amIfStarted " + amIfStarted);
+ }
+
+ if (!Resources.lessThanOrEqual(
+ resourceCalculator, lastClusterResource, amIfStarted, amLimit)) {
+ if (getNumActiveApplications() < 1) {
+ LOG.warn("maximum-am-resource-percent is insufficient to start a" +
+ " single application in queue, it is likely set too low." +
+ " skipping enforcement to allow at least one application to start");
+ } else {
+ LOG.info("not starting application as amIfStarted exceeds amLimit");
+ continue;
+ }
}
- // Check user limit
+ // Check user am resource limit
+
User user = getUser(application.getUser());
- if (user.getActiveApplications() < getMaximumActiveApplicationsPerUser()) {
- user.activateApplication();
- activeApplications.add(application);
- i.remove();
- LOG.info("Application " + application.getApplicationId() +
- " from user: " + application.getUser() +
- " activated in queue: " + getQueueName());
+
+ Resource userAmIfStarted =
+ Resources.add(application.getAMResource(),
+ user.getConsumedAMResources());
+
+ if (!Resources.lessThanOrEqual(
+ resourceCalculator, lastClusterResource, userAmIfStarted,
+ userAMLimit)) {
+ if (getNumActiveApplications() < 1) {
+ LOG.warn("maximum-am-resource-percent is insufficient to start a" +
+ " single application in queue for user, it is likely set too low." +
+ " skipping enforcement to allow at least one application to start");
+ } else {
+ LOG.info("not starting application as amIfStarted exceeds " +
+ "userAmLimit");
+ continue;
+ }
}
+ user.activateApplication();
+ activeApplications.add(application);
+ Resources.addTo(usedAMResources, application.getAMResource());
+ Resources.addTo(user.getConsumedAMResources(),
+ application.getAMResource());
+ i.remove();
+ LOG.info("Application " + application.getApplicationId() +
+ " from user: " + application.getUser() +
+ " activated in queue: " + getQueueName());
}
}
@@ -678,6 +737,10 @@ public class LeafQueue extends AbstractCSQueue {
boolean wasActive = activeApplications.remove(application);
if (!wasActive) {
pendingApplications.remove(application);
+ } else {
+ Resources.subtractFrom(usedAMResources, application.getAMResource());
+ Resources.subtractFrom(user.getConsumedAMResources(),
+ application.getAMResource());
}
applicationAttemptMap.remove(application.getApplicationAttemptId());
@@ -1015,6 +1078,25 @@ public class LeafQueue extends AbstractCSQueue {
return canAssign;
}
+
+ private Resource updateHeadroomInfo(Resource clusterResource,
+ float absoluteMaxAvailCapacity) {
+
+ Resource queueMaxCap =
+ Resources.multiplyAndNormalizeDown(
+ resourceCalculator,
+ clusterResource,
+ absoluteMaxAvailCapacity,
+ minimumAllocation);
+
+ synchronized (queueHeadroomInfo) {
+ queueHeadroomInfo.setQueueMaxCap(queueMaxCap);
+ queueHeadroomInfo.setClusterResource(clusterResource);
+ }
+
+ return queueMaxCap;
+
+ }
@Lock({LeafQueue.class, FiCaSchedulerApp.class})
Resource computeUserLimitAndSetHeadroom(FiCaSchedulerApp application,
@@ -1027,18 +1109,9 @@ public class LeafQueue extends AbstractCSQueue {
Resource userLimit =
computeUserLimit(application, clusterResource, required,
queueUser, requestedLabels);
-
- Resource queueMaxCap = // Queue Max-Capacity
- Resources.multiplyAndNormalizeDown(
- resourceCalculator,
- clusterResource,
- absoluteMaxAvailCapacity,
- minimumAllocation);
-
- synchronized (queueHeadroomInfo) {
- queueHeadroomInfo.setQueueMaxCap(queueMaxCap);
- queueHeadroomInfo.setClusterResource(clusterResource);
- }
+
+ Resource queueMaxCap =
+ updateHeadroomInfo(clusterResource, absoluteMaxAvailCapacity);
Resource headroom =
getHeadroom(queueUser, queueMaxCap, clusterResource, userLimit);
@@ -1734,25 +1807,25 @@ public class LeafQueue extends AbstractCSQueue {
" used=" + usedResources + " numContainers=" + numContainers +
" user=" + userName + " user-resources=" + user.getTotalConsumedResources());
}
+
+ private void updateAbsoluteCapacityResource(Resource clusterResource) {
+
+ absoluteCapacityResource = Resources.multiplyAndNormalizeUp(
+ resourceCalculator,
+ clusterResource,
+ absoluteCapacity, minimumAllocation);
+
+ }
@Override
public synchronized void updateClusterResource(Resource clusterResource) {
lastClusterResource = clusterResource;
+ updateAbsoluteCapacityResource(clusterResource);
- // Update queue properties
- maxActiveApplications =
- CSQueueUtils.computeMaxActiveApplications(
- resourceCalculator,
- clusterResource, minimumAllocation,
- maxAMResourcePerQueuePercent, absoluteMaxCapacity);
- maxActiveAppsUsingAbsCap =
- CSQueueUtils.computeMaxActiveApplications(
- resourceCalculator,
- clusterResource, minimumAllocation,
- maxAMResourcePerQueuePercent, absoluteCapacity);
- maxActiveApplicationsPerUser =
- CSQueueUtils.computeMaxActiveApplicationsPerUser(
- maxActiveAppsUsingAbsCap, userLimit, userLimitFactor);
+ // Update headroom info based on new cluster resource value
+ // absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
+ // during allocation
+ updateHeadroomInfo(clusterResource, absoluteMaxCapacity);
// Update metrics
CSQueueUtils.updateQueueStatistics(
@@ -1775,6 +1848,7 @@ public class LeafQueue extends AbstractCSQueue {
@VisibleForTesting
public static class User {
Resource consumed = Resources.createResource(0, 0);
+ Resource consumedAMResources = Resources.createResource(0, 0);
Map<String, Resource> consumedByLabel = new HashMap<String, Resource>();
int pendingApplications = 0;
int activeApplications = 0;
@@ -1798,6 +1872,10 @@ public class LeafQueue extends AbstractCSQueue {
public int getActiveApplications() {
return activeApplications;
}
+
+ public Resource getConsumedAMResources() {
+ return consumedAMResources;
+ }
public int getTotalApplications() {
return getPendingApplications() + getActiveApplications();
@@ -1943,6 +2021,10 @@ public class LeafQueue extends AbstractCSQueue {
@Override
public float getAbsActualCapacity() {
+ //? Is this actually used by anything at present?
+ // There is a findbugs warning -re lastClusterResource (now excluded),
+ // when this is used, verify that the access is mt correct and remove
+ // the findbugs exclusion if possible
if (Resources.lessThanOrEqual(resourceCalculator, lastClusterResource,
lastClusterResource, Resources.none())) {
return absoluteCapacity;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 2f9569c..9f97b13 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -52,6 +52,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicat
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
/**
* Represents an application attempt from the viewpoint of the FIFO or Capacity
@@ -72,6 +73,20 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
String user, Queue queue, ActiveUsersManager activeUsersManager,
RMContext rmContext) {
super(applicationAttemptId, user, queue, activeUsersManager, rmContext);
+
+ RMApp rmApp = rmContext.getRMApps().get(getApplicationId());
+
+ Resource amResource;
+ if (rmApp == null || rmApp.getAMResourceRequest() == null) {
+ //the rmApp may be undefined (the resource manager checks for this too)
+ //and unmanaged applications do not provide an amResource request
+ //in these cases, provide a default using the scheduler
+ amResource = rmContext.getScheduler().getMinimumResourceCapability();
+ } else {
+ amResource = rmApp.getAMResourceRequest().getCapability();
+ }
+
+ setAMResource(amResource);
}
synchronized public boolean containerCompleted(RMContainer rmContainer,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
index ffead48..0be361a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/CapacitySchedulerPage.java
@@ -115,8 +115,8 @@ class CapacitySchedulerPage extends RmView {
_("Num Containers:", Integer.toString(lqinfo.getNumContainers())).
_("Max Applications:", Integer.toString(lqinfo.getMaxApplications())).
_("Max Applications Per User:", Integer.toString(lqinfo.getMaxApplicationsPerUser())).
- _("Max Schedulable Applications:", Integer.toString(lqinfo.getMaxActiveApplications())).
- _("Max Schedulable Applications Per User:", Integer.toString(lqinfo.getMaxActiveApplicationsPerUser())).
+ _("Max Application Master Resources:", lqinfo.getAMResourceLimit().toString()).
+ _("Max Application Master Resources Per User:", lqinfo.getUserAMResourceLimit().toString()).
_("Configured Capacity:", percent(lqinfo.getCapacity() / 100)).
_("Configured Max Capacity:", percent(lqinfo.getMaxCapacity() / 100)).
_("Configured Minimum User Limit Percent:", Integer.toString(lqinfo.getUserLimit()) + "%").
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
index d90e963..bb4c749 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/CapacitySchedulerLeafQueueInfo.java
@@ -32,11 +32,11 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
protected int numContainers;
protected int maxApplications;
protected int maxApplicationsPerUser;
- protected int maxActiveApplications;
- protected int maxActiveApplicationsPerUser;
protected int userLimit;
protected UsersInfo users; // To add another level in the XML
protected float userLimitFactor;
+ protected ResourceInfo aMResourceLimit;
+ protected ResourceInfo userAMResourceLimit;
CapacitySchedulerLeafQueueInfo() {
};
@@ -48,11 +48,11 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
numContainers = q.getNumContainers();
maxApplications = q.getMaxApplications();
maxApplicationsPerUser = q.getMaxApplicationsPerUser();
- maxActiveApplications = q.getMaximumActiveApplications();
- maxActiveApplicationsPerUser = q.getMaximumActiveApplicationsPerUser();
userLimit = q.getUserLimit();
users = new UsersInfo(q.getUsers());
userLimitFactor = q.getUserLimitFactor();
+ aMResourceLimit = new ResourceInfo(q.getAMResourceLimit());
+ userAMResourceLimit = new ResourceInfo(q.getUserAMResourceLimit());
}
public int getNumActiveApplications() {
@@ -75,14 +75,6 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
return maxApplicationsPerUser;
}
- public int getMaxActiveApplications() {
- return maxActiveApplications;
- }
-
- public int getMaxActiveApplicationsPerUser() {
- return maxActiveApplicationsPerUser;
- }
-
public int getUserLimit() {
return userLimit;
}
@@ -95,4 +87,12 @@ public class CapacitySchedulerLeafQueueInfo extends CapacitySchedulerQueueInfo {
public float getUserLimitFactor() {
return userLimitFactor;
}
+
+ public ResourceInfo getAMResourceLimit() {
+ return aMResourceLimit;
+ }
+
+ public ResourceInfo getUserAMResourceLimit() {
+ return userAMResourceLimit;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
index 62e3e5c..f8d92aa 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/MockAsm.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
@@ -51,6 +52,7 @@ import com.google.common.collect.Lists;
public abstract class MockAsm extends MockApps {
public static class ApplicationBase implements RMApp {
+ ResourceRequest amReq;
@Override
public String getUser() {
throw new UnsupportedOperationException("Not supported yet.");
@@ -183,6 +185,11 @@ public abstract class MockAsm extends MockApps {
public ReservationId getReservationId() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ @Override
+ public ResourceRequest getAMResourceRequest() {
+ return this.amReq;
+ }
}
public static RMApp newApplication(int i) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
index e93d351..f4cb3b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRMRPCNodeUpdates.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.junit.Assert;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -55,6 +57,13 @@ public class TestAMRMRPCNodeUpdates {
dispatcher = new DrainDispatcher();
this.rm = new MockRM() {
@Override
+ public void init(Configuration conf) {
+ conf.set(
+ CapacitySchedulerConfiguration.MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT,
+ "1.0");
+ super.init(conf);
+ }
+ @Override
protected EventHandler<SchedulerEvent> createSchedulerEventDispatcher() {
return new SchedulerEventDispatcher(this.scheduler) {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
index 4eedd42..f2840b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -95,6 +95,7 @@ public class TestCapacitySchedulerPlanFollower {
.thenReturn(null);
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
+ when(spyRMContext.getScheduler()).thenReturn(scheduler);
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
index 787b5d7..ec990f9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/MockRMApp.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationSubmissionContextPBImpl;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -55,6 +56,7 @@ public class MockRMApp implements RMApp {
StringBuilder diagnostics = new StringBuilder();
RMAppAttempt attempt;
int maxAppAttempts = 1;
+ ResourceRequest amReq;
public MockRMApp(int newid, long time, RMAppState newState) {
finish = time;
@@ -264,4 +266,9 @@ public class MockRMApp implements RMApp {
public ReservationId getReservationId() {
throw new UnsupportedOperationException("Not supported yet.");
}
+
+ @Override
+ public ResourceRequest getAMResourceRequest() {
+ return this.amReq;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 0cd74d0..81a5aad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -28,16 +28,21 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import org.mockito.Matchers;
+import org.mockito.Mockito;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
@@ -47,8 +52,10 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
@@ -56,6 +63,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import org.junit.Ignore;
public class TestApplicationLimits {
@@ -119,8 +127,6 @@ public class TestApplicationLimits {
// Some default values
doReturn(100).when(queue).getMaxApplications();
doReturn(25).when(queue).getMaxApplicationsPerUser();
- doReturn(10).when(queue).getMaximumActiveApplications();
- doReturn(2).when(queue).getMaximumActiveApplicationsPerUser();
}
private static final String A = "a";
@@ -136,10 +142,14 @@ public class TestApplicationLimits {
final String Q_B = CapacitySchedulerConfiguration.ROOT + "." + B;
conf.setCapacity(Q_B, 90);
+ conf.setUserLimit(CapacitySchedulerConfiguration.ROOT + "." + A, 50);
+ conf.setUserLimitFactor(CapacitySchedulerConfiguration.ROOT + "." + A, 5.0f);
+
LOG.info("Setup top-level queues a and b");
}
- private FiCaSchedulerApp getMockApplication(int appId, String user) {
+ private FiCaSchedulerApp getMockApplication(int appId, String user,
+ Resource amResource) {
FiCaSchedulerApp application = mock(FiCaSchedulerApp.class);
ApplicationAttemptId applicationAttemptId =
TestUtils.getMockApplicationAttemptId(appId, 0);
@@ -147,10 +157,90 @@ public class TestApplicationLimits {
when(application).getApplicationId();
doReturn(applicationAttemptId). when(application).getApplicationAttemptId();
doReturn(user).when(application).getUser();
+ doReturn(amResource).when(application).getAMResource();
return application;
}
@Test
+ public void testAMResourceLimit() throws Exception {
+ final String user_0 = "user_0";
+ final String user_1 = "user_1";
+
+ // This uses the default 10% of cluster value for the max am resources
+ // which are allowed, at 80GB = 8GB for AM's at the queue level. The user
+ // am limit is 4G initially (based on the queue absolute capacity)
+ // when there is only 1 user, and drops to 2G (the userlimit) when there
+ // is a second user
+ queue.updateClusterResource(Resource.newInstance(80 * GB, 40));
+
+ ActiveUsersManager activeUsersManager = mock(ActiveUsersManager.class);
+ when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
+
+ assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+ assertEquals(Resource.newInstance(4 * GB, 1),
+ queue.getUserAMResourceLimit());
+
+ // Two apps for user_0, both start
+ int APPLICATION_ID = 0;
+ FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0,
+ Resource.newInstance(2 * GB, 1));
+ queue.submitApplicationAttempt(app_0, user_0);
+ assertEquals(1, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(1, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+
+ when(activeUsersManager.getNumActiveUsers()).thenReturn(1);
+
+ FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0,
+ Resource.newInstance(2 * GB, 1));
+ queue.submitApplicationAttempt(app_1, user_0);
+ assertEquals(2, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(2, queue.getNumActiveApplications(user_0));
+ assertEquals(0, queue.getNumPendingApplications(user_0));
+
+ // AMLimits unchanged
+ assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+ assertEquals(Resource.newInstance(4 * GB, 1),
+ queue.getUserAMResourceLimit());
+
+ // One app for user_1, starts
+ FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_1,
+ Resource.newInstance(2 * GB, 1));
+ queue.submitApplicationAttempt(app_2, user_1);
+ assertEquals(3, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(1, queue.getNumActiveApplications(user_1));
+ assertEquals(0, queue.getNumPendingApplications(user_1));
+
+ when(activeUsersManager.getNumActiveUsers()).thenReturn(2);
+
+ // Now userAMResourceLimit drops to the queue configured 50% as there is
+ // another user active
+ assertEquals(Resource.newInstance(8 * GB, 1), queue.getAMResourceLimit());
+ assertEquals(Resource.newInstance(2 * GB, 1),
+ queue.getUserAMResourceLimit());
+
+ // Second user_1 app cannot start
+ FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_1,
+ Resource.newInstance(2 * GB, 1));
+ queue.submitApplicationAttempt(app_3, user_1);
+ assertEquals(3, queue.getNumActiveApplications());
+ assertEquals(1, queue.getNumPendingApplications());
+ assertEquals(1, queue.getNumActiveApplications(user_1));
+ assertEquals(1, queue.getNumPendingApplications(user_1));
+
+ // Now finish app so another should be activated
+ queue.finishApplicationAttempt(app_2, A);
+ assertEquals(3, queue.getNumActiveApplications());
+ assertEquals(0, queue.getNumPendingApplications());
+ assertEquals(1, queue.getNumActiveApplications(user_1));
+ assertEquals(0, queue.getNumPendingApplications(user_1));
+
+ }
+
+ @Test
public void testLimitsComputation() throws Exception {
CapacitySchedulerConfiguration csConf =
new CapacitySchedulerConfiguration();
@@ -172,7 +262,8 @@ public class TestApplicationLimits {
when(csContext.getRMContext()).thenReturn(rmContext);
// Say cluster has 100 nodes of 16G each
- Resource clusterResource = Resources.createResource(100 * 16 * GB, 100 * 16);
+ Resource clusterResource =
+ Resources.createResource(100 * 16 * GB, 100 * 16);
when(csContext.getClusterResource()).thenReturn(clusterResource);
Map<String, CSQueue> queues = new HashMap<String, CSQueue>();
@@ -183,28 +274,14 @@ public class TestApplicationLimits {
LeafQueue queue = (LeafQueue)queues.get(A);
LOG.info("Queue 'A' -" +
- " maxActiveApplications=" + queue.getMaximumActiveApplications() +
- " maxActiveApplicationsPerUser=" +
- queue.getMaximumActiveApplicationsPerUser());
- int expectedMaxActiveApps =
- Math.max(1,
- (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.
- getMaximumApplicationMasterResourcePerQueuePercent(
- queue.getQueuePath()) *
- queue.getAbsoluteMaximumCapacity()));
- assertEquals(expectedMaxActiveApps,
- queue.getMaximumActiveApplications());
- int expectedMaxActiveAppsUsingAbsCap =
- Math.max(1,
- (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.getMaximumApplicationMasterResourcePercent() *
- queue.getAbsoluteCapacity()));
- assertEquals(
- (int)Math.ceil(
- expectedMaxActiveAppsUsingAbsCap * (queue.getUserLimit() / 100.0f) *
- queue.getUserLimitFactor()),
- queue.getMaximumActiveApplicationsPerUser());
+ " aMResourceLimit=" + queue.getAMResourceLimit() +
+ " UserAMResourceLimit=" +
+ queue.getUserAMResourceLimit());
+
+ assertEquals(queue.getAMResourceLimit(), Resource.newInstance(160*GB, 1));
+ assertEquals(queue.getUserAMResourceLimit(),
+ Resource.newInstance(80*GB, 1));
+
assertEquals(
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
@@ -213,24 +290,11 @@ public class TestApplicationLimits {
// Add some nodes to the cluster & test new limits
clusterResource = Resources.createResource(120 * 16 * GB);
root.updateClusterResource(clusterResource);
- expectedMaxActiveApps =
- Math.max(1,
- (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.
- getMaximumApplicationMasterResourcePerQueuePercent(
- queue.getQueuePath()) *
- queue.getAbsoluteMaximumCapacity()));
- assertEquals(expectedMaxActiveApps,
- queue.getMaximumActiveApplications());
- expectedMaxActiveAppsUsingAbsCap =
- Math.max(1,
- (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.getMaximumApplicationMasterResourcePercent() *
- queue.getAbsoluteCapacity()));
- assertEquals(
- (int)Math.ceil(expectedMaxActiveAppsUsingAbsCap *
- (queue.getUserLimit() / 100.0f) * queue.getUserLimitFactor()),
- queue.getMaximumActiveApplicationsPerUser());
+
+ assertEquals(queue.getAMResourceLimit(), Resource.newInstance(192*GB, 1));
+ assertEquals(queue.getUserAMResourceLimit(),
+ Resource.newInstance(96*GB, 1));
+
assertEquals(
(int)(clusterResource.getMemory() * queue.getAbsoluteCapacity()),
queue.getMetrics().getAvailableMB()
@@ -271,18 +335,15 @@ public class TestApplicationLimits {
clusterResource = Resources.createResource(100 * 16 * GB);
queue = (LeafQueue)queues.get(A);
- expectedMaxActiveApps =
- Math.max(1,
- (int)Math.ceil(((float)clusterResource.getMemory() / (1*GB)) *
- csConf.
- getMaximumApplicationMasterResourcePerQueuePercent(
- queue.getQueuePath()) *
- queue.getAbsoluteMaximumCapacity()));
assertEquals((long) 0.5,
- (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(queue.getQueuePath()));
- assertEquals(expectedMaxActiveApps,
- queue.getMaximumActiveApplications());
+ (long) csConf.getMaximumApplicationMasterResourcePerQueuePercent(
+ queue.getQueuePath())
+ );
+
+ assertEquals(queue.getAMResourceLimit(), Resource.newInstance(800*GB, 1));
+ assertEquals(queue.getUserAMResourceLimit(),
+ Resource.newInstance(400*GB, 1));
// Change the per-queue max applications.
csConf.setInt(
@@ -308,10 +369,16 @@ public class TestApplicationLimits {
public void testActiveApplicationLimits() throws Exception {
final String user_0 = "user_0";
final String user_1 = "user_1";
+ final String user_2 = "user_2";
+
+ assertEquals(Resource.newInstance(16 * GB, 1), queue.getAMResourceLimit());
+ assertEquals(Resource.newInstance(8 * GB, 1),
+ queue.getUserAMResourceLimit());
int APPLICATION_ID = 0;
// Submit first application
- FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_0, user_0);
assertEquals(1, queue.getNumActiveApplications());
assertEquals(0, queue.getNumPendingApplications());
@@ -319,15 +386,17 @@ public class TestApplicationLimits {
assertEquals(0, queue.getNumPendingApplications(user_0));
// Submit second application
- FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_1, user_0);
assertEquals(2, queue.getNumActiveApplications());
assertEquals(0, queue.getNumPendingApplications());
assertEquals(2, queue.getNumActiveApplications(user_0));
assertEquals(0, queue.getNumPendingApplications(user_0));
- // Submit third application, should remain pending
- FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0);
+ // Submit third application, should remain pending due to user amlimit
+ FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_2, user_0);
assertEquals(2, queue.getNumActiveApplications());
assertEquals(1, queue.getNumPendingApplications());
@@ -342,18 +411,17 @@ public class TestApplicationLimits {
assertEquals(0, queue.getNumPendingApplications(user_0));
// Submit another one for user_0
- FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_3, user_0);
assertEquals(2, queue.getNumActiveApplications());
assertEquals(1, queue.getNumPendingApplications());
assertEquals(2, queue.getNumActiveApplications(user_0));
assertEquals(1, queue.getNumPendingApplications(user_0));
- // Change queue limit to be smaller so 2 users can fill it up
- doReturn(3).when(queue).getMaximumActiveApplications();
-
// Submit first app for user_1
- FiCaSchedulerApp app_4 = getMockApplication(APPLICATION_ID++, user_1);
+ FiCaSchedulerApp app_4 = getMockApplication(APPLICATION_ID++, user_1,
+ Resources.createResource(8 * GB, 0));
queue.submitApplicationAttempt(app_4, user_1);
assertEquals(3, queue.getNumActiveApplications());
assertEquals(1, queue.getNumPendingApplications());
@@ -362,15 +430,17 @@ public class TestApplicationLimits {
assertEquals(1, queue.getNumActiveApplications(user_1));
assertEquals(0, queue.getNumPendingApplications(user_1));
- // Submit second app for user_1, should block due to queue-limit
- FiCaSchedulerApp app_5 = getMockApplication(APPLICATION_ID++, user_1);
- queue.submitApplicationAttempt(app_5, user_1);
+ // Submit first app for user_2, should block due to queue amlimit
+ FiCaSchedulerApp app_5 = getMockApplication(APPLICATION_ID++, user_2,
+ Resources.createResource(8 * GB, 0));
+ queue.submitApplicationAttempt(app_5, user_2);
assertEquals(3, queue.getNumActiveApplications());
assertEquals(2, queue.getNumPendingApplications());
assertEquals(2, queue.getNumActiveApplications(user_0));
assertEquals(1, queue.getNumPendingApplications(user_0));
assertEquals(1, queue.getNumActiveApplications(user_1));
- assertEquals(1, queue.getNumPendingApplications(user_1));
+ assertEquals(0, queue.getNumPendingApplications(user_1));
+ assertEquals(1, queue.getNumPendingApplications(user_2));
// Now finish one app of user_1 so app_5 should be activated
queue.finishApplicationAttempt(app_4, A);
@@ -378,21 +448,22 @@ public class TestApplicationLimits {
assertEquals(1, queue.getNumPendingApplications());
assertEquals(2, queue.getNumActiveApplications(user_0));
assertEquals(1, queue.getNumPendingApplications(user_0));
- assertEquals(1, queue.getNumActiveApplications(user_1));
+ assertEquals(0, queue.getNumActiveApplications(user_1));
assertEquals(0, queue.getNumPendingApplications(user_1));
+ assertEquals(1, queue.getNumActiveApplications(user_2));
+ assertEquals(0, queue.getNumPendingApplications(user_2));
+
}
-
+
@Test
public void testActiveLimitsWithKilledApps() throws Exception {
final String user_0 = "user_0";
int APPLICATION_ID = 0;
- // set max active to 2
- doReturn(2).when(queue).getMaximumActiveApplications();
-
// Submit first application
- FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_0 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_0, user_0);
assertEquals(1, queue.getNumActiveApplications());
assertEquals(0, queue.getNumPendingApplications());
@@ -401,7 +472,8 @@ public class TestApplicationLimits {
assertTrue(queue.activeApplications.contains(app_0));
// Submit second application
- FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_1 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_1, user_0);
assertEquals(2, queue.getNumActiveApplications());
assertEquals(0, queue.getNumPendingApplications());
@@ -410,7 +482,8 @@ public class TestApplicationLimits {
assertTrue(queue.activeApplications.contains(app_1));
// Submit third application, should remain pending
- FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_2 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_2, user_0);
assertEquals(2, queue.getNumActiveApplications());
assertEquals(1, queue.getNumPendingApplications());
@@ -419,7 +492,8 @@ public class TestApplicationLimits {
assertTrue(queue.pendingApplications.contains(app_2));
// Submit fourth application, should remain pending
- FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0);
+ FiCaSchedulerApp app_3 = getMockApplication(APPLICATION_ID++, user_0,
+ Resources.createResource(4 * GB, 0));
queue.submitApplicationAttempt(app_3, user_0);
assertEquals(2, queue.getNumActiveApplications());
assertEquals(2, queue.getNumPendingApplications());
@@ -506,6 +580,18 @@ public class TestApplicationLimits {
RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
RMContext rmContext = TestUtils.getMockRMContext();
+ RMContext spyRMContext = spy(rmContext);
+
+ ConcurrentMap<ApplicationId, RMApp> spyApps =
+ spy(new ConcurrentHashMap<ApplicationId, RMApp>());
+ RMApp rmApp = mock(RMApp.class);
+ ResourceRequest amResourceRequest = mock(ResourceRequest.class);
+ Resource amResource = Resources.createResource(0, 0);
+ when(amResourceRequest.getCapability()).thenReturn(amResource);
+ when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
+ Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
+ when(spyRMContext.getRMApps()).thenReturn(spyApps);
+
Priority priority_1 = TestUtils.createMockPriority(1);
@@ -513,9 +599,9 @@ public class TestApplicationLimits {
// and check headroom
final ApplicationAttemptId appAttemptId_0_0 =
TestUtils.getMockApplicationAttemptId(0, 0);
- FiCaSchedulerApp app_0_0 =
- spy(new FiCaSchedulerApp(appAttemptId_0_0, user_0, queue,
- queue.getActiveUsersManager(), rmContext));
+ FiCaSchedulerApp app_0_0 = new FiCaSchedulerApp(
+ appAttemptId_0_0, user_0, queue,
+ queue.getActiveUsersManager(), spyRMContext);
queue.submitApplicationAttempt(app_0_0, user_0);
List<ResourceRequest> app_0_0_requests = new ArrayList<ResourceRequest>();
@@ -532,9 +618,9 @@ public class TestApplicationLimits {
// Submit second application from user_0, check headroom
final ApplicationAttemptId appAttemptId_0_1 =
TestUtils.getMockApplicationAttemptId(1, 0);
- FiCaSchedulerApp app_0_1 =
- spy(new FiCaSchedulerApp(appAttemptId_0_1, user_0, queue,
- queue.getActiveUsersManager(), rmContext));
+ FiCaSchedulerApp app_0_1 = new FiCaSchedulerApp(
+ appAttemptId_0_1, user_0, queue,
+ queue.getActiveUsersManager(), spyRMContext);
queue.submitApplicationAttempt(app_0_1, user_0);
List<ResourceRequest> app_0_1_requests = new ArrayList<ResourceRequest>();
@@ -551,9 +637,9 @@ public class TestApplicationLimits {
// Submit first application from user_1, check for new headroom
final ApplicationAttemptId appAttemptId_1_0 =
TestUtils.getMockApplicationAttemptId(2, 0);
- FiCaSchedulerApp app_1_0 =
- spy(new FiCaSchedulerApp(appAttemptId_1_0, user_1, queue,
- queue.getActiveUsersManager(), rmContext));
+ FiCaSchedulerApp app_1_0 = new FiCaSchedulerApp(
+ appAttemptId_1_0, user_1, queue,
+ queue.getActiveUsersManager(), spyRMContext);
queue.submitApplicationAttempt(app_1_0, user_1);
List<ResourceRequest> app_1_0_requests = new ArrayList<ResourceRequest>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index fb7bb2c..ead5719 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -101,6 +101,7 @@ public class TestLeafQueue {
RMContext rmContext;
RMContext spyRMContext;
+ ResourceRequest amResourceRequest;
CapacityScheduler cs;
CapacitySchedulerConfiguration csConf;
CapacitySchedulerContext csContext;
@@ -124,6 +125,10 @@ public class TestLeafQueue {
spy(new ConcurrentHashMap<ApplicationId, RMApp>());
RMApp rmApp = mock(RMApp.class);
when(rmApp.getRMAppAttempt((ApplicationAttemptId)Matchers.any())).thenReturn(null);
+ amResourceRequest = mock(ResourceRequest.class);
+ when(amResourceRequest.getCapability()).thenReturn(
+ Resources.createResource(0, 0));
+ when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
@@ -265,26 +270,37 @@ public class TestLeafQueue {
@Test
public void testInitializeQueue() throws Exception {
- final float epsilon = 1e-5f;
- //can add more sturdy test with 3-layer queues
- //once MAPREDUCE:3410 is resolved
- LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
- assertEquals(0.085, a.getCapacity(), epsilon);
- assertEquals(0.085, a.getAbsoluteCapacity(), epsilon);
- assertEquals(0.2, a.getMaximumCapacity(), epsilon);
- assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
+ final float epsilon = 1e-5f;
+ //can add more sturdy test with 3-layer queues
+ //once MAPREDUCE:3410 is resolved
+ LeafQueue a = stubLeafQueue((LeafQueue)queues.get(A));
+ assertEquals(0.085, a.getCapacity(), epsilon);
+ assertEquals(0.085, a.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.2, a.getMaximumCapacity(), epsilon);
+ assertEquals(0.2, a.getAbsoluteMaximumCapacity(), epsilon);
+
+ LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
+ assertEquals(0.80, b.getCapacity(), epsilon);
+ assertEquals(0.80, b.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.99, b.getMaximumCapacity(), epsilon);
+ assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
+
+ ParentQueue c = (ParentQueue)queues.get(C);
+ assertEquals(0.015, c.getCapacity(), epsilon);
+ assertEquals(0.015, c.getAbsoluteCapacity(), epsilon);
+ assertEquals(0.1, c.getMaximumCapacity(), epsilon);
+ assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
+
+ //Verify the value for getAMResourceLimit for queues with < .1 maxcap
+ Resource clusterResource = Resource.newInstance(50 * GB, 50);
- LeafQueue b = stubLeafQueue((LeafQueue)queues.get(B));
- assertEquals(0.80, b.getCapacity(), epsilon);
- assertEquals(0.80, b.getAbsoluteCapacity(), epsilon);
- assertEquals(0.99, b.getMaximumCapacity(), epsilon);
- assertEquals(0.99, b.getAbsoluteMaximumCapacity(), epsilon);
-
- ParentQueue c = (ParentQueue)queues.get(C);
- assertEquals(0.015, c.getCapacity(), epsilon);
- assertEquals(0.015, c.getAbsoluteCapacity(), epsilon);
- assertEquals(0.1, c.getMaximumCapacity(), epsilon);
- assertEquals(0.1, c.getAbsoluteMaximumCapacity(), epsilon);
+ a.updateClusterResource(clusterResource);
+ assertEquals(Resource.newInstance(1 * GB, 1),
+ a.getAMResourceLimit());
+
+ b.updateClusterResource(clusterResource);
+ assertEquals(Resource.newInstance(5 * GB, 1),
+ b.getAMResourceLimit());
}
@Test
@@ -679,7 +695,7 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 =
new FiCaSchedulerApp(appAttemptId_0, user_0, qb,
- qb.getActiveUsersManager(), rmContext);
+ qb.getActiveUsersManager(), spyRMContext);
qb.submitApplicationAttempt(app_0, user_0);
Priority u0Priority = TestUtils.createMockPriority(1);
app_0.updateResourceRequests(Collections.singletonList(
@@ -702,7 +718,7 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(2, 0);
FiCaSchedulerApp app_2 =
new FiCaSchedulerApp(appAttemptId_2, user_1, qb,
- qb.getActiveUsersManager(), rmContext);
+ qb.getActiveUsersManager(), spyRMContext);
Priority u1Priority = TestUtils.createMockPriority(2);
app_2.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 4*GB, 1, true,
@@ -736,12 +752,12 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 =
new FiCaSchedulerApp(appAttemptId_1, user_0, qb,
- qb.getActiveUsersManager(), rmContext);
+ qb.getActiveUsersManager(), spyRMContext);
final ApplicationAttemptId appAttemptId_3 =
TestUtils.getMockApplicationAttemptId(3, 0);
FiCaSchedulerApp app_3 =
new FiCaSchedulerApp(appAttemptId_3, user_1, qb,
- qb.getActiveUsersManager(), rmContext);
+ qb.getActiveUsersManager(), spyRMContext);
app_1.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 2*GB, 1, true,
u0Priority, recordFactory)));
@@ -764,7 +780,7 @@ public class TestLeafQueue {
TestUtils.getMockApplicationAttemptId(4, 0);
FiCaSchedulerApp app_4 =
new FiCaSchedulerApp(appAttemptId_4, user_0, qb,
- qb.getActiveUsersManager(), rmContext);
+ qb.getActiveUsersManager(), spyRMContext);
qb.submitApplicationAttempt(app_4, user_0);
app_4.updateResourceRequests(Collections.singletonList(
TestUtils.createResourceRequest(ResourceRequest.ANY, 6*GB, 1, true,
@@ -980,7 +996,6 @@ public class TestLeafQueue {
assertEquals(0*GB, app_1.getHeadroom().getMemory());
// Check headroom for app_2
- LOG.info("here");
app_1.updateResourceRequests(Collections.singletonList( // unset
TestUtils.createResourceRequest(ResourceRequest.ANY, 1*GB, 0, true,
priority, recordFactory)));
@@ -1904,6 +1919,9 @@ public class TestLeafQueue {
// Users
final String user_e = "user_e";
+
+ when(amResourceRequest.getCapability()).thenReturn(
+ Resources.createResource(1 * GB, 0));
// Submit applications
final ApplicationAttemptId appAttemptId_0 =
@@ -1942,7 +1960,7 @@ public class TestLeafQueue {
newQueues, queues,
TestUtils.spyHook);
queues = newQueues;
- root.reinitialize(newRoot, cs.getClusterResource());
+ root.reinitialize(newRoot, csContext.getClusterResource());
// after reinitialization
assertEquals(3, e.activeApplications.size());
@@ -1982,6 +2000,9 @@ public class TestLeafQueue {
// Users
final String user_e = "user_e";
+
+ when(amResourceRequest.getCapability()).thenReturn(
+ Resources.createResource(1 * GB, 0));
// Submit applications
final ApplicationAttemptId appAttemptId_0 =
@@ -2291,20 +2312,20 @@ public class TestLeafQueue {
csConf.setCapacity(CapacitySchedulerConfiguration.ROOT + "." + A, 80);
LeafQueue a = new LeafQueue(csContext, A, root, null);
assertEquals(0.1f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(160, a.getMaximumActiveApplications());
+ assertEquals(a.getAMResourceLimit(), Resources.createResource(160 * GB, 1));
csConf.setFloat(CapacitySchedulerConfiguration.
MAXIMUM_APPLICATION_MASTERS_RESOURCE_PERCENT, 0.2f);
LeafQueue newA = new LeafQueue(csContext, A, root, null);
a.reinitialize(newA, clusterResource);
assertEquals(0.2f, a.getMaxAMResourcePerQueuePercent(), 1e-3f);
- assertEquals(320, a.getMaximumActiveApplications());
+ assertEquals(a.getAMResourceLimit(), Resources.createResource(320 * GB, 1));
Resource newClusterResource = Resources.createResource(100 * 20 * GB,
100 * 32);
a.updateClusterResource(newClusterResource);
// 100 * 20 * 0.2 = 400
- assertEquals(400, a.getMaximumActiveApplications());
+ assertEquals(a.getAMResourceLimit(), Resources.createResource(400 * GB, 1));
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f1b35ffd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 2a49545..985609e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -77,6 +77,7 @@ public class TestReservations {
.getRecordFactory(null);
RMContext rmContext;
+ RMContext spyRMContext;
CapacityScheduler cs;
// CapacitySchedulerConfiguration csConf;
CapacitySchedulerContext csContext;
@@ -132,7 +133,10 @@ public class TestReservations {
root = CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues, TestUtils.spyHook);
- cs.setRMContext(rmContext);
+ spyRMContext = spy(rmContext);
+ when(spyRMContext.getScheduler()).thenReturn(cs);
+
+ cs.setRMContext(spyRMContext);
cs.init(csConf);
cs.start();
}
@@ -212,14 +216,14 @@ public class TestReservations {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -361,14 +365,14 @@ public class TestReservations {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -506,14 +510,14 @@ public class TestReservations {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -618,7 +622,7 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
String host_0 = "host_0";
FiCaSchedulerNode node_0 = TestUtils.getMockNode(host_0, DEFAULT_RACK, 0,
@@ -685,7 +689,7 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
LeafQueue a = stubLeafQueue((LeafQueue) queues.get(A));
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
String host_1 = "host_1";
FiCaSchedulerNode node_1 = TestUtils.getMockNode(host_1, DEFAULT_RACK, 0,
@@ -742,14 +746,14 @@ public class TestReservations {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -916,14 +920,14 @@ public class TestReservations {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -1042,14 +1046,14 @@ public class TestReservations {
final ApplicationAttemptId appAttemptId_0 = TestUtils
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_0, user_0);
final ApplicationAttemptId appAttemptId_1 = TestUtils
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
- mock(ActiveUsersManager.class), rmContext);
+ mock(ActiveUsersManager.class), spyRMContext);
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
[28/50] [abbrv] hadoop git commit: Preparing for release 2.6.1: mvn
versions:set -DnewVersion=2.6.1
Posted by sj...@apache.org.
Preparing for release 2.6.1: mvn versions:set -DnewVersion=2.6.1
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3462a00d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3462a00d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3462a00d
Branch: refs/heads/branch-2.6
Commit: 3462a00dd2792cf0264513cf0ed50536ab15fe28
Parents: cfb012a
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 9 15:29:57 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 9 15:29:57 2015 -0700
----------------------------------------------------------------------
hadoop-assemblies/pom.xml | 4 ++--
hadoop-client/pom.xml | 4 ++--
hadoop-common-project/hadoop-annotations/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth-examples/pom.xml | 4 ++--
hadoop-common-project/hadoop-auth/pom.xml | 4 ++--
hadoop-common-project/hadoop-common/pom.xml | 4 ++--
hadoop-common-project/hadoop-kms/pom.xml | 4 ++--
hadoop-common-project/hadoop-minikdc/pom.xml | 4 ++--
hadoop-common-project/hadoop-nfs/pom.xml | 4 ++--
hadoop-common-project/pom.xml | 4 ++--
hadoop-dist/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 4 ++--
hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml | 4 ++--
hadoop-hdfs-project/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml | 4 ++--
.../hadoop-mapreduce-client-common/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml | 4 ++--
.../hadoop-mapreduce-client-hs-plugins/pom.xml | 4 ++--
.../hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml | 4 ++--
.../hadoop-mapreduce-client-jobclient/pom.xml | 4 ++--
.../hadoop-mapreduce-client-shuffle/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml | 4 ++--
hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml | 4 ++--
hadoop-mapreduce-project/pom.xml | 4 ++--
hadoop-maven-plugins/pom.xml | 2 +-
hadoop-minicluster/pom.xml | 4 ++--
hadoop-project-dist/pom.xml | 4 ++--
hadoop-project/pom.xml | 4 ++--
hadoop-tools/hadoop-ant/pom.xml | 4 ++--
hadoop-tools/hadoop-archives/pom.xml | 4 ++--
hadoop-tools/hadoop-aws/pom.xml | 4 ++--
hadoop-tools/hadoop-datajoin/pom.xml | 4 ++--
hadoop-tools/hadoop-distcp/pom.xml | 4 ++--
hadoop-tools/hadoop-extras/pom.xml | 4 ++--
hadoop-tools/hadoop-gridmix/pom.xml | 4 ++--
hadoop-tools/hadoop-openstack/pom.xml | 4 ++--
hadoop-tools/hadoop-pipes/pom.xml | 4 ++--
hadoop-tools/hadoop-rumen/pom.xml | 4 ++--
hadoop-tools/hadoop-sls/pom.xml | 4 ++--
hadoop-tools/hadoop-streaming/pom.xml | 4 ++--
hadoop-tools/hadoop-tools-dist/pom.xml | 4 ++--
hadoop-tools/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml | 4 ++--
.../hadoop-yarn-applications-distributedshell/pom.xml | 4 ++--
.../hadoop-yarn-applications-unmanaged-am-launcher/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml | 4 ++--
.../hadoop-yarn-server-applicationhistoryservice/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-common/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml | 4 ++--
.../hadoop-yarn-server-resourcemanager/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml | 4 ++--
.../hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml | 4 ++--
hadoop-yarn-project/hadoop-yarn/pom.xml | 4 ++--
hadoop-yarn-project/pom.xml | 4 ++--
pom.xml | 2 +-
62 files changed, 122 insertions(+), 122 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-assemblies/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/pom.xml b/hadoop-assemblies/pom.xml
index 8fb4ba2..323ed12 100644
--- a/hadoop-assemblies/pom.xml
+++ b/hadoop-assemblies/pom.xml
@@ -23,12 +23,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-assemblies</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>Apache Hadoop Assemblies</name>
<description>Apache Hadoop Assemblies</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-client/pom.xml b/hadoop-client/pom.xml
index e3f7fd9..ad66680 100644
--- a/hadoop-client/pom.xml
+++ b/hadoop-client/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>jar</packaging>
<description>Apache Hadoop Client</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-annotations/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-annotations/pom.xml b/hadoop-common-project/hadoop-annotations/pom.xml
index ed2a47b..e501c2b 100644
--- a/hadoop-common-project/hadoop-annotations/pom.xml
+++ b/hadoop-common-project/hadoop-annotations/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-annotations</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Annotations</description>
<name>Apache Hadoop Annotations</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-auth-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth-examples/pom.xml b/hadoop-common-project/hadoop-auth-examples/pom.xml
index 71a7fd4..40449e8 100644
--- a/hadoop-common-project/hadoop-auth-examples/pom.xml
+++ b/hadoop-common-project/hadoop-auth-examples/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth-examples</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>war</packaging>
<name>Apache Hadoop Auth Examples</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-auth/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/pom.xml b/hadoop-common-project/hadoop-auth/pom.xml
index c802186..9e4bea9 100644
--- a/hadoop-common-project/hadoop-auth/pom.xml
+++ b/hadoop-common-project/hadoop-auth/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-auth</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>jar</packaging>
<name>Apache Hadoop Auth</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/pom.xml b/hadoop-common-project/hadoop-common/pom.xml
index 4b72880..59be1ab 100644
--- a/hadoop-common-project/hadoop-common/pom.xml
+++ b/hadoop-common-project/hadoop-common/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Common</description>
<name>Apache Hadoop Common</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-kms/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/pom.xml b/hadoop-common-project/hadoop-kms/pom.xml
index 7962827..a54ab7a 100644
--- a/hadoop-common-project/hadoop-kms/pom.xml
+++ b/hadoop-common-project/hadoop-kms/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-kms</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>war</packaging>
<name>Apache Hadoop KMS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-minikdc/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-minikdc/pom.xml b/hadoop-common-project/hadoop-minikdc/pom.xml
index ddaa5b1..5dc0d2c 100644
--- a/hadoop-common-project/hadoop-minikdc/pom.xml
+++ b/hadoop-common-project/hadoop-minikdc/pom.xml
@@ -18,13 +18,13 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minikdc</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop MiniKDC</description>
<name>Apache Hadoop MiniKDC</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/hadoop-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-nfs/pom.xml b/hadoop-common-project/hadoop-nfs/pom.xml
index 9549dd5..1924e5a 100644
--- a/hadoop-common-project/hadoop-nfs/pom.xml
+++ b/hadoop-common-project/hadoop-nfs/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-nfs</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>jar</packaging>
<name>Apache Hadoop NFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-common-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/pom.xml b/hadoop-common-project/pom.xml
index 1bf343f..926cb0d 100644
--- a/hadoop-common-project/pom.xml
+++ b/hadoop-common-project/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Common Project</description>
<name>Apache Hadoop Common Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-dist/pom.xml b/hadoop-dist/pom.xml
index 6b07b7b..d9b40e4 100644
--- a/hadoop-dist/pom.xml
+++ b/hadoop-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Distribution</description>
<name>Apache Hadoop Distribution</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
index e0c4b03..abdef37 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml
@@ -22,12 +22,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-httpfs</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>war</packaging>
<name>Apache Hadoop HttpFS</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
index e76cef5..b377521 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-nfs</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop HDFS-NFS</description>
<name>Apache Hadoop HDFS-NFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-hdfs-project/hadoop-hdfs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
index bf5f928..8bafaf1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop HDFS</description>
<name>Apache Hadoop HDFS</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
index 305970a..c8004c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/pom.xml
@@ -20,13 +20,13 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../../../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop.contrib</groupId>
<artifactId>hadoop-hdfs-bkjournal</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop HDFS BookKeeper Journal</description>
<name>Apache Hadoop HDFS BookKeeper Journal</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-hdfs-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/pom.xml b/hadoop-hdfs-project/pom.xml
index 025416c..37d7702 100644
--- a/hadoop-hdfs-project/pom.xml
+++ b/hadoop-hdfs-project/pom.xml
@@ -20,12 +20,12 @@ http://maven.apache.org/xsd/maven-4.0.0.xsd">
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-hdfs-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop HDFS Project</description>
<name>Apache Hadoop HDFS Project</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index f19b67c..cf059e86 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-app</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-app</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index 25a168f..0c64fcb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-common</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
index 869d017..78156dc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-core</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
index e5c2b17..de34ce0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs-plugins/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs-plugins</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-hs-plugins</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
index c70b3eb..32d693d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-hs</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-hs</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
index caaac06..2f6bf3a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-jobclient</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
index 5705a3f..be7c522 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-mapreduce-client</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-shuffle</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client-shuffle</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
index 4a7b1f3..8627c89 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-mapreduce-client</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
index 44f7c08..fbae3c0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-examples</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop MapReduce Examples</description>
<name>Apache Hadoop MapReduce Examples</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-mapreduce-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/pom.xml b/hadoop-mapreduce-project/pom.xml
index 439c9eb..683256d 100644
--- a/hadoop-mapreduce-project/pom.xml
+++ b/hadoop-mapreduce-project/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>pom</packaging>
<name>hadoop-mapreduce</name>
<url>http://hadoop.apache.org/mapreduce/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-maven-plugins/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-maven-plugins/pom.xml b/hadoop-maven-plugins/pom.xml
index dee790a..1bbdbfd 100644
--- a/hadoop-maven-plugins/pom.xml
+++ b/hadoop-maven-plugins/pom.xml
@@ -19,7 +19,7 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-minicluster/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-minicluster/pom.xml b/hadoop-minicluster/pom.xml
index 5919297..dccb3d1 100644
--- a/hadoop-minicluster/pom.xml
+++ b/hadoop-minicluster/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-minicluster</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>jar</packaging>
<description>Apache Hadoop Mini-Cluster</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-project-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project-dist/pom.xml b/hadoop-project-dist/pom.xml
index 8b8c676..e090c69 100644
--- a/hadoop-project-dist/pom.xml
+++ b/hadoop-project-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Project Dist POM</description>
<name>Apache Hadoop Project Dist POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8a3f82c..da78853 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -20,11 +20,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Project POM</description>
<name>Apache Hadoop Project POM</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-ant/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-ant/pom.xml b/hadoop-tools/hadoop-ant/pom.xml
index 3300422..e693347 100644
--- a/hadoop-tools/hadoop-ant/pom.xml
+++ b/hadoop-tools/hadoop-ant/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-ant</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Ant Tasks</description>
<name>Apache Hadoop Ant Tasks</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-archives/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archives/pom.xml b/hadoop-tools/hadoop-archives/pom.xml
index b2b8b57..2792efb 100644
--- a/hadoop-tools/hadoop-archives/pom.xml
+++ b/hadoop-tools/hadoop-archives/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-archives</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Archives</description>
<name>Apache Hadoop Archives</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 1304b76..2420d30 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-aws</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>Apache Hadoop Amazon Web Services support</name>
<description>
This module contains code to support integration with Amazon Web Services.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-datajoin/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-datajoin/pom.xml b/hadoop-tools/hadoop-datajoin/pom.xml
index 87e6b41..d107993 100644
--- a/hadoop-tools/hadoop-datajoin/pom.xml
+++ b/hadoop-tools/hadoop-datajoin/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-datajoin</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Data Join</description>
<name>Apache Hadoop Data Join</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-distcp/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/pom.xml b/hadoop-tools/hadoop-distcp/pom.xml
index 2fc52c2..2e577fc 100644
--- a/hadoop-tools/hadoop-distcp/pom.xml
+++ b/hadoop-tools/hadoop-distcp/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-distcp</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Distributed Copy</description>
<name>Apache Hadoop Distributed Copy</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-extras/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/pom.xml b/hadoop-tools/hadoop-extras/pom.xml
index 0f73389..d7df4b0 100644
--- a/hadoop-tools/hadoop-extras/pom.xml
+++ b/hadoop-tools/hadoop-extras/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-extras</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Extras</description>
<name>Apache Hadoop Extras</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-gridmix/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/pom.xml b/hadoop-tools/hadoop-gridmix/pom.xml
index fff7a27..3202537 100644
--- a/hadoop-tools/hadoop-gridmix/pom.xml
+++ b/hadoop-tools/hadoop-gridmix/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-gridmix</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Gridmix</description>
<name>Apache Hadoop Gridmix</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-openstack/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/pom.xml b/hadoop-tools/hadoop-openstack/pom.xml
index eccc513..f2c76be 100644
--- a/hadoop-tools/hadoop-openstack/pom.xml
+++ b/hadoop-tools/hadoop-openstack/pom.xml
@@ -19,11 +19,11 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<artifactId>hadoop-openstack</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>Apache Hadoop OpenStack support</name>
<description>
This module contains code to support integration with OpenStack.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-pipes/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/pom.xml b/hadoop-tools/hadoop-pipes/pom.xml
index 03c4d21..380f1e3 100644
--- a/hadoop-tools/hadoop-pipes/pom.xml
+++ b/hadoop-tools/hadoop-pipes/pom.xml
@@ -17,12 +17,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-pipes</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Pipes</description>
<name>Apache Hadoop Pipes</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-rumen/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/pom.xml b/hadoop-tools/hadoop-rumen/pom.xml
index f82c557..a045373 100644
--- a/hadoop-tools/hadoop-rumen/pom.xml
+++ b/hadoop-tools/hadoop-rumen/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-rumen</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Rumen</description>
<name>Apache Hadoop Rumen</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-sls/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/pom.xml b/hadoop-tools/hadoop-sls/pom.xml
index c106970..4420234 100644
--- a/hadoop-tools/hadoop-sls/pom.xml
+++ b/hadoop-tools/hadoop-sls/pom.xml
@@ -19,12 +19,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-sls</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Scheduler Load Simulator</description>
<name>Apache Hadoop Scheduler Load Simulator</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-streaming/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/pom.xml b/hadoop-tools/hadoop-streaming/pom.xml
index c7d3bc1..863df08 100644
--- a/hadoop-tools/hadoop-streaming/pom.xml
+++ b/hadoop-tools/hadoop-streaming/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-streaming</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop MapReduce Streaming</description>
<name>Apache Hadoop MapReduce Streaming</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/hadoop-tools-dist/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-tools-dist/pom.xml b/hadoop-tools/hadoop-tools-dist/pom.xml
index 5a2a8dc..a3d2c0f 100644
--- a/hadoop-tools/hadoop-tools-dist/pom.xml
+++ b/hadoop-tools/hadoop-tools-dist/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project-dist</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools-dist</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Tools Dist</description>
<name>Apache Hadoop Tools Dist</name>
<packaging>jar</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-tools/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml
index 094c761..c835c4e 100644
--- a/hadoop-tools/pom.xml
+++ b/hadoop-tools/pom.xml
@@ -20,12 +20,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-tools</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Tools</description>
<name>Apache Hadoop Tools</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
index fd880c3..5919d17 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-api</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-api</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
index 2d51802..36572cb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-distributedshell</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-applications-distributedshell</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
index 3bf2256..76eef6c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-unmanaged-am-launcher/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-applications</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications-unmanaged-am-launcher</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-applications-unmanaged-am-launcher</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
index 5c4b9f0..ace6a23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-applications</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-applications</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
index 730113d..8dd7597 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/pom.xml
@@ -17,11 +17,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-client</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-client</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
index 2987d61..679942e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-common</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
index 1faed3e..ebaa1d3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<artifactId>hadoop-yarn-registry</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-registry</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
index 2623212..7cc1877 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/pom.xml
@@ -22,12 +22,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-applicationhistoryservice</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server-applicationhistoryservice</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 0da564e..5c4ec1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-common</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server-common</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
index 9b9a245..2ed0fcc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-nodemanager</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server-nodemanager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index c516031..58842ed 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-resourcemanager</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server-resourcemanager</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
index be880c0..623bdd7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml
@@ -19,11 +19,11 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-tests</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server-tests</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
index d782212..0c6d1f5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn-server</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server-web-proxy</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server-web-proxy</name>
<properties>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
index 354ee56..27e5830 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-server</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-server</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
index bb5c305..b156f9e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml
@@ -19,12 +19,12 @@
<parent>
<artifactId>hadoop-yarn</artifactId>
<groupId>org.apache.hadoop</groupId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
</parent>
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-site</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<name>hadoop-yarn-site</name>
<packaging>pom</packaging>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/hadoop-yarn/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/pom.xml b/hadoop-yarn-project/hadoop-yarn/pom.xml
index ee1c0c5..296f9be 100644
--- a/hadoop-yarn-project/hadoop-yarn/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/pom.xml
@@ -16,12 +16,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>pom</packaging>
<name>hadoop-yarn</name>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/hadoop-yarn-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/pom.xml b/hadoop-yarn-project/pom.xml
index 655c85c..8508bae 100644
--- a/hadoop-yarn-project/pom.xml
+++ b/hadoop-yarn-project/pom.xml
@@ -18,12 +18,12 @@
<parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<relativePath>../hadoop-project</relativePath>
</parent>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-yarn-project</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<packaging>pom</packaging>
<name>hadoop-yarn-project</name>
<url>http://hadoop.apache.org/yarn/</url>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3462a00d/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 037a59e..ce20d86 100644
--- a/pom.xml
+++ b/pom.xml
@@ -18,7 +18,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-main</artifactId>
- <version>2.6.1-SNAPSHOT</version>
+ <version>2.6.1</version>
<description>Apache Hadoop Main</description>
<name>Apache Hadoop Main</name>
<packaging>pom</packaging>
[31/50] [abbrv] hadoop git commit: Set the release date for 2.6.1-RC0
Posted by sj...@apache.org.
Set the release date for 2.6.1-RC0
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/075d899d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/075d899d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/075d899d
Branch: refs/heads/branch-2.6
Commit: 075d899d10e73a56246757caace41473ebd5bc06
Parents: c86d9bd
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Wed Sep 9 15:39:02 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Wed Sep 9 15:39:02 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +-
hadoop-mapreduce-project/CHANGES.txt | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +-
4 files changed, 4 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/075d899d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 7f868be..bfe6a96 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop Change Log
-Release 2.6.1 - UNRELEASED
+Release 2.6.1 - 2015-09-09
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/075d899d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2eeb79f..8800a83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop HDFS Change Log
-Release 2.6.1 - UNRELEASED
+Release 2.6.1 - 2015-09-09
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/075d899d/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 497d208..0fa1e45 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop MapReduce Change Log
-Release 2.6.1 - UNRELEASED
+Release 2.6.1 - 2015-09-09
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/075d899d/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7ff9d72..b6bafb8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1,6 +1,6 @@
Hadoop YARN Change Log
-Release 2.6.1 - UNRELEASED
+Release 2.6.1 - 2015-09-09
INCOMPATIBLE CHANGES
[22/50] [abbrv] hadoop git commit: YARN-3978. Configurably turn off
the saving of container info in Generic AHS (Eric Payne via jeagles)
Posted by sj...@apache.org.
YARN-3978. Configurably turn off the saving of container info in Generic AHS (Eric Payne via jeagles)
(cherry picked from commit 3cd02b95224e9d43fd63a4ef9ac5c44f113f710d)
(cherry picked from commit 899df5bce03ea4f94487e48c1d38bd30ae10c26f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6ed2486c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6ed2486c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6ed2486c
Branch: refs/heads/branch-2.6
Commit: 6ed2486c7e15ffcb19c5a70270f060db76b1d678
Parents: 92742b4
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Mon Aug 3 10:38:05 2015 -0500
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 22:57:34 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 9 +++
.../hadoop/yarn/server/webapp/AppBlock.java | 12 +--
.../rmcontainer/RMContainerImpl.java | 43 ++++++++++-
.../resourcemanager/TestClientRMService.java | 1 +
.../rmcontainer/TestRMContainerImpl.java | 79 +++++++++++++++++++-
.../scheduler/capacity/TestChildQueueOrder.java | 1 +
.../scheduler/capacity/TestLeafQueue.java | 3 +
.../scheduler/capacity/TestReservations.java | 6 +-
.../scheduler/fifo/TestFifoScheduler.java | 2 +
10 files changed, 147 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f2b8c07..9fae242 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -24,6 +24,9 @@ Release 2.6.1 - UNRELEASED
YARN-2301. Improved yarn container command. (Naganarasimha G R via jianhe)
+ YARN-3978. Configurably turn off the saving of container info in Generic AHS
+ (Eric Payne via jeagles)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index bc89ed0..39e12d4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1234,6 +1234,15 @@ public class YarnConfiguration extends Configuration {
public static final String APPLICATION_HISTORY_STORE =
APPLICATION_HISTORY_PREFIX + "store-class";
+ /** Save container meta-info in the application history store. */
+ @Private
+ public static final String
+ APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO =
+ APPLICATION_HISTORY_PREFIX + "save-non-am-container-meta-info";
+ @Private
+ public static final boolean
+ DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO = true;
+
/** URI for FileSystemApplicationHistoryStore */
@Private
public static final String FS_APPLICATION_HISTORY_STORE_URI =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index 37b60bb..d52975b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -269,11 +269,13 @@ public class AppBlock extends HtmlBlock {
@Override
public ContainerReport run() throws Exception {
ContainerReport report = null;
- try {
- report = appBaseProt.getContainerReport(request)
- .getContainerReport();
- } catch (ContainerNotFoundException ex) {
- LOG.warn(ex.getMessage());
+ if (request.getContainerId() != null) {
+ try {
+ report = appBaseProt.getContainerReport(request)
+ .getContainerReport();
+ } catch (ContainerNotFoundException ex) {
+ LOG.warn(ex.getMessage());
+ }
}
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index e37d8fd..b69968f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -38,6 +38,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
@@ -164,6 +165,8 @@ public class RMContainerImpl implements RMContainer {
private boolean isAMContainer;
private List<ResourceRequest> resourceRequests;
+ private boolean saveNonAMContainerMetaInfo;
+
public RMContainerImpl(Container container,
ApplicationAttemptId appAttemptId, NodeId nodeId, String user,
RMContext rmContext) {
@@ -191,9 +194,21 @@ public class RMContainerImpl implements RMContainer {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
+ saveNonAMContainerMetaInfo = rmContext.getYarnConfiguration().getBoolean(
+ YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
+ YarnConfiguration
+ .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+
rmContext.getRMApplicationHistoryWriter().containerStarted(this);
- rmContext.getSystemMetricsPublisher().containerCreated(
- this, this.creationTime);
+
+ // If saveNonAMContainerMetaInfo is true, store system metrics for all
+ // containers. If false, and if this container is marked as the AM, metrics
+ // will still be published for this container, but that calculation happens
+ // later.
+ if (saveNonAMContainerMetaInfo) {
+ rmContext.getSystemMetricsPublisher().containerCreated(
+ this, this.creationTime);
+ }
}
@Override
@@ -366,6 +381,15 @@ public class RMContainerImpl implements RMContainer {
} finally {
writeLock.unlock();
}
+
+ // Even if saveNonAMContainerMetaInfo is not true, the AM container's system
+ // metrics still need to be saved so that the AM's logs can be accessed.
+ // This call to getSystemMetricsPublisher().containerCreated() is mutually
+ // exclusive with the one in the RMContainerImpl constructor.
+ if (!saveNonAMContainerMetaInfo && this.isAMContainer) {
+ rmContext.getSystemMetricsPublisher().containerCreated(
+ this, this.creationTime);
+ }
}
@Override
@@ -505,8 +529,19 @@ public class RMContainerImpl implements RMContainer {
container.rmContext.getRMApplicationHistoryWriter().containerFinished(
container);
- container.rmContext.getSystemMetricsPublisher().containerFinished(
- container, container.finishTime);
+
+ boolean saveNonAMContainerMetaInfo =
+ container.rmContext.getYarnConfiguration().getBoolean(
+ YarnConfiguration
+ .APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
+ YarnConfiguration
+ .DEFAULT_APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO);
+
+ if (saveNonAMContainerMetaInfo || container.isAMContainer()) {
+ container.rmContext.getSystemMetricsPublisher().containerFinished(
+ container, container.finishTime);
+ }
+
}
private static void updateAttemptMetrics(RMContainerImpl container) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
index a344e9a..9b12e04 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestClientRMService.java
@@ -1139,6 +1139,7 @@ public class TestClientRMService {
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
ConcurrentHashMap<ApplicationId, RMApp> apps = getRMApps(rmContext,
yarnScheduler);
when(rmContext.getRMApps()).thenReturn(apps);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
index 21aba3b..e4e2049 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
@@ -19,12 +19,14 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmcontainer;
import static org.junit.Assert.assertEquals;
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.anyLong;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -270,4 +272,77 @@ public class TestRMContainerImpl {
Assert.assertNull(scheduler.getRMContainer(containerId2)
.getResourceRequests());
}
+
+ @Test (timeout = 180000)
+ public void testStoreAllContainerMetrics() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+ MockRM rm1 = new MockRM(conf);
+
+ SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
+ rm1.getRMContext().setSystemMetricsPublisher(publisher);
+
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
+ RMApp app1 = rm1.submitApp(1024);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING);
+
+ // request a container.
+ am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
+ ContainerId containerId2 = ContainerId.newContainerId(
+ am1.getApplicationAttemptId(), 2);
+ rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
+ am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
+ .getAllocatedContainers();
+ rm1.waitForState(nm1, containerId2, RMContainerState.ACQUIRED);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.COMPLETE);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ rm1.waitForState(nm1, containerId2, RMContainerState.COMPLETED);
+ rm1.stop();
+
+ // RMContainer should be publishing system metrics for all containers.
+ // Since there is 1 AM container and 1 non-AM container, there should be 2
+ // container created events and 2 container finished events.
+ verify(publisher, times(2)).containerCreated(any(RMContainer.class), anyLong());
+ verify(publisher, times(2)).containerFinished(any(RMContainer.class), anyLong());
+ }
+
+ @Test (timeout = 180000)
+ public void testStoreOnlyAMContainerMetrics() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
+ conf.setBoolean(
+ YarnConfiguration.APPLICATION_HISTORY_SAVE_NON_AM_CONTAINER_META_INFO,
+ false);
+ MockRM rm1 = new MockRM(conf);
+
+ SystemMetricsPublisher publisher = mock(SystemMetricsPublisher.class);
+ rm1.getRMContext().setSystemMetricsPublisher(publisher);
+
+ rm1.start();
+ MockNM nm1 = rm1.registerNode("unknownhost:1234", 8000);
+ RMApp app1 = rm1.submitApp(1024);
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.RUNNING);
+
+ // request a container.
+ am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
+ ContainerId containerId2 = ContainerId.newContainerId(
+ am1.getApplicationAttemptId(), 2);
+ rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
+ am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>())
+ .getAllocatedContainers();
+ rm1.waitForState(nm1, containerId2, RMContainerState.ACQUIRED);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 2, ContainerState.COMPLETE);
+ nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
+ rm1.waitForState(nm1, containerId2, RMContainerState.COMPLETED);
+ rm1.stop();
+
+ // RMContainer should be publishing system metrics only for AM container.
+ verify(publisher, times(1)).containerCreated(any(RMContainer.class), anyLong());
+ verify(publisher, times(1)).containerFinished(any(RMContainer.class), anyLong());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
index af58a43..57ac609 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestChildQueueOrder.java
@@ -257,6 +257,7 @@ public class TestChildQueueOrder {
when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app_0.getApplicationId(), 1);
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index ead5719..736f942 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -171,6 +171,9 @@ public class TestLeafQueue {
cs.setRMContext(spyRMContext);
cs.init(csConf);
cs.start();
+
+ when(spyRMContext.getYarnConfiguration())
+ .thenReturn(new YarnConfiguration());
}
private static final String A = "a";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 985609e..c651018 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -135,7 +135,9 @@ public class TestReservations {
spyRMContext = spy(rmContext);
when(spyRMContext.getScheduler()).thenReturn(cs);
-
+ when(spyRMContext.getYarnConfiguration())
+ .thenReturn(new YarnConfiguration());
+
cs.setRMContext(spyRMContext);
cs.init(csConf);
cs.start();
@@ -645,6 +647,7 @@ public class TestReservations {
when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app_0.getApplicationId(), 1);
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
@@ -709,6 +712,7 @@ public class TestReservations {
when(rmContext.getDispatcher()).thenReturn(drainDispatcher);
when(rmContext.getRMApplicationHistoryWriter()).thenReturn(writer);
when(rmContext.getSystemMetricsPublisher()).thenReturn(publisher);
+ when(rmContext.getYarnConfiguration()).thenReturn(new YarnConfiguration());
ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(
app_0.getApplicationId(), 1);
ContainerId containerId = BuilderUtils.newContainerId(appAttemptId, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6ed2486c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
index 3918bf7..7f9a817 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
@@ -197,6 +197,7 @@ public class TestFifoScheduler {
scheduler);
((RMContextImpl) rmContext).setSystemMetricsPublisher(
mock(SystemMetricsPublisher.class));
+ ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
scheduler.setRMContext(rmContext);
scheduler.init(conf);
@@ -275,6 +276,7 @@ public class TestFifoScheduler {
scheduler);
((RMContextImpl) rmContext).setSystemMetricsPublisher(
mock(SystemMetricsPublisher.class));
+ ((RMContextImpl) rmContext).setYarnConfiguration(new YarnConfiguration());
scheduler.setRMContext(rmContext);
scheduler.init(conf);
[25/50] [abbrv] hadoop git commit: HDFS-8863. The remaining space
check in BlockPlacementPolicyDefault is flawed. (Kihwal Lee via yliu)
Posted by sj...@apache.org.
HDFS-8863. The remaining space check in BlockPlacementPolicyDefault is flawed. (Kihwal Lee via yliu)
(cherry picked from commit 146db49f7ff0dec82cd51f366311030404b770d7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0adcfe6c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0adcfe6c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0adcfe6c
Branch: refs/heads/branch-2.6
Commit: 0adcfe6c7e456fd790ddfafdaf7c484437570b9b
Parents: 2b526ba
Author: yliu <yl...@apache.org>
Authored: Thu Aug 20 20:46:45 2015 +0800
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 22:57:36 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../BlockPlacementPolicyDefault.java | 3 +-
.../blockmanagement/DatanodeDescriptor.java | 23 ++++--
.../blockmanagement/TestReplicationPolicy.java | 78 +++++++++++++++-----
4 files changed, 81 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adcfe6c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 9bc7974..2234da3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -172,6 +172,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7470. SecondaryNameNode need twice memory when calling
reloadFromImageFile. (zhaoyunjiong via cnauroth)
+ HDFS-8863. The remaining space check in BlockPlacementPolicyDefault is
+ flawed. (Kihwal Lee via yliu)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adcfe6c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 5b02384..19cb701 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -780,7 +780,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
final long requiredSize = blockSize * HdfsConstants.MIN_BLOCKS_FOR_WRITE;
final long scheduledSize = blockSize * node.getBlocksScheduled(storage.getStorageType());
- final long remaining = node.getRemaining(storage.getStorageType());
+ final long remaining =
+ node.getRemaining(storage.getStorageType(), requiredSize);
if (requiredSize > remaining - scheduledSize) {
logNodeIsNotChosen(storage, "the node does not have enough "
+ storage.getStorageType() + " space"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adcfe6c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 1bef805..da85a1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
@@ -642,16 +643,26 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
/**
- * @return Approximate number of blocks currently scheduled to be written
+ * Return the sum of remaining spaces of the specified type. If the remaining
+ * space of a storage is less than minSize, it won't be counted toward the
+ * sum.
+ *
+ * @param t The storage type. If null, the type is ignored.
+ * @param minSize The minimum free space required.
+ * @return the sum of remaining spaces that are bigger than minSize.
*/
- public long getRemaining(StorageType t) {
+ public long getRemaining(StorageType t, long minSize) {
long remaining = 0;
- for(DatanodeStorageInfo s : getStorageInfos()) {
- if (s.getStorageType() == t) {
- remaining += s.getRemaining();
+ for (DatanodeStorageInfo s : getStorageInfos()) {
+ if (s.getState() == State.NORMAL &&
+ (t == null || s.getStorageType() == t)) {
+ long r = s.getRemaining();
+ if (r >= minSize) {
+ remaining += r;
+ }
}
}
- return remaining;
+ return remaining;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0adcfe6c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 1e514af..c85ecb3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -101,6 +101,16 @@ public class TestReplicationPolicy {
dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures);
}
+ private static void updateHeartbeatForExtraStorage(long capacity,
+ long dfsUsed, long remaining, long blockPoolUsed) {
+ DatanodeDescriptor dn = dataNodes[5];
+ dn.getStorageInfos()[1].setUtilizationForTesting(
+ capacity, dfsUsed, remaining, blockPoolUsed);
+ dn.updateHeartbeat(
+ BlockManagerTestUtil.getStorageReportsForDatanode(dn),
+ 0L, 0L, 0, 0);
+ }
+
@BeforeClass
public static void setupCluster() throws Exception {
Configuration conf = new HdfsConfiguration();
@@ -114,6 +124,16 @@ public class TestReplicationPolicy {
storages = DFSTestUtil.createDatanodeStorageInfos(racks);
dataNodes = DFSTestUtil.toDatanodeDescriptor(storages);
+ // create an extra storage for dn5.
+ DatanodeStorage extraStorage = new DatanodeStorage(
+ storages[5].getStorageID() + "-extra", DatanodeStorage.State.NORMAL,
+ StorageType.DEFAULT);
+/* DatanodeStorageInfo si = new DatanodeStorageInfo(
+ storages[5].getDatanodeDescriptor(), extraStorage);
+*/
+ BlockManagerTestUtil.updateStorage(storages[5].getDatanodeDescriptor(),
+ extraStorage);
+
FileSystem.setDefaultUri(conf, "hdfs://localhost:0");
conf.set(DFSConfigKeys.DFS_NAMENODE_HTTP_ADDRESS_KEY, "0.0.0.0:0");
File baseDir = PathUtils.getTestDir(TestReplicationPolicy.class);
@@ -136,11 +156,17 @@ public class TestReplicationPolicy {
bm.getDatanodeManager().getHeartbeatManager().addDatanode(
dataNodes[i]);
}
+ resetHeartbeatForStorages();
+ }
+
+ private static void resetHeartbeatForStorages() {
for (int i=0; i < NUM_OF_DATANODES; i++) {
updateHeartbeatWithUsage(dataNodes[i],
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
}
+ // No available space in the extra storage of dn0
+ updateHeartbeatForExtraStorage(0L, 0L, 0L, 0L);
}
private static boolean isOnSameRack(DatanodeStorageInfo left, DatanodeStorageInfo right) {
@@ -150,6 +176,31 @@ public class TestReplicationPolicy {
private static boolean isOnSameRack(DatanodeStorageInfo left, DatanodeDescriptor right) {
return cluster.isOnSameRack(left.getDatanodeDescriptor(), right);
}
+
+ /**
+ * Test whether the remaining space per storage is individually
+ * considered.
+ */
+ @Test
+ public void testChooseNodeWithMultipleStorages() throws Exception {
+ updateHeartbeatWithUsage(dataNodes[5],
+ 2* HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ (2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE)/3, 0L,
+ 0L, 0L, 0, 0);
+
+ updateHeartbeatForExtraStorage(
+ 2* HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
+ (2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE)/3, 0L);
+
+ DatanodeStorageInfo[] targets;
+ targets = chooseTarget (1, dataNodes[5],
+ new ArrayList<DatanodeStorageInfo>(), null);
+ assertEquals(1, targets.length);
+ assertEquals(storages[4], targets[0]);
+
+ resetHeartbeatForStorages();
+ }
+
/**
* In this testcase, client is dataNodes[0]. So the 1st replica should be
* placed on dataNodes[0], the 2nd replica should be placed on
@@ -191,10 +242,8 @@ public class TestReplicationPolicy {
assertTrue(isOnSameRack(targets[1], targets[2]) ||
isOnSameRack(targets[2], targets[3]));
assertFalse(isOnSameRack(targets[0], targets[2]));
-
- updateHeartbeatWithUsage(dataNodes[0],
- 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
- HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
+
+ resetHeartbeatForStorages();
}
private static DatanodeStorageInfo[] chooseTarget(int numOfReplicas) {
@@ -349,9 +398,7 @@ public class TestReplicationPolicy {
isOnSameRack(targets[2], targets[3]));
assertFalse(isOnSameRack(targets[1], targets[3]));
- updateHeartbeatWithUsage(dataNodes[0],
- 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
- HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
+ resetHeartbeatForStorages();
}
/**
@@ -392,12 +439,8 @@ public class TestReplicationPolicy {
assertTrue(isOnSameRack(targets[0], targets[1]) ||
isOnSameRack(targets[1], targets[2]));
assertFalse(isOnSameRack(targets[0], targets[2]));
-
- for(int i=0; i<2; i++) {
- updateHeartbeatWithUsage(dataNodes[i],
- 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
- HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
- }
+
+ resetHeartbeatForStorages();
}
/**
@@ -475,6 +518,7 @@ public class TestReplicationPolicy {
} finally {
bm.getDatanodeManager().getNetworkTopology().remove(newDn);
}
+ resetHeartbeatForStorages();
}
@@ -528,12 +572,8 @@ public class TestReplicationPolicy {
// Suppose to place replicas on each node but two data nodes are not
// available for placing replica, so here we expect a short of 2
assertTrue(((String)lastLogEntry.getMessage()).contains("in need of 2"));
-
- for(int i=0; i<2; i++) {
- updateHeartbeatWithUsage(dataNodes[i],
- 2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
- HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L, 0L, 0L, 0, 0);
- }
+
+ resetHeartbeatForStorages();
}
private boolean containsWithinRange(DatanodeStorageInfo target,
[18/50] [abbrv] hadoop git commit: HDFS-8846. Add a unit test for
INotify functionality across a layout version upgrade (Zhe Zhang via Colin P.
McCabe)
Posted by sj...@apache.org.
HDFS-8846. Add a unit test for INotify functionality across a layout version upgrade (Zhe Zhang via Colin P. McCabe)
(cherry picked from commit a4d9acc51d1a977bc333da17780c00c72e8546f1)
(cherry picked from commit 9264b7e119efb70fb355904652beeb97e7ad90b9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a976acc0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a976acc0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a976acc0
Branch: refs/heads/branch-2.6
Commit: a976acc02dadb8b29391f25a6bbabcbaf01449c8
Parents: 2e86d1f
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Aug 25 14:09:13 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Tue Sep 8 16:18:30 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/TestDFSInotifyEventInputStream.java | 2 +-
.../org/apache/hadoop/hdfs/TestDFSUpgrade.java | 79 +-------------
.../hadoop/hdfs/TestDFSUpgradeFromImage.java | 107 ++++++++++++++++++-
.../src/test/resources/hadoop-252-dfs-dir.tgz | Bin 0 -> 14112 bytes
5 files changed, 109 insertions(+), 82 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a976acc0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 622fb16..287ffcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -163,6 +163,9 @@ Release 2.6.1 - UNRELEASED
HDFS-7446. HDFS inotify should have the ability to determine what txid it
has read up to (cmccabe)
+ HDFS-8846. Add a unit test for INotify functionality across a layout
+ version upgrade (Zhe Zhang via Colin P. McCabe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a976acc0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index 82db110..b386e45 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -50,7 +50,7 @@ public class TestDFSInotifyEventInputStream {
private static final Log LOG = LogFactory.getLog(
TestDFSInotifyEventInputStream.class);
- private static EventBatch waitForNextEvents(DFSInotifyEventInputStream eis)
+ public static EventBatch waitForNextEvents(DFSInotifyEventInputStream eis)
throws IOException, MissingEventsException {
EventBatch batch = null;
while ((batch = eis.poll()) == null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a976acc0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
index e5777c7..ab2fc49 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgrade.java
@@ -28,21 +28,14 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import java.io.File;
-import java.io.FilenameFilter;
import java.io.IOException;
-import java.util.Collections;
-import java.util.LinkedList;
-import java.nio.file.Files;
-import java.util.List;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.hdfs.inotify.Event;
-import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NodeType;
@@ -50,11 +43,6 @@ import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.common.InconsistentFSStateException;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.StorageInfo;
-import org.apache.hadoop.hdfs.server.namenode.EditLogFileInputStream;
-import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
-import org.apache.hadoop.hdfs.server.namenode.NNStorage;
-import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.server.namenode.TestParallelImageWrite;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.StringUtils;
@@ -62,8 +50,6 @@ import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
-import static org.apache.hadoop.hdfs.inotify.Event.CreateEvent;
-
import com.google.common.base.Charsets;
import com.google.common.base.Joiner;
@@ -81,7 +67,7 @@ public class TestDFSUpgrade {
private Configuration conf;
private int testCounter = 0;
private MiniDFSCluster cluster = null;
-
+
/**
* Writes an INFO log message containing the parameters.
*/
@@ -466,67 +452,6 @@ public class TestDFSUpgrade {
}
}
- @Test
- public void testPreserveEditLogs() throws Exception {
- conf = new HdfsConfiguration();
- conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
- String[] nameNodeDirs = conf.getStrings(DFSConfigKeys.DFS_NAMENODE_NAME_DIR_KEY);
- conf.setBoolean(DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION, false);
-
- log("Normal NameNode upgrade", 1);
- File[] created =
- UpgradeUtilities.createNameNodeStorageDirs(nameNodeDirs, "current");
- for (final File createdDir : created) {
- String[] fileNameList = createdDir.list(EditLogsFilter.INSTANCE);
- for (String fileName : fileNameList) {
- String tmpFileName = fileName + ".tmp";
- File existingFile = new File(createdDir, fileName);
- File tmpFile = new File(createdDir, tmpFileName);
- Files.move(existingFile.toPath(), tmpFile.toPath());
- File newFile = new File(createdDir, fileName);
- Preconditions.checkState(newFile.createNewFile(),
- "Cannot create new edits log file in " + createdDir);
- EditLogFileInputStream in = new EditLogFileInputStream(tmpFile,
- HdfsConstants.INVALID_TXID, HdfsConstants.INVALID_TXID,
- false);
- EditLogFileOutputStream out = new EditLogFileOutputStream(conf, newFile,
- (int)tmpFile.length());
- out.create(NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION + 1);
- FSEditLogOp logOp = in.readOp();
- while (logOp != null) {
- out.write(logOp);
- logOp = in.readOp();
- }
- out.setReadyToFlush();
- out.flushAndSync(true);
- out.close();
- Files.delete(tmpFile.toPath());
- }
- }
-
- cluster = createCluster();
-
- DFSInotifyEventInputStream ieis =
- cluster.getFileSystem().getInotifyEventStream(0);
- EventBatch batch = ieis.poll();
- Event[] events = batch.getEvents();
- assertTrue("Should be able to get transactions before the upgrade.",
- events.length > 0);
- assertEquals(events[0].getEventType(), Event.EventType.CREATE);
- assertEquals(((CreateEvent) events[0]).getPath(), "/TestUpgrade");
- cluster.shutdown();
- UpgradeUtilities.createEmptyDirs(nameNodeDirs);
- }
-
- private enum EditLogsFilter implements FilenameFilter {
- INSTANCE;
-
- @Override
- public boolean accept(File dir, String name) {
- return name.startsWith(NNStorage.NameNodeFile.EDITS.getName());
- }
- }
-
public static void main(String[] args) throws Exception {
TestDFSUpgrade t = new TestDFSUpgrade();
TestDFSUpgrade.initialize();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a976acc0/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
index ad907f6..ffbaa89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUpgradeFromImage.java
@@ -18,10 +18,6 @@
package org.apache.hadoop.hdfs;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
-
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
@@ -40,10 +36,13 @@ import org.apache.hadoop.fs.FSInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.inotify.Event;
+import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
+import org.apache.hadoop.hdfs.server.namenode.FSImage;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormat;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
import org.apache.hadoop.test.GenericTestUtils;
@@ -51,6 +50,9 @@ import org.apache.hadoop.util.StringUtils;
import org.apache.log4j.Logger;
import org.junit.Test;
+import static org.apache.hadoop.hdfs.inotify.Event.CreateEvent;
+import static org.junit.Assert.*;
+
/**
* This tests data transfer protocol handling in the Datanode. It sends
* various forms of wrong data and verifies that Datanode handles it well.
@@ -74,6 +76,7 @@ public class TestDFSUpgradeFromImage {
private static final String HADOOP023_RESERVED_IMAGE =
"hadoop-0.23-reserved.tgz";
private static final String HADOOP2_RESERVED_IMAGE = "hadoop-2-reserved.tgz";
+ private static final String HADOOP252_IMAGE = "hadoop-252-dfs-dir.tgz";
private static class ReferenceFileInfo {
String path;
@@ -620,4 +623,100 @@ public class TestDFSUpgradeFromImage {
numDataNodes(1).enableManagedDfsDirsRedundancy(false).
manageDataDfsDirs(false), null);
}
+
+ @Test
+ public void testPreserveEditLogs() throws Exception {
+ unpackStorage(HADOOP252_IMAGE, HADOOP_DFS_DIR_TXT);
+ /**
+ * The pre-created image has the following edits:
+ * mkdir /input; mkdir /input/dir1~5
+ * copyFromLocal randome_file_1 /input/dir1
+ * copyFromLocal randome_file_2 /input/dir2
+ * mv /input/dir1/randome_file_1 /input/dir3/randome_file_3
+ * rmdir /input/dir1
+ */
+ Configuration conf = new HdfsConfiguration();
+ conf = UpgradeUtilities.initializeStorageStateConf(1, conf);
+ MiniDFSCluster cluster = new MiniDFSCluster.Builder(conf).numDataNodes(0)
+ .format(false)
+ .manageDataDfsDirs(false)
+ .manageNameDfsDirs(false)
+ .startupOption(StartupOption.UPGRADE)
+ .build();
+ DFSInotifyEventInputStream ieis =
+ cluster.getFileSystem().getInotifyEventStream(0);
+
+ EventBatch batch;
+ Event.CreateEvent ce;
+ Event.RenameEvent re;
+
+ // mkdir /input
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ assertEquals(ce.getPath(), "/input");
+
+ // mkdir /input/dir1~5
+ for (int i = 1; i <= 5; i++) {
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+ ce = (Event.CreateEvent) batch.getEvents()[0];
+ assertEquals(ce.getPath(), "/input/dir" + i);
+ }
+ // copyFromLocal randome_file_1~2 /input/dir1~2
+ for (int i = 1; i <= 2; i++) {
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ if (batch.getEvents()[0].getEventType() != Event.EventType.CREATE) {
+ FSImage.LOG.debug("");
+ }
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CREATE);
+
+ // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.CLOSE);
+
+ // copyFromLocal randome_file_1 /input/dir1, CLOSE
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() ==
+ Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir" + i + "/randome_file_" + i);
+ }
+
+ // mv /input/dir1/randome_file_1 /input/dir3/randome_file_3
+ long txIDBeforeRename = batch.getTxid();
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
+
+
+ // rmdir /input/dir1
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.UNLINK);
+ assertEquals(((Event.UnlinkEvent) batch.getEvents()[0]).getPath(),
+ "/input/dir1");
+ long lastTxID = batch.getTxid();
+
+ // Start inotify from the tx before rename /input/dir1/randome_file_1
+ ieis = cluster.getFileSystem().getInotifyEventStream(txIDBeforeRename);
+ batch = TestDFSInotifyEventInputStream.waitForNextEvents(ieis);
+ assertEquals(1, batch.getEvents().length);
+ assertTrue(batch.getEvents()[0].getEventType() == Event.EventType.RENAME);
+ re = (Event.RenameEvent) batch.getEvents()[0];
+ assertEquals(re.getDstPath(), "/input/dir3/randome_file_3");
+
+ // Try to read beyond available edits
+ ieis = cluster.getFileSystem().getInotifyEventStream(lastTxID + 1);
+ assertNull(ieis.poll());
+
+ cluster.shutdown();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a976acc0/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz
new file mode 100644
index 0000000..4ad3e25
Binary files /dev/null and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/hadoop-252-dfs-dir.tgz differ
[35/50] [abbrv] hadoop git commit: YARN-3379. Fixed missing data in
localityTable and ResourceRequests table in RM WebUI. Contributed by Xuan
Gong (cherry picked from commit 4e886eb9cbd2dcb128bbfd17309c734083093a4c)
Posted by sj...@apache.org.
YARN-3379. Fixed missing data in localityTable and ResourceRequests table in RM WebUI. Contributed by Xuan Gong
(cherry picked from commit 4e886eb9cbd2dcb128bbfd17309c734083093a4c)
(cherry picked from commit 3f0c9e5fe36d201de021d989b23ebaeb2d9a027b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e914220a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e914220a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e914220a
Branch: refs/heads/branch-2.6
Commit: e914220ab9d090b709d445bb415e45eab0c4dea5
Parents: 3ab820e
Author: Jian He <ji...@apache.org>
Authored: Thu Mar 19 22:27:21 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Mon Sep 14 12:54:01 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/server/webapp/AppAttemptBlock.java | 3 +-
.../hadoop/yarn/server/webapp/AppBlock.java | 47 ++--------
.../resourcemanager/webapp/AppAttemptPage.java | 3 +-
.../server/resourcemanager/webapp/AppPage.java | 3 +-
.../webapp/RMAppAttemptBlock.java | 56 ++++++++++++
.../resourcemanager/webapp/RMAppBlock.java | 94 ++++++++++++++++++++
7 files changed, 165 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6641bb2..e7d847f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -214,6 +214,9 @@ Release 2.6.1 - 2015-09-09
YARN-3740. Fixed the typo in the configuration name:
APPLICATION_HISTORY_PREFIX_MAX_APPS. (Xuan Gong via zjshen)
+ YARN-3379. Fixed missing data in localityTable and ResourceRequests table
+ in RM WebUI. (Xuan Gong via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
index ae5f092..abf7c86 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppAttemptBlock.java
@@ -43,13 +43,13 @@ import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import org.apache.hadoop.yarn.webapp.view.InfoBlock;
-
import com.google.inject.Inject;
public class AppAttemptBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppAttemptBlock.class);
protected ApplicationBaseProtocol appBaseProt;
+ protected ApplicationAttemptId appAttemptId = null;
@Inject
public AppAttemptBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx) {
@@ -65,7 +65,6 @@ public class AppAttemptBlock extends HtmlBlock {
return;
}
- ApplicationAttemptId appAttemptId = null;
try {
appAttemptId = ConverterUtils.toApplicationAttemptId(attemptid);
} catch (IllegalArgumentException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
index e62c783..93b24a3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/webapp/AppBlock.java
@@ -21,15 +21,9 @@ package org.apache.hadoop.yarn.server.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.join;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.APPLICATION_ID;
import static org.apache.hadoop.yarn.webapp.YarnWebParams.WEB_UI_TYPE;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._EVEN;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._ODD;
-import static org.apache.hadoop.yarn.webapp.view.JQueryUI._TH;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
-import java.util.List;
-
import org.apache.commons.lang.StringEscapeUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -43,12 +37,9 @@ import org.apache.hadoop.yarn.api.protocolrecords.GetContainerReportRequest;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ApplicationReport;
-import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
-import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
@@ -57,10 +48,8 @@ import org.apache.hadoop.yarn.server.webapp.dao.AppInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.Times;
-import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
-import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TABLE;
import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.TBODY;
import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
@@ -73,9 +62,11 @@ public class AppBlock extends HtmlBlock {
private static final Log LOG = LogFactory.getLog(AppBlock.class);
protected ApplicationBaseProtocol appBaseProt;
protected Configuration conf;
+ protected ApplicationId appID = null;
@Inject
- AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx, Configuration conf) {
+ protected AppBlock(ApplicationBaseProtocol appBaseProt, ViewContext ctx,
+ Configuration conf) {
super(ctx);
this.appBaseProt = appBaseProt;
this.conf = conf;
@@ -90,7 +81,6 @@ public class AppBlock extends HtmlBlock {
return;
}
- ApplicationId appID = null;
try {
appID = Apps.toAppID(aid);
} catch (Exception e) {
@@ -217,31 +207,7 @@ public class AppBlock extends HtmlBlock {
return;
}
- //TODO:YARN-3284
- //The preemption metrics will be exposed from ApplicationReport
- // and ApplicationAttemptReport
- ApplicationResourceUsageReport usageReport =
- appReport.getApplicationResourceUsageReport();
- DIV<Hamlet> pdiv = html.
- _(InfoBlock.class).
- div(_INFO_WRAP);
- info("Application Overview").clear();
- info("Application Metrics")
- ._("Total Resource Preempted:",
- Resources.none()) // TODO: YARN-3284
- ._("Total Number of Non-AM Containers Preempted:",
- String.valueOf(0)) // TODO: YARN-3284
- ._("Total Number of AM Containers Preempted:",
- String.valueOf(0)) // TODO: YARN-3284
- ._("Resource Preempted from Current Attempt:",
- Resources.none()) // TODO: YARN-3284
- ._("Number of Non-AM Containers Preempted from Current Attempt:",
- 0) // TODO: YARN-3284
- ._("Aggregate Resource Allocation:",
- String.format("%d MB-seconds, %d vcore-seconds", usageReport == null
- ? 0 : usageReport.getMemorySeconds(), usageReport == null ? 0
- : usageReport.getVcoreSeconds()));
- pdiv._();
+ createApplicationMetricsTable(html);
html._(InfoBlock.class);
@@ -352,4 +318,9 @@ public class AppBlock extends HtmlBlock {
}
return status.toString();
}
+
+ // The preemption metrics only need to be shown in RM WebUI
+ protected void createApplicationMetricsTable(Block html) {
+
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
index 92eae48..2bbe4ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppAttemptPage.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -49,7 +48,7 @@ public class AppAttemptPage extends RmView {
@Override
protected Class<? extends SubView> content() {
- return AppAttemptBlock.class;
+ return RMAppAttemptBlock.class;
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
index 77ae44c..b0698af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/AppPage.java
@@ -23,7 +23,6 @@ import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.DATATABLES_ID;
import static org.apache.hadoop.yarn.webapp.view.JQueryUI.initID;
-import org.apache.hadoop.yarn.server.webapp.AppBlock;
import org.apache.hadoop.yarn.server.webapp.WebPageUtils;
import org.apache.hadoop.yarn.webapp.SubView;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -50,6 +49,6 @@ public class AppPage extends RmView {
@Override
protected Class<? extends SubView> content() {
- return AppBlock.class;
+ return RMAppBlock.class;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
new file mode 100644
index 0000000..7dc7030
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppAttemptBlock.java
@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
+import org.apache.hadoop.yarn.server.webapp.AppAttemptBlock;
+
+import com.google.inject.Inject;
+
+public class RMAppAttemptBlock extends AppAttemptBlock{
+
+ private final ResourceManager rm;
+ protected Configuration conf;
+
+ @Inject
+ RMAppAttemptBlock(ViewContext ctx, ResourceManager rm, Configuration conf) {
+ super(rm.getClientRMService(), ctx);
+ this.rm = rm;
+ this.conf = conf;
+ }
+
+ @Override
+ protected void render(Block html) {
+ super.render(html);
+ }
+
+ private RMAppAttempt getRMAppAttempt() {
+ ApplicationId appId = this.appAttemptId.getApplicationId();
+ RMAppAttempt attempt = null;
+ RMApp rmApp = rm.getRMContext().getRMApps().get(appId);
+ if (rmApp != null) {
+ attempt = rmApp.getAppAttempts().get(appAttemptId);
+ }
+ return attempt;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e914220a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
new file mode 100644
index 0000000..64c5747
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMAppBlock.java
@@ -0,0 +1,94 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp;
+
+import static org.apache.hadoop.yarn.webapp.view.JQueryUI._INFO_WRAP;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptMetrics;
+import org.apache.hadoop.yarn.server.webapp.AppBlock;
+import org.apache.hadoop.yarn.util.resource.Resources;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet;
+import org.apache.hadoop.yarn.webapp.hamlet.Hamlet.DIV;
+import org.apache.hadoop.yarn.webapp.view.InfoBlock;
+
+import com.google.inject.Inject;
+
+public class RMAppBlock extends AppBlock{
+
+ private final ResourceManager rm;
+
+ @Inject
+ RMAppBlock(ViewContext ctx, Configuration conf, ResourceManager rm) {
+ super(rm.getClientRMService(), ctx, conf);
+ this.rm = rm;
+ }
+
+ @Override
+ protected void render(Block html) {
+ super.render(html);
+ }
+
+ @Override
+ protected void createApplicationMetricsTable(Block html){
+ RMApp rmApp = this.rm.getRMContext().getRMApps().get(appID);
+ RMAppMetrics appMetrics = rmApp == null ? null : rmApp.getRMAppMetrics();
+ // Get attempt metrics and fields, it is possible currentAttempt of RMApp is
+ // null. In that case, we will assume resource preempted and number of Non
+ // AM container preempted on that attempt is 0
+ RMAppAttemptMetrics attemptMetrics;
+ if (rmApp == null || null == rmApp.getCurrentAppAttempt()) {
+ attemptMetrics = null;
+ } else {
+ attemptMetrics = rmApp.getCurrentAppAttempt().getRMAppAttemptMetrics();
+ }
+ Resource attemptResourcePreempted =
+ attemptMetrics == null ? Resources.none() : attemptMetrics
+ .getResourcePreempted();
+ int attemptNumNonAMContainerPreempted =
+ attemptMetrics == null ? 0 : attemptMetrics
+ .getNumNonAMContainersPreempted();
+ DIV<Hamlet> pdiv = html.
+ _(InfoBlock.class).
+ div(_INFO_WRAP);
+ info("Application Overview").clear();
+ info("Application Metrics")
+ ._("Total Resource Preempted:",
+ appMetrics == null ? "N/A" : appMetrics.getResourcePreempted())
+ ._("Total Number of Non-AM Containers Preempted:",
+ appMetrics == null ? "N/A"
+ : appMetrics.getNumNonAMContainersPreempted())
+ ._("Total Number of AM Containers Preempted:",
+ appMetrics == null ? "N/A"
+ : appMetrics.getNumAMContainersPreempted())
+ ._("Resource Preempted from Current Attempt:",
+ attemptResourcePreempted)
+ ._("Number of Non-AM Containers Preempted from Current Attempt:",
+ attemptNumNonAMContainerPreempted)
+ ._("Aggregate Resource Allocation:",
+ String.format("%d MB-seconds, %d vcore-seconds",
+ appMetrics == null ? "N/A" : appMetrics.getMemorySeconds(),
+ appMetrics == null ? "N/A" : appMetrics.getVcoreSeconds()));
+ pdiv._();
+ }
+}