You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by wa...@apache.org on 2015/12/19 01:36:22 UTC
[01/20] hadoop git commit: HDFS-9436. Make
NNThroughputBenchmark$BlockReportStats run with 10 datanodes by default.
Contributed by Mingliang. (cherry picked from commit
3aa73a867a3a5048de72e08d73ffaabe84f8ddee)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 cef21b7aa -> b74391ce1
HDFS-9436. Make NNThroughputBenchmark$BlockReportStats run with 10 datanodes by default. Contributed by Mingliang.
(cherry picked from commit 3aa73a867a3a5048de72e08d73ffaabe84f8ddee)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/69d3a71f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/69d3a71f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/69d3a71f
Branch: refs/heads/branch-2.8
Commit: 69d3a71f148c931fc827ebdda4bb219a20b7454a
Parents: 15b3ff1
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Dec 3 14:54:06 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:33:53 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/NNThroughputBenchmark.java | 4 +++-
2 files changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d3a71f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 56f9602..a571a08 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -921,6 +921,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8335. FSNamesystem should construct FSPermissionChecker only if
permission is enabled. (Gabor Liptak via wheat9)
+ HDFS-9436. Make NNThroughputBenchmark$BlockReportStats run with 10
+ datanodes by default. (Mingliang Liu via shv)
+
BUG FIXES
HDFS-8091: ACLStatus and XAttributes should be presented to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/69d3a71f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index affbe2f..91f9793 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -1098,6 +1098,8 @@ public class NNThroughputBenchmark implements Tool {
BlockReportStats(List<String> args) {
super();
+ numThreads = 10;
+ numOpsRequired = 30;
this.blocksPerReport = 100;
this.blocksPerFile = 10;
// set heartbeat interval to 3 min, so that expiration were 40 min
@@ -1258,7 +1260,7 @@ public class NNThroughputBenchmark implements Tool {
ReplicationStats(List<String> args) {
super();
numThreads = 1;
- numDatanodes = 3;
+ numDatanodes = 10;
nodesToDecommission = 1;
nodeReplicationLimit = 100;
totalBlocks = 100;
[04/20] hadoop git commit: Add missing file for YARN-4419
Posted by wa...@apache.org.
Add missing file for YARN-4419
(cherry picked from commit 8a97ca4b00c4eea044d90e8ed0dc75038bd67bc6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/12772f5a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/12772f5a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/12772f5a
Branch: refs/heads/branch-2.8
Commit: 12772f5a450350627c94aa1e8d6b006255aca522
Parents: f355645
Author: Jian He <ji...@apache.org>
Authored: Thu Dec 3 19:07:28 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:42 2015 -0800
----------------------------------------------------------------------
.../NonAppendableFSNodeLabelStore.java | 134 +++++++++++++++++++
1 file changed, 134 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/12772f5a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
new file mode 100644
index 0000000..6be5715
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NonAppendableFSNodeLabelStore.java
@@ -0,0 +1,134 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.nodelabels;
+
+import java.io.IOException;
+import java.util.Collection;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.ReplaceLabelsOnNodeRequest;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.AddToClusterNodeLabelsRequestPBImpl;
+import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOnNodeRequestPBImpl;
+
+public class NonAppendableFSNodeLabelStore extends FileSystemNodeLabelsStore {
+ protected static final Log
+ LOG = LogFactory.getLog(NonAppendableFSNodeLabelStore.class);
+
+ @Override
+ public void close() throws IOException {
+ }
+
+
+ @Override
+ public void recover() throws YarnException,
+ IOException {
+ Path newMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new");
+ Path oldMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
+ loadFromMirror(newMirrorPath, oldMirrorPath);
+
+ // if new mirror exists, remove old mirror and rename new mirror
+ if (fs.exists(newMirrorPath)) {
+ // remove old mirror
+ try {
+ fs.delete(oldMirrorPath, false);
+ } catch (IOException e) {
+ // do nothing
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exception while removing old mirror", e);
+ }
+ }
+
+ // rename new to old
+ fs.rename(newMirrorPath, oldMirrorPath);
+ }
+
+ LOG.info("Node label store recover is completed");
+ }
+
+ @Override
+ public void updateNodeToLabelsMappings(
+ Map<NodeId, Set<String>> nodeToLabels) throws IOException {
+ writeNewMirror();
+ }
+
+ @Override
+ public void storeNewClusterNodeLabels(List<NodeLabel> labels)
+ throws IOException {
+ writeNewMirror();
+ }
+
+ @Override
+ public void removeClusterNodeLabels(Collection<String> labels)
+ throws IOException {
+ writeNewMirror();
+ }
+
+ private void writeNewMirror() throws IOException {
+ ReentrantReadWriteLock.ReadLock readLock = mgr.readLock;
+ try {
+ // Acquire readlock to make sure we get cluster node labels and
+ // node-to-labels mapping atomically.
+ readLock.lock();
+ List<NodeLabel> nodeLabels = mgr.getClusterNodeLabels();
+ Map<NodeId, Set<String>> nodeToLabels = mgr.getNodeLabels();
+
+ // Write mirror to mirror.new.tmp file
+ Path newTmpPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new.tmp");
+ FSDataOutputStream os = fs
+ .create(newTmpPath, true);
+ ((AddToClusterNodeLabelsRequestPBImpl) AddToClusterNodeLabelsRequest
+ .newInstance(nodeLabels)).getProto().writeDelimitedTo(os);
+
+ if (mgr.isCentralizedConfiguration()) {
+ // Only save node-to-labels mapping while using centralized configuration
+ ((ReplaceLabelsOnNodeRequestPBImpl) ReplaceLabelsOnNodeRequest
+ .newInstance(nodeToLabels)).getProto().writeDelimitedTo(os);
+ }
+
+ os.close();
+
+ // Rename mirror.new.tmp to mirror.new (will remove .new if it's existed)
+ Path newPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".new");
+ if (fs.exists(newPath)) {
+ fs.delete(newPath, false);
+ }
+ fs.rename(newTmpPath, newPath);
+
+ // Remove existing mirror and rename mirror.new to mirror
+ Path mirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
+ if (fs.exists(mirrorPath)) {
+ fs.delete(mirrorPath, false);
+ }
+ fs.rename(newPath, mirrorPath);
+ } finally {
+ readLock.unlock();
+ }
+ }
+}
[08/20] hadoop git commit: YARN-3623-Addendum: Improve the
description for Timeline Service Version configuration in yarn-default.xml
Posted by wa...@apache.org.
YARN-3623-Addendum: Improve the description for Timeline Service Version configuration in yarn-default.xml
(cherry picked from commit 21daa6c68a0bff51a14e748bf14d56b2f5a5580f)
(cherry picked from commit 455c6f39c582f5f482f0dc255d111e6fe149e66a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/41986d53
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/41986d53
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/41986d53
Branch: refs/heads/branch-2.8
Commit: 41986d5373386469650a28d4a3ff1fb7a772233c
Parents: a4ba6c8
Author: Xuan <xg...@apache.org>
Authored: Thu Dec 10 10:56:12 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:43 2015 -0800
----------------------------------------------------------------------
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/41986d53/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 0917366..5b71b1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1769,7 +1769,7 @@
<description>Indicate what is the current version of the running
timeline service. For example, if "yarn.timeline-service.version" is 1.5,
and "yarn.timeline-service.enabled" is true, it means the cluster will and
- should bring up the timeline service v.1.5 (and nothing else).
+ should bring up the timeline service v.1.5.
On the client side, if the client uses the same version of timeline service,
it should succeed. If the client chooses to use a smaller version in spite of this,
then depending on how robust the compatibility story is between versions,
[14/20] hadoop git commit: YARN-4309. Add container launch related
debug information to container logs when a container fails. (Varun Vasudev
via wangda)
Posted by wa...@apache.org.
YARN-4309. Add container launch related debug information to container logs when a container fails. (Varun Vasudev via wangda)
(cherry picked from commit dfcbbddb0963c89c0455d41223427165b9f9e537)
(cherry picked from commit 9f979107c22f365633811a38363d0dbd34573374)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c7e10d70
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c7e10d70
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c7e10d70
Branch: refs/heads/branch-2.8
Commit: c7e10d70de3d564e396dd61b5d2ba232d06e966a
Parents: 8ee8376
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Dec 14 11:13:22 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:44 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 5 +
.../src/main/resources/yarn-default.xml | 12 +++
.../server/nodemanager/ContainerExecutor.java | 24 ++++-
.../nodemanager/DockerContainerExecutor.java | 11 +-
.../launcher/ContainerLaunch.java | 74 +++++++++++++-
.../launcher/TestContainerLaunch.java | 102 +++++++++++++++++--
7 files changed, 217 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9f0e8e1..6d3f519 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -545,6 +545,9 @@ Release 2.8.0 - UNRELEASED
YARN-3946. Update exact reason as to why a submitted app is in ACCEPTED state to
app's diagnostic message. (Naganarasimha G R via wangda)
+ YARN-4309. Add container launch related debug information to container logs
+ when a container fails. (Varun Vasudev via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 521bb3d..35ad12f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -92,6 +92,11 @@ public class YarnConfiguration extends Configuration {
/** Delay before deleting resource to ease debugging of NM issues */
public static final String DEBUG_NM_DELETE_DELAY_SEC =
YarnConfiguration.NM_PREFIX + "delete.debug-delay-sec";
+
+ public static final String NM_LOG_CONTAINER_DEBUG_INFO =
+ YarnConfiguration.NM_PREFIX + "log-container-debug-info.enabled";
+
+ public static final boolean DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO = false;
////////////////////////////////
// IPC Configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 5b71b1f..13a7b1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1128,6 +1128,18 @@
</property>
<property>
+ <description>Generate additional logs about container launches.
+ Currently, this creates a copy of the launch script and lists the
+ directory contents of the container work dir. When listing directory
+ contents, we follow symlinks to a max-depth of 5(including symlinks
+ which point to outside the container work dir) which may lead to a
+ slowness in launching containers.
+ </description>
+ <name>yarn.nodemanager.log-container-debug-info.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>Amount of physical memory, in MB, that can be allocated
for containers. If set to -1 and
yarn.nodemanager.resource.detect-hardware-capabilities is true, it is
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 6d75a1c..8c74bf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -34,6 +34,7 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -65,6 +66,8 @@ public abstract class ContainerExecutor implements Configurable {
final public static FsPermission TASK_LAUNCH_SCRIPT_PERMISSION =
FsPermission.createImmutable((short) 0700);
+ public static final String DIRECTORY_CONTENTS = "directory.info";
+
private Configuration conf;
private ConcurrentMap<ContainerId, Path> pidFiles =
@@ -241,11 +244,22 @@ public abstract class ContainerExecutor implements Configurable {
* @param resources The resources which have been localized for this container
* Symlinks will be created to these localized resources
* @param command The command that will be run.
+ * @param logDir The log dir to copy debugging information to
* @throws IOException if any errors happened writing to the OutputStream,
* while creating symlinks
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
- Map<Path, List<String>> resources, List<String> command) throws IOException{
+ Map<Path, List<String>> resources, List<String> command, Path logDir)
+ throws IOException {
+ this.writeLaunchEnv(out, environment, resources, command, logDir,
+ ContainerLaunch.CONTAINER_SCRIPT);
+ }
+
+ @VisibleForTesting
+ public void writeLaunchEnv(OutputStream out,
+ Map<String, String> environment, Map<Path, List<String>> resources,
+ List<String> command, Path logDir, String outFilename)
+ throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
Set<String> whitelist = new HashSet<String>();
@@ -272,6 +286,14 @@ public abstract class ContainerExecutor implements Configurable {
}
}
+ // dump debugging information if configured
+ if (getConf() != null && getConf().getBoolean(
+ YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
+ YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO)) {
+ sb.copyDebugInformation(new Path(outFilename), new Path(logDir, outFilename));
+ sb.listDebugInformation(new Path(logDir, DIRECTORY_CONTENTS));
+ }
+
sb.command(command);
PrintStream pout = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index 96f8b51..b089947 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -329,7 +329,7 @@ public class DockerContainerExecutor extends ContainerExecutor {
* the docker image and write them out to an OutputStream.
*/
public void writeLaunchEnv(OutputStream out, Map<String, String> environment,
- Map<Path, List<String>> resources, List<String> command)
+ Map<Path, List<String>> resources, List<String> command, Path logDir)
throws IOException {
ContainerLaunch.ShellScriptBuilder sb =
ContainerLaunch.ShellScriptBuilder.create();
@@ -358,6 +358,15 @@ public class DockerContainerExecutor extends ContainerExecutor {
}
}
+ // dump debugging information if configured
+ if (getConf() != null && getConf().getBoolean(
+ YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
+ YarnConfiguration.DEFAULT_NM_LOG_CONTAINER_DEBUG_INFO)) {
+ sb.copyDebugInformation(new Path(ContainerLaunch.CONTAINER_SCRIPT),
+ new Path(logDir, ContainerLaunch.CONTAINER_SCRIPT));
+ sb.listDebugInformation(new Path(logDir, DIRECTORY_CONTENTS));
+ }
+
sb.command(command);
PrintStream pout = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index fc8615b..b7a97d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -272,7 +272,8 @@ public class ContainerLaunch implements Callable<Integer> {
// Write out the environment
exec.writeLaunchEnv(containerScriptOutStream, environment,
- localResources, launchContext.getCommands());
+ localResources, launchContext.getCommands(),
+ new Path(containerLogDirs.get(0)));
// /////////// End of writing out container-script
@@ -641,6 +642,28 @@ public class ContainerLaunch implements Callable<Integer> {
link(src, dst);
}
+ /**
+ * Method to copy files that are useful for debugging container failures.
+ * This method will be called by ContainerExecutor when setting up the
+ * container launch script. The method should take care to make sure files
+ * are read-able by the yarn user if the files are to undergo
+ * log-aggregation.
+ * @param src path to the source file
+ * @param dst path to the destination file - should be absolute
+ * @throws IOException
+ */
+ public abstract void copyDebugInformation(Path src, Path dst)
+ throws IOException;
+
+ /**
+ * Method to dump debug information to the a target file. This method will
+ * be called by ContainerExecutor when setting up the container launch
+ * script.
+ * @param output the file to which debug information is to be written
+ * @throws IOException
+ */
+ public abstract void listDebugInformation(Path output) throws IOException;
+
@Override
public String toString() {
return sb.toString();
@@ -704,6 +727,36 @@ public class ContainerLaunch implements Callable<Integer> {
line("mkdir -p ", path.toString());
errorCheck();
}
+
+ @Override
+ public void copyDebugInformation(Path src, Path dest) throws IOException {
+ line("# Creating copy of launch script");
+ line("cp \"", src.toUri().getPath(), "\" \"", dest.toUri().getPath(),
+ "\"");
+ // set permissions to 640 because we need to be able to run
+ // log aggregation in secure mode as well
+ if(dest.isAbsolute()) {
+ line("chmod 640 \"", dest.toUri().getPath(), "\"");
+ }
+ }
+
+ @Override
+ public void listDebugInformation(Path output) throws IOException {
+ line("# Determining directory contents");
+ line("echo \"ls -l:\" 1>\"", output.toString(), "\"");
+ line("ls -l 1>>\"", output.toString(), "\"");
+
+ // don't run error check because if there are loops
+ // find will exit with an error causing container launch to fail
+ // find will follow symlinks outside the work dir if such sylimks exist
+ // (like public/app local resources)
+ line("echo \"find -L . -maxdepth 5 -ls:\" 1>>\"", output.toString(),
+ "\"");
+ line("find -L . -maxdepth 5 -ls 1>>\"", output.toString(), "\"");
+ line("echo \"broken symlinks(find -L . -maxdepth 5 -type l -ls):\" 1>>\"",
+ output.toString(), "\"");
+ line("find -L . -maxdepth 5 -type l -ls 1>>\"", output.toString(), "\"");
+ }
}
private static final class WindowsShellScriptBuilder
@@ -757,6 +810,25 @@ public class ContainerLaunch implements Callable<Integer> {
path.toString(), path.toString()));
errorCheck();
}
+
+ @Override
+ public void copyDebugInformation(Path src, Path dest)
+ throws IOException {
+ // no need to worry about permissions - in secure mode
+ // WindowsSecureContainerExecutor will set permissions
+ // to allow NM to read the file
+ line("rem Creating copy of launch script");
+ lineWithLenCheck(String.format("copy \"%s\" \"%s\"", src.toString(),
+ dest.toString()));
+ }
+
+ @Override
+ public void listDebugInformation(Path output) throws IOException {
+ line("rem Determining directory contents");
+ lineWithLenCheck(
+ String.format("@echo \"dir:\" > \"%s\"", output.toString()));
+ lineWithLenCheck(String.format("dir >> \"%s\"", output.toString()));
+ }
}
private static void putEnvIfNotNull(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c7e10d70/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
index 0abae2b..502ecde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/TestContainerLaunch.java
@@ -100,6 +100,7 @@ import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.LinuxResourceCalculatorPlugin;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.hamcrest.CoreMatchers;
import org.junit.Assert;
import org.junit.Assume;
import org.junit.Before;
@@ -166,7 +167,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
commands.add("/bin/sh ./\\\"" + badSymlink + "\\\"");
}
- new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
+ new DefaultContainerExecutor()
+ .writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), tempFile.getName());
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -233,7 +236,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
} else {
commands.add("/bin/sh ./\\\"" + symLink + "\\\"");
}
- new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
+ new DefaultContainerExecutor()
+ .writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()));
fos.flush();
fos.close();
FileUtil.setExecutable(tempFile, true);
@@ -286,7 +291,9 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"\"workflowName\":\"\n\ninsert table " +
"\npartition (cd_education_status)\nselect cd_demo_sk, cd_gender, " );
List<String> commands = new ArrayList<String>();
- new DefaultContainerExecutor().writeLaunchEnv(fos, env, resources, commands);
+ new DefaultContainerExecutor()
+ .writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()));
fos.flush();
fos.close();
@@ -364,7 +371,8 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
List<String> commands = new ArrayList<String>();
commands.add(command);
ContainerExecutor exec = new DefaultContainerExecutor();
- exec.writeLaunchEnv(fos, env, resources, commands);
+ exec.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()));
fos.flush();
fos.close();
@@ -994,7 +1002,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"X", Shell.WINDOWS_MAX_SHELL_LENGTH -callCmd.length() + 1)));
fail("longCommand was expected to throw");
} catch(IOException e) {
- assertThat(e.getMessage(), containsString(expectedMessage));
+ assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage));
}
// Composite tests, from parts: less, exact and +
@@ -1016,7 +1024,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
org.apache.commons.lang.StringUtils.repeat("X", 2048 - callCmd.length())));
fail("long commands was expected to throw");
} catch(IOException e) {
- assertThat(e.getMessage(), containsString(expectedMessage));
+ assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage));
}
}
@@ -1039,7 +1047,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"A", Shell.WINDOWS_MAX_SHELL_LENGTH - ("@set somekey=").length()) + 1);
fail("long env was expected to throw");
} catch(IOException e) {
- assertThat(e.getMessage(), containsString(expectedMessage));
+ assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage));
}
}
@@ -1057,14 +1065,14 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
// test mkdir
builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat("A", 1024)));
- builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat(
- "E", (Shell.WINDOWS_MAX_SHELL_LENGTH - mkDirCmd.length())/2)));
+ builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat("E",
+ (Shell.WINDOWS_MAX_SHELL_LENGTH - mkDirCmd.length()) / 2)));
try {
builder.mkdir(new Path(org.apache.commons.lang.StringUtils.repeat(
"X", (Shell.WINDOWS_MAX_SHELL_LENGTH - mkDirCmd.length())/2 +1)));
fail("long mkdir was expected to throw");
} catch(IOException e) {
- assertThat(e.getMessage(), containsString(expectedMessage));
+ assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage));
}
}
@@ -1095,7 +1103,7 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
"Y", (Shell.WINDOWS_MAX_SHELL_LENGTH - linkCmd.length())/2) + 1));
fail("long link was expected to throw");
} catch(IOException e) {
- assertThat(e.getMessage(), containsString(expectedMessage));
+ assertThat(e.getMessage(), CoreMatchers.containsString(expectedMessage));
}
}
@@ -1208,4 +1216,76 @@ public class TestContainerLaunch extends BaseContainerManagerTest {
Assert.assertEquals(ExitCode.FORCE_KILLED.getExitCode(),
containerStatus.getExitStatus());
}
+
+ @Test
+ public void testDebuggingInformation() throws IOException {
+
+ File shellFile = null;
+ File tempFile = null;
+ Configuration conf = new YarnConfiguration();
+ try {
+ shellFile = Shell.appendScriptExtension(tmpDir, "hello");
+ tempFile = Shell.appendScriptExtension(tmpDir, "temp");
+ String testCommand = Shell.WINDOWS ? "@echo \"hello\"" :
+ "echo \"hello\"";
+ PrintWriter writer = new PrintWriter(new FileOutputStream(shellFile));
+ FileUtil.setExecutable(shellFile, true);
+ writer.println(testCommand);
+ writer.close();
+
+ Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
+ Map<String, String> env = new HashMap<String, String>();
+ List<String> commands = new ArrayList<String>();
+ if (Shell.WINDOWS) {
+ commands.add("cmd");
+ commands.add("/c");
+ commands.add("\"" + shellFile.getAbsolutePath() + "\"");
+ } else {
+ commands.add("/bin/sh \\\"" + shellFile.getAbsolutePath() + "\\\"");
+ }
+
+ boolean[] debugLogsExistArray = { false, true };
+ for (boolean debugLogsExist : debugLogsExistArray) {
+
+ conf.setBoolean(YarnConfiguration.NM_LOG_CONTAINER_DEBUG_INFO,
+ debugLogsExist);
+ FileOutputStream fos = new FileOutputStream(tempFile);
+ ContainerExecutor exec = new DefaultContainerExecutor();
+ exec.setConf(conf);
+ exec.writeLaunchEnv(fos, env, resources, commands,
+ new Path(localLogDir.getAbsolutePath()), tempFile.getName());
+ fos.flush();
+ fos.close();
+ FileUtil.setExecutable(tempFile, true);
+
+ Shell.ShellCommandExecutor shexc = new Shell.ShellCommandExecutor(
+ new String[] { tempFile.getAbsolutePath() }, tmpDir);
+
+ shexc.execute();
+ assertEquals(shexc.getExitCode(), 0);
+ File directorInfo =
+ new File(localLogDir, ContainerExecutor.DIRECTORY_CONTENTS);
+ File scriptCopy = new File(localLogDir, tempFile.getName());
+
+ Assert.assertEquals("Directory info file missing", debugLogsExist,
+ directorInfo.exists());
+ Assert.assertEquals("Copy of launch script missing", debugLogsExist,
+ scriptCopy.exists());
+ if (debugLogsExist) {
+ Assert.assertTrue("Directory info file size is 0",
+ directorInfo.length() > 0);
+ Assert.assertTrue("Size of copy of launch script is 0",
+ scriptCopy.length() > 0);
+ }
+ }
+ } finally {
+ // cleanup
+ if (shellFile != null && shellFile.exists()) {
+ shellFile.delete();
+ }
+ if (tempFile != null && tempFile.exists()) {
+ tempFile.delete();
+ }
+ }
+ }
}
[03/20] hadoop git commit: YARN-4398. Remove unnecessary
synchronization in RMStateStore. Contributed by Ning Ding
Posted by wa...@apache.org.
YARN-4398. Remove unnecessary synchronization in RMStateStore. Contributed by Ning Ding
(cherry picked from commit 48cbde908319775b82ae30f262cca0503866e88e)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c229105
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c229105
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c229105
Branch: refs/heads/branch-2.8
Commit: 9c2291057071ca8d44199c39b28d536801834399
Parents: cef21b7
Author: Jian He <ji...@apache.org>
Authored: Wed Dec 2 11:07:18 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:33:53 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../org/apache/hadoop/yarn/event/AsyncDispatcher.java | 5 +----
.../server/resourcemanager/recovery/RMStateStore.java | 10 +++++-----
3 files changed, 8 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c229105/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1d2a618..02dbf3d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1099,6 +1099,8 @@ Release 2.7.3 - UNRELEASED
YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R
via junping_du)
+ YARN-4398. Remove unnecessary synchronization in RMStateStore. (Ning Ding via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c229105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
index 6cdfaad..403381b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
@@ -64,7 +64,7 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
// For drainEventsOnStop enabled only, block newly coming events into the
// queue while stopping.
private volatile boolean blockNewEvents = false;
- private EventHandler handlerInstance = null;
+ private final EventHandler handlerInstance = new GenericEventHandler();
private Thread eventHandlingThread;
protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
@@ -224,9 +224,6 @@ public class AsyncDispatcher extends AbstractService implements Dispatcher {
@Override
public EventHandler getEventHandler() {
- if (handlerInstance == null) {
- handlerInstance = new GenericEventHandler();
- }
return handlerInstance;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c229105/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
index bced5b8..ec42cbe 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStore.java
@@ -737,7 +737,7 @@ public abstract class RMStateStore extends AbstractService {
* RMAppStoredEvent will be sent on completion to notify the RMApp
*/
@SuppressWarnings("unchecked")
- public synchronized void storeNewApplication(RMApp app) {
+ public void storeNewApplication(RMApp app) {
ApplicationSubmissionContext context = app
.getApplicationSubmissionContext();
assert context instanceof ApplicationSubmissionContextPBImpl;
@@ -748,7 +748,7 @@ public abstract class RMStateStore extends AbstractService {
}
@SuppressWarnings("unchecked")
- public synchronized void updateApplicationState(
+ public void updateApplicationState(
ApplicationStateData appState) {
dispatcher.getEventHandler().handle(new RMStateUpdateAppEvent(appState));
}
@@ -780,7 +780,7 @@ public abstract class RMStateStore extends AbstractService {
* This does not block the dispatcher threads
* RMAppAttemptStoredEvent will be sent on completion to notify the RMAppAttempt
*/
- public synchronized void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
+ public void storeNewApplicationAttempt(RMAppAttempt appAttempt) {
Credentials credentials = getCredentialsFromAppAttempt(appAttempt);
AggregateAppResourceUsage resUsage =
@@ -798,7 +798,7 @@ public abstract class RMStateStore extends AbstractService {
}
@SuppressWarnings("unchecked")
- public synchronized void updateApplicationAttemptState(
+ public void updateApplicationAttemptState(
ApplicationAttemptStateData attemptState) {
dispatcher.getEventHandler().handle(
new RMStateUpdateAppAttemptEvent(attemptState));
@@ -963,7 +963,7 @@ public abstract class RMStateStore extends AbstractService {
* There is no notification of completion for this operation.
*/
@SuppressWarnings("unchecked")
- public synchronized void removeApplication(RMApp app) {
+ public void removeApplication(RMApp app) {
ApplicationStateData appState =
ApplicationStateData.newInstance(app.getSubmitTime(),
app.getStartTime(), app.getApplicationSubmissionContext(),
[15/20] hadoop git commit: YARN-4418. AM Resource Limit per partition
can be updated to ResourceUsage as well. (Sunil G via wangda)
Posted by wa...@apache.org.
YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well. (Sunil G via wangda)
(cherry picked from commit 07b0fb996a32020678bd2ce482b672f0434651f0)
(cherry picked from commit c915a658c8a5481ca88d8b2f6ce9a72f2e1f5b6a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d1c6c1f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d1c6c1f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d1c6c1f
Branch: refs/heads/branch-2.8
Commit: 1d1c6c1f8dbb221f41bc5bb87cbea37f71771a41
Parents: c7e10d7
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Dec 14 11:24:30 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:44 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/ResourceUsage.java | 62 ++++++++++++++++++--
.../scheduler/capacity/LeafQueue.java | 1 +
.../scheduler/capacity/QueueCapacities.java | 9 +++
.../scheduler/TestResourceUsage.java | 22 +++++--
.../scheduler/capacity/TestQueueCapacities.java | 3 +-
6 files changed, 88 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6d3f519..155ba2a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.8.0 - UNRELEASED
YARN-4309. Add container launch related debug information to container logs
when a container fails. (Varun Vasudev via wangda)
+ YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well.
+ (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
index 88e93c1..2857379 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceUsage.java
@@ -61,7 +61,7 @@ public class ResourceUsage {
//CACHED_USED and CACHED_PENDING may be read by anyone, but must only
//be written by ordering policies
USED(0), PENDING(1), AMUSED(2), RESERVED(3), CACHED_USED(4),
- CACHED_PENDING(5);
+ CACHED_PENDING(5), AMLIMIT(6);
private int idx;
@@ -92,6 +92,7 @@ public class ResourceUsage {
sb.append("pending=" + resArr[1] + "%, ");
sb.append("am_used=" + resArr[2] + "%, ");
sb.append("reserved=" + resArr[3] + "%}");
+ sb.append("am_limit=" + resArr[6] + "%, ");
return sb.toString();
}
}
@@ -106,11 +107,19 @@ public class ResourceUsage {
public Resource getUsed(String label) {
return _get(label, ResourceType.USED);
}
-
+
+ public Resource getCachedUsed() {
+ return _get(NL, ResourceType.CACHED_USED);
+ }
+
public Resource getCachedUsed(String label) {
return _get(label, ResourceType.CACHED_USED);
}
-
+
+ public Resource getCachedPending() {
+ return _get(NL, ResourceType.CACHED_PENDING);
+ }
+
public Resource getCachedPending(String label) {
return _get(label, ResourceType.CACHED_PENDING);
}
@@ -149,15 +158,23 @@ public class ResourceUsage {
public void setUsed(String label, Resource res) {
_set(label, ResourceType.USED, res);
}
-
+
public void setCachedUsed(String label, Resource res) {
_set(label, ResourceType.CACHED_USED, res);
}
-
+
+ public void setCachedUsed(Resource res) {
+ _set(NL, ResourceType.CACHED_USED, res);
+ }
+
public void setCachedPending(String label, Resource res) {
_set(label, ResourceType.CACHED_PENDING, res);
}
+ public void setCachedPending(Resource res) {
+ _set(NL, ResourceType.CACHED_PENDING, res);
+ }
+
/*
* Pending
*/
@@ -263,6 +280,41 @@ public class ResourceUsage {
_set(label, ResourceType.AMUSED, res);
}
+ /*
+ * AM-Resource Limit
+ */
+ public Resource getAMLimit() {
+ return getAMLimit(NL);
+ }
+
+ public Resource getAMLimit(String label) {
+ return _get(label, ResourceType.AMLIMIT);
+ }
+
+ public void incAMLimit(String label, Resource res) {
+ _inc(label, ResourceType.AMLIMIT, res);
+ }
+
+ public void incAMLimit(Resource res) {
+ incAMLimit(NL, res);
+ }
+
+ public void decAMLimit(Resource res) {
+ decAMLimit(NL, res);
+ }
+
+ public void decAMLimit(String label, Resource res) {
+ _dec(label, ResourceType.AMLIMIT, res);
+ }
+
+ public void setAMLimit(Resource res) {
+ setAMLimit(NL, res);
+ }
+
+ public void setAMLimit(String label, Resource res) {
+ _set(label, ResourceType.AMLIMIT, res);
+ }
+
private static Resource normalize(Resource res) {
if (res == null) {
return Resources.none();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index f58aba2..b74ce45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -605,6 +605,7 @@ public class LeafQueue extends AbstractCSQueue {
minimumAllocation);
metrics.setAMResouceLimit(amResouceLimit);
+ queueUsage.setAMLimit(nodePartition, amResouceLimit);
return amResouceLimit;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
index 65100f6..f2c2632 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/QueueCapacities.java
@@ -217,6 +217,11 @@ public class QueueCapacities {
}
/* Absolute Maximum AM resource percentage Getter and Setter */
+
+ public float getMaxAMResourcePercentage() {
+ return _get(NL, CapacityType.MAX_AM_PERC);
+ }
+
public float getMaxAMResourcePercentage(String label) {
return _get(label, CapacityType.MAX_AM_PERC);
}
@@ -225,6 +230,10 @@ public class QueueCapacities {
_set(label, CapacityType.MAX_AM_PERC, value);
}
+ public void setMaxAMResourcePercentage(float value) {
+ _set(NL, CapacityType.MAX_AM_PERC, value);
+ }
+
/**
* Clear configurable fields, like
* (absolute)capacity/(absolute)maximum-capacity, this will be used by queue
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
index f0bf892..1ddeb0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestResourceUsage.java
@@ -37,8 +37,8 @@ public class TestResourceUsage {
@Parameterized.Parameters
public static Collection<String[]> getParameters() {
- return Arrays.asList(new String[][] { { "Pending" }, { "Used" },
- { "Reserved" }, { "AMUsed" } });
+ return Arrays.asList(new String[][]{{"Pending"}, {"Used"}, {"Reserved"},
+ {"AMUsed"}, {"AMLimit"}, {"CachedUsed"}, {"CachedPending"}});
}
public TestResourceUsage(String suffix) {
@@ -112,16 +112,26 @@ public class TestResourceUsage {
check(0, 0, res);
// Add 1,1 should returns 1,1
- inc(usage, suffix, Resource.newInstance(1, 1), label);
- check(1, 1, get(usage, suffix, label));
+ try {
+ inc(usage, suffix, Resource.newInstance(1, 1), label);
+ check(1, 1, get(usage, suffix, label));
+ } catch (NoSuchMethodException e) {
+ // Few operations need not have to be verified as some resources doesn't
+ // inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
+ }
// Set 2,2
set(usage, suffix, Resource.newInstance(2, 2), label);
check(2, 2, get(usage, suffix, label));
// dec 2,2
- dec(usage, suffix, Resource.newInstance(2, 2), label);
- check(0, 0, get(usage, suffix, label));
+ try {
+ dec(usage, suffix, Resource.newInstance(2, 2), label);
+ check(0, 0, get(usage, suffix, label));
+ } catch (NoSuchMethodException e) {
+ // Few operations need not have to be verified, as some resources doesn't
+ // inc/dec apis exposed (For Eg: CachedUsed and CachedPending).
+ }
}
void check(int mem, int cpu, Resource res) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d1c6c1f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
index d543f80..9d2fa15 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueCapacities.java
@@ -43,7 +43,8 @@ public class TestQueueCapacities {
{ "UsedCapacity" },
{ "AbsoluteUsedCapacity" },
{ "MaximumCapacity" },
- { "AbsoluteMaximumCapacity" } });
+ { "AbsoluteMaximumCapacity" },
+ { "MaxAMResourcePercentage" } });
}
public TestQueueCapacities(String suffix) {
[09/20] hadoop git commit: YARN-4392. ApplicationCreatedEvent event
time resets after RM restart/failover. Contributed by Naganarasimha G R and
Xuan Gong
Posted by wa...@apache.org.
YARN-4392. ApplicationCreatedEvent event time resets after RM
restart/failover. Contributed by Naganarasimha G R and Xuan Gong
(cherry picked from commit 4546c7582b6762c18ba150d80a8976eb51a8290c)
(cherry picked from commit 7decba83ef541b1f89745ad040c6137d24b187b0)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/46cb47c0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/46cb47c0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/46cb47c0
Branch: refs/heads/branch-2.8
Commit: 46cb47c03eefef5403c8826acc20df380b348aaf
Parents: f3ee0e7
Author: Xuan <xg...@apache.org>
Authored: Mon Dec 7 12:24:55 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:43 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/resourcemanager/RMAppManager.java | 1 -
.../server/resourcemanager/rmapp/RMAppImpl.java | 14 ++++++++---
.../server/resourcemanager/TestRMRestart.java | 25 +++++++++++++++++---
.../rmapp/TestRMAppTransitions.java | 4 +---
5 files changed, 37 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46cb47c0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 727868d..cb86de2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1063,6 +1063,9 @@ Release 2.8.0 - UNRELEASED
YARN-4452. NPE when submit Unmanaged application. (Naganarasimha G R
via junping_du)
+ YARN-4392. ApplicationCreatedEvent event time resets after RM restart/failover.
+ (Naganarasimha G R and Xuan Gong via xgong)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46cb47c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
index c9ea1b8..711a7a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMAppManager.java
@@ -26,7 +26,6 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
-import org.apache.hadoop.ipc.CallerContext;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.StringUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46cb47c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index ff31643..b1a5814 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -444,9 +444,6 @@ public class RMAppImpl implements RMApp, Recoverable {
this.callerContext = CallerContext.getCurrent();
- rmContext.getRMApplicationHistoryWriter().applicationStarted(this);
- rmContext.getSystemMetricsPublisher().appCreated(this, startTime);
-
long localLogAggregationStatusTimeout =
conf.getLong(YarnConfiguration.LOG_AGGREGATION_STATUS_TIME_OUT_MS,
YarnConfiguration.DEFAULT_LOG_AGGREGATION_STATUS_TIME_OUT_MS);
@@ -813,6 +810,9 @@ public class RMAppImpl implements RMApp, Recoverable {
this.startTime = appState.getStartTime();
this.callerContext = appState.getCallerContext();
+ // send the ATS create Event
+ sendATSCreateEvent(this, this.startTime);
+
for(int i=0; i<appState.getAttemptCount(); ++i) {
// create attempt
createNewAttempt();
@@ -1074,6 +1074,9 @@ public class RMAppImpl implements RMApp, Recoverable {
// communication
LOG.info("Storing application with id " + app.applicationId);
app.rmContext.getStateStore().storeNewApplication(app);
+
+ // send the ATS create Event
+ app.sendATSCreateEvent(app, app.startTime);
}
}
@@ -1724,4 +1727,9 @@ public class RMAppImpl implements RMApp, Recoverable {
public CallerContext getCallerContext() {
return callerContext;
}
+
+ private void sendATSCreateEvent(RMApp app, long startTime) {
+ rmContext.getRMApplicationHistoryWriter().applicationStarted(app);
+ rmContext.getSystemMetricsPublisher().appCreated(app, startTime);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46cb47c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index cd84208..d2b8eee 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -18,15 +18,15 @@
package org.apache.hadoop.yarn.server.resourcemanager;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.timeout;
-import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import com.google.common.base.Supplier;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
@@ -87,6 +87,7 @@ import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
+import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
@@ -113,7 +114,9 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
+import com.google.common.base.Supplier;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Sets;
@@ -896,7 +899,13 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
memStore.init(conf);
// start RM
- MockRM rm1 = createMockRM(conf, memStore);
+ MockRM rm1 = new MockRM(conf, memStore) {
+ @Override
+ protected SystemMetricsPublisher createSystemMetricsPublisher() {
+ return spy(super.createSystemMetricsPublisher());
+ }
+ };
+ rms.add(rm1);
rm1.start();
MockNM nm1 =
new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
@@ -925,6 +934,8 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.KILLED);
+ verify(rm1.getRMContext().getSystemMetricsPublisher(),Mockito.times(3))
+ .appCreated(any(RMApp.class), anyLong());
// restart rm
MockRM rm2 = new MockRM(conf, memStore) {
@@ -932,10 +943,18 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
protected RMAppManager createRMAppManager() {
return spy(super.createRMAppManager());
}
+
+ @Override
+ protected SystemMetricsPublisher createSystemMetricsPublisher() {
+ return spy(super.createSystemMetricsPublisher());
+ }
};
rms.add(rm2);
rm2.start();
+ verify(rm2.getRMContext().getSystemMetricsPublisher(),Mockito.times(3))
+ .appCreated(any(RMApp.class), anyLong());
+
GetApplicationsRequest request1 =
GetApplicationsRequest.newInstance(EnumSet.of(
YarnApplicationState.FINISHED, YarnApplicationState.KILLED,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/46cb47c0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
index 280294f..d9f452a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.rmapp;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Mockito.doReturn;
@@ -64,6 +63,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.ahs.RMApplicationHistoryWri
import org.apache.hadoop.yarn.server.resourcemanager.metrics.SystemMetricsPublisher;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AMLivelinessMonitor;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEvent;
@@ -368,8 +368,6 @@ public class TestRMAppTransitions {
protected RMApp testCreateAppNewSaving(
ApplicationSubmissionContext submissionContext) throws IOException {
RMApp application = createNewTestApp(submissionContext);
- verify(writer).applicationStarted(any(RMApp.class));
- verify(publisher).appCreated(any(RMApp.class), anyLong());
// NEW => NEW_SAVING event RMAppEventType.START
RMAppEvent event =
new RMAppEvent(application.getApplicationId(), RMAppEventType.START);
[07/20] hadoop git commit: HDFS-9484.
NNThroughputBenchmark$BlockReportStats should not send empty block reports.
Contributed by Mingliang Liu.
Posted by wa...@apache.org.
HDFS-9484. NNThroughputBenchmark$BlockReportStats should not send empty block reports. Contributed by Mingliang Liu.
(cherry picked from commit 33d4588a7c57a0945f82fb14d8127d49c9ec68d2)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0b88a7f2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0b88a7f2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0b88a7f2
Branch: refs/heads/branch-2.8
Commit: 0b88a7f259f86708425cea2feb6ab08c14887c0e
Parents: 69d3a71
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Thu Dec 3 14:51:06 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:42 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/namenode/NNThroughputBenchmark.java | 6 +++---
2 files changed, 6 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b88a7f2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a571a08..baaa58d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1616,6 +1616,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9347. Invariant assumption in TestQuorumJournalManager.shutdown()
is wrong. (Wei-Chiu Chuang via zhz)
+ HDFS-9484. NNThroughputBenchmark$BlockReportStats should not send empty
+ block reports. (Mingliang Liu via shv)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0b88a7f2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index 91f9793..8a594ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -925,7 +925,7 @@ public class NNThroughputBenchmark implements Tool {
NamespaceInfo nsInfo;
DatanodeRegistration dnRegistration;
DatanodeStorage storage; //only one storage
- final ArrayList<BlockReportReplica> blocks;
+ final List<BlockReportReplica> blocks;
int nrBlocks; // actual number of blocks
BlockListAsLongs blockReportList;
final int dnIdx;
@@ -938,7 +938,7 @@ public class NNThroughputBenchmark implements Tool {
TinyDatanode(int dnIdx, int blockCapacity) throws IOException {
this.dnIdx = dnIdx;
- this.blocks = new ArrayList<BlockReportReplica>(blockCapacity);
+ this.blocks = Arrays.asList(new BlockReportReplica[blockCapacity]);
this.nrBlocks = 0;
}
@@ -1013,7 +1013,7 @@ public class NNThroughputBenchmark implements Tool {
Block block = new Block(blocks.size() - idx, 0, 0);
blocks.set(idx, new BlockReportReplica(block));
}
- blockReportList = BlockListAsLongs.EMPTY;
+ blockReportList = BlockListAsLongs.encode(blocks);
}
BlockListAsLongs getBlockReportList() {
[18/20] hadoop git commit: YARN-4416. Deadlock due to synchronised
get Methods in AbstractCSQueue. (Naganarasimha G R via wangda)
Posted by wa...@apache.org.
YARN-4416. Deadlock due to synchronised get Methods in AbstractCSQueue. (Naganarasimha G R via wangda)
(cherry picked from commit 9b856d9787be5ec88ef34574b9b98755d7b669ea)
(cherry picked from commit 5060fd1840ba2962cdfd7d91e2f405d7bdcfd169)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b74391ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b74391ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b74391ce
Branch: refs/heads/branch-2.8
Commit: b74391ce1495b5fd65216a252d8c4f61955d0018
Parents: c4b1386
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Dec 16 13:22:37 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:45 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../scheduler/capacity/AbstractCSQueue.java | 7 +++----
.../scheduler/capacity/LeafQueue.java | 16 ----------------
3 files changed, 6 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b74391ce/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 67a9c54..845a6f2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1083,6 +1083,9 @@ Release 2.8.0 - UNRELEASED
YARN-4225. Add preemption status to yarn queue -status for capacity scheduler.
(Eric Payne via wangda)
+ YARN-4416. Deadlock due to synchronised get Methods in AbstractCSQueue.
+ (Naganarasimha G R via wangda)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b74391ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index 0d70e90..acd7ae9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -46,7 +46,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsMana
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -135,7 +134,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized float getAbsoluteUsedCapacity() {
+ public float getAbsoluteUsedCapacity() {
return queueCapacities.getAbsoluteUsedCapacity();
}
@@ -154,7 +153,7 @@ public abstract class AbstractCSQueue implements CSQueue {
return queueUsage.getUsed();
}
- public synchronized int getNumContainers() {
+ public int getNumContainers() {
return numContainers;
}
@@ -203,7 +202,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Override
- public synchronized void setAbsoluteUsedCapacity(float absUsedCapacity) {
+ public void setAbsoluteUsedCapacity(float absUsedCapacity) {
queueCapacities.setAbsoluteUsedCapacity(absUsedCapacity);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b74391ce/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index b74ce45..be1ba89 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -109,9 +109,6 @@ public class LeafQueue extends AbstractCSQueue {
// cache last cluster resource to compute actual capacity
private Resource lastClusterResource = Resources.none();
- // absolute capacity as a resource (based on cluster resource)
- private Resource absoluteCapacityResource = Resources.none();
-
private final QueueResourceLimitsInfo queueResourceLimitsInfo =
new QueueResourceLimitsInfo();
@@ -144,7 +141,6 @@ public class LeafQueue extends AbstractCSQueue {
super.setupQueueConfigs(clusterResource);
this.lastClusterResource = clusterResource;
- updateAbsoluteCapacityResource(clusterResource);
this.cachedResourceLimitsForHeadroom = new ResourceLimits(clusterResource);
@@ -345,10 +341,6 @@ public class LeafQueue extends AbstractCSQueue {
public synchronized int getNumActiveApplications(String user) {
return getUser(user).getActiveApplications();
}
-
- public synchronized int getNumContainers() {
- return numContainers;
- }
@Override
public synchronized QueueState getState() {
@@ -1327,13 +1319,6 @@ public class LeafQueue extends AbstractCSQueue {
}
}
- private void updateAbsoluteCapacityResource(Resource clusterResource) {
- absoluteCapacityResource =
- Resources.multiplyAndNormalizeUp(resourceCalculator, labelManager
- .getResourceByLabel(RMNodeLabelsManager.NO_LABEL, clusterResource),
- queueCapacities.getAbsoluteCapacity(), minimumAllocation);
- }
-
private void updateCurrentResourceLimits(
ResourceLimits currentResourceLimits, Resource clusterResource) {
// TODO: need consider non-empty node labels when resource limits supports
@@ -1359,7 +1344,6 @@ public class LeafQueue extends AbstractCSQueue {
ResourceLimits currentResourceLimits) {
updateCurrentResourceLimits(currentResourceLimits, clusterResource);
lastClusterResource = clusterResource;
- updateAbsoluteCapacityResource(clusterResource);
// Update headroom info based on new cluster resource value
// absoluteMaxCapacity now, will be replaced with absoluteMaxAvailCapacity
[17/20] hadoop git commit: YARN-4293. ResourceUtilization should be a
part of yarn node CLI. (Sunil G via wangda)
Posted by wa...@apache.org.
YARN-4293. ResourceUtilization should be a part of yarn node CLI. (Sunil G via wangda)
(cherry picked from commit 79c41b1d83e981ae74cb8b58ffcf7907b7612ad4)
(cherry picked from commit 9f4a6ec6630f3776859f8d677532370095a58d0c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ed9eb26
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ed9eb26
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ed9eb26
Branch: refs/heads/branch-2.8
Commit: 9ed9eb265fb1b6500799ea391567cdb16c35f6ca
Parents: e4f638d
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Dec 16 13:18:19 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:45 2015 -0800
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 2 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/api/records/NodeReport.java | 25 ++++
.../yarn/api/records/ResourceUtilization.java | 150 +++++++++++++++++++
.../src/main/proto/yarn_protos.proto | 8 +
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 21 +++
.../hadoop/yarn/client/cli/TestYarnCLI.java | 56 ++++++-
.../api/records/impl/pb/NodeReportPBImpl.java | 73 +++++++++
.../impl/pb/ResourceUtilizationPBImpl.java | 104 +++++++++++++
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +
.../yarn/server/api/records/NodeStatus.java | 1 +
.../server/api/records/ResourceUtilization.java | 133 ----------------
.../api/records/impl/pb/NodeStatusPBImpl.java | 5 +-
.../impl/pb/ResourceUtilizationPBImpl.java | 104 -------------
.../hadoop/yarn/server/utils/BuilderUtils.java | 15 +-
.../main/proto/yarn_server_common_protos.proto | 5 -
.../server/nodemanager/NodeResourceMonitor.java | 2 +-
.../nodemanager/NodeResourceMonitorImpl.java | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../monitor/ContainersMonitor.java | 2 +-
.../monitor/ContainersMonitorImpl.java | 2 +-
.../server/resourcemanager/ClientRMService.java | 5 +-
.../server/resourcemanager/rmnode/RMNode.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 2 +-
.../rmnode/RMNodeStatusEvent.java | 2 +-
.../scheduler/SchedulerNode.java | 2 +-
.../webapp/dao/ResourceUtilizationInfo.java | 2 +-
.../yarn/server/resourcemanager/MockNodes.java | 2 +-
.../webapp/TestRMWebServicesNodes.java | 2 +-
.../TestMiniYarnClusterNodeUtilization.java | 2 +-
31 files changed, 476 insertions(+), 263 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index f5943a8..92d586b 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -34,8 +34,8 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index e778188..a082e20 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -27,8 +27,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4c171d1..37e707b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -551,6 +551,8 @@ Release 2.8.0 - UNRELEASED
YARN-4418. AM Resource Limit per partition can be updated to ResourceUsage as well.
(Sunil G via wangda)
+ YARN-4293. ResourceUtilization should be a part of yarn node CLI. (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
index aac19bd..412010a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/NodeReport.java
@@ -196,4 +196,29 @@ public abstract class NodeReport {
@Private
@Unstable
public abstract void setNodeLabels(Set<String> nodeLabels);
+
+ /**
+ * Get containers aggregated resource utilization in a node
+ * @return containers resource utilization
+ */
+ @Public
+ @Stable
+ public abstract ResourceUtilization getAggregatedContainersUtilization();
+
+ @Private
+ @Unstable
+ public abstract void setAggregatedContainersUtilization(ResourceUtilization
+ containersUtilization);
+
+ /**
+ * Get node resource utilization
+ * @return node resource utilization
+ */
+ @Public
+ @Stable
+ public abstract ResourceUtilization getNodeUtilization();
+
+ @Private
+ @Unstable
+ public abstract void setNodeUtilization(ResourceUtilization nodeUtilization);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
new file mode 100644
index 0000000..5f52f85
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ResourceUtilization.java
@@ -0,0 +1,150 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records;
+
+import org.apache.hadoop.classification.InterfaceAudience.Public;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * <p>
+ * <code>ResourceUtilization</code> models the utilization of a set of computer
+ * resources in the cluster.
+ * </p>
+ */
+@Public
+@Unstable
+public abstract class ResourceUtilization implements
+ Comparable<ResourceUtilization> {
+
+ @Public
+ @Unstable
+ public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) {
+ ResourceUtilization utilization =
+ Records.newRecord(ResourceUtilization.class);
+ utilization.setPhysicalMemory(pmem);
+ utilization.setVirtualMemory(vmem);
+ utilization.setCPU(cpu);
+ return utilization;
+ }
+
+ /**
+ * Get used <em>virtual memory</em>.
+ *
+ * @return <em>virtual memory</em> in MB
+ */
+ @Public
+ @Unstable
+ public abstract int getVirtualMemory();
+
+ /**
+ * Set used <em>virtual memory</em>.
+ *
+ * @param vmem <em>virtual memory</em> in MB
+ */
+ @Public
+ @Unstable
+ public abstract void setVirtualMemory(int vmem);
+
+ /**
+ * Get <em>physical memory</em>.
+ *
+ * @return <em>physical memory</em> in MB
+ */
+ @Public
+ @Unstable
+ public abstract int getPhysicalMemory();
+
+ /**
+ * Set <em>physical memory</em>.
+ *
+ * @param pmem <em>physical memory</em> in MB
+ */
+ @Public
+ @Unstable
+ public abstract void setPhysicalMemory(int pmem);
+
+ /**
+ * Get <em>CPU</em> utilization.
+ *
+ * @return <em>CPU utilization</em> normalized to 1 CPU
+ */
+ @Public
+ @Unstable
+ public abstract float getCPU();
+
+ /**
+ * Set <em>CPU</em> utilization.
+ *
+ * @param cpu <em>CPU utilization</em> normalized to 1 CPU
+ */
+ @Public
+ @Unstable
+ public abstract void setCPU(float cpu);
+
+ @Override
+ public int hashCode() {
+ final int prime = 263167;
+ int result = 3571;
+ result = prime * result + getVirtualMemory();
+ result = prime * result + getPhysicalMemory();
+ result = 31 * result + Float.valueOf(getCPU()).hashCode();
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (!(obj instanceof ResourceUtilization)) {
+ return false;
+ }
+ ResourceUtilization other = (ResourceUtilization) obj;
+ if (getVirtualMemory() != other.getVirtualMemory()
+ || getPhysicalMemory() != other.getPhysicalMemory()
+ || getCPU() != other.getCPU()) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "<pmem:" + getPhysicalMemory() + ", vmem:" + getVirtualMemory()
+ + ", vCores:" + getCPU() + ">";
+ }
+
+ /**
+ * Add utilization to the current one.
+ * @param pmem Physical memory used to add.
+ * @param vmem Virtual memory used to add.
+ * @param cpu CPU utilization to add.
+ */
+ @Public
+ @Unstable
+ public void addTo(int pmem, int vmem, float cpu) {
+ this.setPhysicalMemory(this.getPhysicalMemory() + pmem);
+ this.setVirtualMemory(this.getVirtualMemory() + vmem);
+ this.setCPU(this.getCPU() + cpu);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 28ff576..3c9877b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -58,6 +58,12 @@ message ResourceProto {
optional int32 virtual_cores = 2;
}
+message ResourceUtilizationProto {
+ optional int32 pmem = 1;
+ optional int32 vmem = 2;
+ optional float cpu = 3;
+}
+
message ResourceOptionProto {
optional ResourceProto resource = 1;
optional int32 over_commit_timeout = 2;
@@ -254,6 +260,8 @@ message NodeReportProto {
optional string health_report = 8;
optional int64 last_health_report_time = 9;
repeated string node_labels = 10;
+ optional ResourceUtilizationProto containers_utilization = 11;
+ optional ResourceUtilizationProto node_utilization = 12;
}
message NodeIdToLabelsInfoProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
index 4f0ddfe..c93b2df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/NodeCLI.java
@@ -218,6 +218,27 @@ public class NodeCLI extends YarnCLI {
new ArrayList<String>(report.getNodeLabels());
Collections.sort(nodeLabelsList);
nodeReportStr.println(StringUtils.join(nodeLabelsList.iterator(), ','));
+
+ nodeReportStr.print("\tResource Utilization by Node : ");
+ if (nodeReport.getNodeUtilization() != null) {
+ nodeReportStr.print("PMem:"
+ + nodeReport.getNodeUtilization().getPhysicalMemory()
+ + " MB, VMem:" + nodeReport.getNodeUtilization().getVirtualMemory()
+ + " MB, VCores:" + nodeReport.getNodeUtilization().getCPU());
+ }
+ nodeReportStr.println();
+
+ nodeReportStr.print("\tResource Utilization by Containers : ");
+ if (nodeReport.getAggregatedContainersUtilization() != null) {
+ nodeReportStr.print("PMem:"
+ + nodeReport.getAggregatedContainersUtilization()
+ .getPhysicalMemory()
+ + " MB, VMem:"
+ + nodeReport.getAggregatedContainersUtilization()
+ .getVirtualMemory() + " MB, VCores:"
+ + nodeReport.getAggregatedContainersUtilization().getCPU());
+ }
+ nodeReportStr.println();
}
if (nodeReport == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 9ee7001..4e66600 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -1178,6 +1179,8 @@ public class TestYarnCLI {
pw.println("\tCPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : a,b,c,x,y,z");
+ pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
+ pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
pw.close();
String nodeStatusStr = baos.toString("UTF-8");
verify(sysOut, times(1)).println(isA(String.class));
@@ -1212,6 +1215,44 @@ public class TestYarnCLI {
pw.println("\tCPU-Used : 0 vcores");
pw.println("\tCPU-Capacity : 0 vcores");
pw.println("\tNode-Labels : ");
+ pw.println("\tResource Utilization by Node : PMem:2048 MB, VMem:4096 MB, VCores:8.0");
+ pw.println("\tResource Utilization by Containers : PMem:1024 MB, VMem:2048 MB, VCores:4.0");
+ pw.close();
+ String nodeStatusStr = baos.toString("UTF-8");
+ verify(sysOut, times(1)).println(isA(String.class));
+ verify(sysOut).println(nodeStatusStr);
+ }
+
+ @Test
+ public void testNodeStatusWithEmptyResourceUtilization() throws Exception {
+ NodeId nodeId = NodeId.newInstance("host0", 0);
+ NodeCLI cli = new NodeCLI();
+ when(client.getNodeReports()).thenReturn(
+ getNodeReports(3, NodeState.RUNNING, false, true));
+ cli.setClient(client);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+ int result = cli.run(new String[] { "-status", nodeId.toString() });
+ assertEquals(0, result);
+ verify(client).getNodeReports();
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ PrintWriter pw = new PrintWriter(baos);
+ pw.println("Node Report : ");
+ pw.println("\tNode-Id : host0:0");
+ pw.println("\tRack : rack1");
+ pw.println("\tNode-State : RUNNING");
+ pw.println("\tNode-Http-Address : host1:8888");
+ pw.println("\tLast-Health-Update : "
+ + DateFormatUtils.format(new Date(0), "E dd/MMM/yy hh:mm:ss:SSzz"));
+ pw.println("\tHealth-Report : ");
+ pw.println("\tContainers : 0");
+ pw.println("\tMemory-Used : 0MB");
+ pw.println("\tMemory-Capacity : 0MB");
+ pw.println("\tCPU-Used : 0 vcores");
+ pw.println("\tCPU-Capacity : 0 vcores");
+ pw.println("\tNode-Labels : a,b,c,x,y,z");
+ pw.println("\tResource Utilization by Node : ");
+ pw.println("\tResource Utilization by Containers : ");
pw.close();
String nodeStatusStr = baos.toString("UTF-8");
verify(sysOut, times(1)).println(isA(String.class));
@@ -1463,11 +1504,16 @@ public class TestYarnCLI {
}
private List<NodeReport> getNodeReports(int noOfNodes, NodeState state) {
- return getNodeReports(noOfNodes, state, true);
+ return getNodeReports(noOfNodes, state, true, false);
}
private List<NodeReport> getNodeReports(int noOfNodes, NodeState state,
boolean emptyNodeLabel) {
+ return getNodeReports(noOfNodes, state, emptyNodeLabel, false);
+ }
+
+ private List<NodeReport> getNodeReports(int noOfNodes, NodeState state,
+ boolean emptyNodeLabel, boolean emptyResourceUtilization) {
List<NodeReport> nodeReports = new ArrayList<NodeReport>();
for (int i = 0; i < noOfNodes; i++) {
@@ -1481,6 +1527,14 @@ public class TestYarnCLI {
.newInstance("host" + i, 0), state, "host" + 1 + ":8888",
"rack1", Records.newRecord(Resource.class), Records
.newRecord(Resource.class), 0, "", 0, nodeLabels);
+ if (!emptyResourceUtilization) {
+ ResourceUtilization containersUtilization = ResourceUtilization
+ .newInstance(1024, 2048, 4);
+ ResourceUtilization nodeUtilization = ResourceUtilization.newInstance(
+ 2048, 4096, 8);
+ nodeReport.setAggregatedContainersUtilization(containersUtilization);
+ nodeReport.setNodeUtilization(nodeUtilization);
+ }
nodeReports.add(nodeReport);
}
return nodeReports;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
index 09bad36..1b88ab0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/NodeReportPBImpl.java
@@ -27,10 +27,12 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeReportProtoOrBuilder;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
import com.google.protobuf.TextFormat;
@@ -44,6 +46,8 @@ public class NodeReportPBImpl extends NodeReport {
private NodeId nodeId;
private Resource used;
private Resource capability;
+ private ResourceUtilization containersUtilization = null;
+ private ResourceUtilization nodeUtilization = null;
Set<String> labels;
public NodeReportPBImpl() {
@@ -263,6 +267,18 @@ public class NodeReportPBImpl extends NodeReport {
builder.clearNodeLabels();
builder.addAllNodeLabels(this.labels);
}
+ if (this.nodeUtilization != null
+ && !((ResourceUtilizationPBImpl) this.nodeUtilization).getProto()
+ .equals(builder.getNodeUtilization())) {
+ builder.setNodeUtilization(convertToProtoFormat(this.nodeUtilization));
+ }
+ if (this.containersUtilization != null
+ && !((ResourceUtilizationPBImpl) this.containersUtilization).getProto()
+ .equals(builder.getContainersUtilization())) {
+ builder
+ .setContainersUtilization(convertToProtoFormat(
+ this.containersUtilization));
+ }
}
private void mergeLocalToProto() {
@@ -297,6 +313,15 @@ public class NodeReportPBImpl extends NodeReport {
return ((ResourcePBImpl) r).getProto();
}
+ private ResourceUtilizationPBImpl convertFromProtoFormat(
+ ResourceUtilizationProto p) {
+ return new ResourceUtilizationPBImpl(p);
+ }
+
+ private ResourceUtilizationProto convertToProtoFormat(ResourceUtilization r) {
+ return ((ResourceUtilizationPBImpl) r).getProto();
+ }
+
@Override
public Set<String> getNodeLabels() {
initNodeLabels();
@@ -318,4 +343,52 @@ public class NodeReportPBImpl extends NodeReport {
this.labels = new HashSet<String>();
this.labels.addAll(p.getNodeLabelsList());
}
+
+ @Override
+ public ResourceUtilization getAggregatedContainersUtilization() {
+ if (this.containersUtilization != null) {
+ return this.containersUtilization;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasContainersUtilization()) {
+ return null;
+ }
+ this.containersUtilization = convertFromProtoFormat(p
+ .getContainersUtilization());
+ return this.containersUtilization;
+ }
+
+ @Override
+ public void setAggregatedContainersUtilization(
+ ResourceUtilization containersResourceUtilization) {
+ maybeInitBuilder();
+ if (containersResourceUtilization == null) {
+ builder.clearContainersUtilization();
+ }
+ this.containersUtilization = containersResourceUtilization;
+ }
+
+ @Override
+ public ResourceUtilization getNodeUtilization() {
+ if (this.nodeUtilization != null) {
+ return this.nodeUtilization;
+ }
+
+ NodeReportProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasNodeUtilization()) {
+ return null;
+ }
+ this.nodeUtilization = convertFromProtoFormat(p.getNodeUtilization());
+ return this.nodeUtilization;
+ }
+
+ @Override
+ public void setNodeUtilization(ResourceUtilization nodeResourceUtilization) {
+ maybeInitBuilder();
+ if (nodeResourceUtilization == null) {
+ builder.clearNodeUtilization();
+ }
+ this.nodeUtilization = nodeResourceUtilization;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java
new file mode 100644
index 0000000..e37adbe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceUtilizationPBImpl.java
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.api.records.impl.pb;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProtoOrBuilder;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
+
+@Private
+@Unstable
+public class ResourceUtilizationPBImpl extends ResourceUtilization {
+ private ResourceUtilizationProto proto = ResourceUtilizationProto
+ .getDefaultInstance();
+ private ResourceUtilizationProto.Builder builder = null;
+ private boolean viaProto = false;
+
+ public ResourceUtilizationPBImpl() {
+ builder = ResourceUtilizationProto.newBuilder();
+ }
+
+ public ResourceUtilizationPBImpl(ResourceUtilizationProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public ResourceUtilizationProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = ResourceUtilizationProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public int getPhysicalMemory() {
+ ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getPmem());
+ }
+
+ @Override
+ public void setPhysicalMemory(int pmem) {
+ maybeInitBuilder();
+ builder.setPmem(pmem);
+ }
+
+ @Override
+ public int getVirtualMemory() {
+ ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getVmem());
+ }
+
+ @Override
+ public void setVirtualMemory(int vmem) {
+ maybeInitBuilder();
+ builder.setVmem(vmem);
+ }
+
+ @Override
+ public float getCPU() {
+ ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
+ return p.getCpu();
+ }
+
+ @Override
+ public void setCPU(float cpu) {
+ maybeInitBuilder();
+ builder.setCpu(cpu);
+ }
+
+ @Override
+ public int compareTo(ResourceUtilization other) {
+ int diff = this.getPhysicalMemory() - other.getPhysicalMemory();
+ if (diff == 0) {
+ diff = this.getVirtualMemory() - other.getVirtualMemory();
+ if (diff == 0) {
+ diff = Float.compare(this.getCPU(), other.getCPU());
+ }
+ }
+ return diff;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 5f707b5..db33672 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -142,6 +142,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceBlacklistRequest;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.SerializedException;
import org.apache.hadoop.yarn.api.records.StrictPreemptionContract;
import org.apache.hadoop.yarn.api.records.Token;
@@ -496,6 +497,7 @@ public class TestPBImplRecords {
generateByNewInstance(ReservationRequest.class);
generateByNewInstance(ReservationRequests.class);
generateByNewInstance(ReservationDefinition.class);
+ generateByNewInstance(ResourceUtilization.class);
}
private class GetSetPair {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
index 2d62db5..836cd4b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/NodeStatus.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.util.Records;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceUtilization.java
deleted file mode 100644
index 39896a3..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/ResourceUtilization.java
+++ /dev/null
@@ -1,133 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.api.records;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Evolving;
-import org.apache.hadoop.yarn.util.Records;
-
-/**
- * <p>
- * <code>ResourceUtilization</code> models the utilization of a set of computer
- * resources in the cluster.
- * </p>
- */
-@Private
-@Evolving
-public abstract class ResourceUtilization implements
- Comparable<ResourceUtilization> {
- public static ResourceUtilization newInstance(int pmem, int vmem, float cpu) {
- ResourceUtilization utilization =
- Records.newRecord(ResourceUtilization.class);
- utilization.setPhysicalMemory(pmem);
- utilization.setVirtualMemory(vmem);
- utilization.setCPU(cpu);
- return utilization;
- }
-
- /**
- * Get used <em>virtual memory</em>.
- *
- * @return <em>virtual memory</em> in MB
- */
- public abstract int getVirtualMemory();
-
- /**
- * Set used <em>virtual memory</em>.
- *
- * @param vmem <em>virtual memory</em> in MB
- */
- public abstract void setVirtualMemory(int vmem);
-
- /**
- * Get <em>physical memory</em>.
- *
- * @return <em>physical memory</em> in MB
- */
- public abstract int getPhysicalMemory();
-
- /**
- * Set <em>physical memory</em>.
- *
- * @param pmem <em>physical memory</em> in MB
- */
- public abstract void setPhysicalMemory(int pmem);
-
- /**
- * Get <em>CPU</em> utilization.
- *
- * @return <em>CPU utilization</em> normalized to 1 CPU
- */
- public abstract float getCPU();
-
- /**
- * Set <em>CPU</em> utilization.
- *
- * @param cpu <em>CPU utilization</em> normalized to 1 CPU
- */
- public abstract void setCPU(float cpu);
-
- @Override
- public int hashCode() {
- final int prime = 263167;
- int result = 3571;
- result = prime * result + getVirtualMemory();
- result = prime * result + getPhysicalMemory();
- result = 31 * result + Float.valueOf(getCPU()).hashCode();
- return result;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (this == obj) {
- return true;
- }
- if (obj == null) {
- return false;
- }
- if (!(obj instanceof ResourceUtilization)) {
- return false;
- }
- ResourceUtilization other = (ResourceUtilization) obj;
- if (getVirtualMemory() != other.getVirtualMemory()
- || getPhysicalMemory() != other.getPhysicalMemory()
- || getCPU() != other.getCPU()) {
- return false;
- }
- return true;
- }
-
- @Override
- public String toString() {
- return "<pmem:" + getPhysicalMemory() + ", vmem:" + getVirtualMemory()
- + ", vCores:" + getCPU() + ">";
- }
-
- /**
- * Add utilization to the current one.
- * @param pmem Physical memory used to add.
- * @param vmem Virtual memory used to add.
- * @param cpu CPU utilization to add.
- */
- public void addTo(int pmem, int vmem, float cpu) {
- this.setPhysicalMemory(this.getPhysicalMemory() + pmem);
- this.setVirtualMemory(this.getVirtualMemory() + vmem);
- this.setCPU(this.getCPU() + cpu);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
index e34451d..8dd4832 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/NodeStatusPBImpl.java
@@ -27,10 +27,12 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.impl.pb.ApplicationIdPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerStatusPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.NodeIdPBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.ResourceUtilizationPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ApplicationIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStatusProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
@@ -38,10 +40,9 @@ import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeHealthStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeStatusProtoOrBuilder;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ResourceUtilizationProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.ResourceUtilizationProto;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
public class NodeStatusPBImpl extends NodeStatus {
NodeStatusProto proto = NodeStatusProto.getDefaultInstance();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceUtilizationPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceUtilizationPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceUtilizationPBImpl.java
deleted file mode 100644
index 29f79d4..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/ResourceUtilizationPBImpl.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.api.records.impl.pb;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ResourceUtilizationProto;
-import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.ResourceUtilizationProtoOrBuilder;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
-
-@Private
-@Unstable
-public class ResourceUtilizationPBImpl extends ResourceUtilization {
- private ResourceUtilizationProto proto = ResourceUtilizationProto
- .getDefaultInstance();
- private ResourceUtilizationProto.Builder builder = null;
- private boolean viaProto = false;
-
- public ResourceUtilizationPBImpl() {
- builder = ResourceUtilizationProto.newBuilder();
- }
-
- public ResourceUtilizationPBImpl(ResourceUtilizationProto proto) {
- this.proto = proto;
- viaProto = true;
- }
-
- public ResourceUtilizationProto getProto() {
- proto = viaProto ? proto : builder.build();
- viaProto = true;
- return proto;
- }
-
- private void maybeInitBuilder() {
- if (viaProto || builder == null) {
- builder = ResourceUtilizationProto.newBuilder(proto);
- }
- viaProto = false;
- }
-
- @Override
- public int getPhysicalMemory() {
- ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
- return (p.getPmem());
- }
-
- @Override
- public void setPhysicalMemory(int pmem) {
- maybeInitBuilder();
- builder.setPmem(pmem);
- }
-
- @Override
- public int getVirtualMemory() {
- ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
- return (p.getVmem());
- }
-
- @Override
- public void setVirtualMemory(int vmem) {
- maybeInitBuilder();
- builder.setVmem(vmem);
- }
-
- @Override
- public float getCPU() {
- ResourceUtilizationProtoOrBuilder p = viaProto ? proto : builder;
- return p.getCpu();
- }
-
- @Override
- public void setCPU(float cpu) {
- maybeInitBuilder();
- builder.setCpu(cpu);
- }
-
- @Override
- public int compareTo(ResourceUtilization other) {
- int diff = this.getPhysicalMemory() - other.getPhysicalMemory();
- if (diff == 0) {
- diff = this.getVirtualMemory() - other.getVirtualMemory();
- if (diff == 0) {
- diff = Float.compare(this.getCPU(), other.getCPU());
- }
- }
- return diff;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
index d3a1a1d..4fdd43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/utils/BuilderUtils.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
@@ -180,11 +181,21 @@ public class BuilderUtils {
return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
capability, numContainers, healthReport, lastHealthReportTime, null);
}
-
+
public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
String httpAddress, String rackName, Resource used, Resource capability,
int numContainers, String healthReport, long lastHealthReportTime,
Set<String> nodeLabels) {
+ return newNodeReport(nodeId, nodeState, httpAddress, rackName, used,
+ capability, numContainers, healthReport, lastHealthReportTime,
+ nodeLabels, null, null);
+ }
+
+ public static NodeReport newNodeReport(NodeId nodeId, NodeState nodeState,
+ String httpAddress, String rackName, Resource used, Resource capability,
+ int numContainers, String healthReport, long lastHealthReportTime,
+ Set<String> nodeLabels, ResourceUtilization containersUtilization,
+ ResourceUtilization nodeUtilization) {
NodeReport nodeReport = recordFactory.newRecordInstance(NodeReport.class);
nodeReport.setNodeId(nodeId);
nodeReport.setNodeState(nodeState);
@@ -196,6 +207,8 @@ public class BuilderUtils {
nodeReport.setHealthReport(healthReport);
nodeReport.setLastHealthReportTime(lastHealthReportTime);
nodeReport.setNodeLabels(nodeLabels);
+ nodeReport.setAggregatedContainersUtilization(containersUtilization);
+ nodeReport.setNodeUtilization(nodeUtilization);
return nodeReport;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
index b161f5b..77064a0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
@@ -57,8 +57,3 @@ message VersionProto {
optional int32 minor_version = 2;
}
-message ResourceUtilizationProto {
- optional int32 pmem = 1;
- optional int32 vmem = 2;
- optional float cpu = 3;
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
index 6c5a15a..9d8b80d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitor.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
/**
* Interface for monitoring the resources of a node.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index dcdaa0a..b161f14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -23,7 +23,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index ba915c2..5806731 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
@@ -74,7 +75,6 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.NodeManager.NMContext;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
index f0dd2e7..4d69dbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitor.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import org.apache.hadoop.service.Service;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.server.nodemanager.ResourceView;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 82ad53e..446e7a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -34,7 +34,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
index 701cd74..1952427 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ClientRMService.java
@@ -988,13 +988,14 @@ public class ClientRMService extends AbstractService implements
used = schedulerNodeReport.getUsedResource();
numContainers = schedulerNodeReport.getNumContainers();
}
-
+
NodeReport report =
BuilderUtils.newNodeReport(rmNode.getNodeID(), rmNode.getState(),
rmNode.getHttpAddress(), rmNode.getRackName(), used,
rmNode.getTotalCapability(), numContainers,
rmNode.getHealthReport(), rmNode.getLastHealthReportTime(),
- rmNode.getNodeLabels());
+ rmNode.getNodeLabels(), rmNode.getAggregatedContainersUtilization(),
+ rmNode.getNodeUtilization());
return report;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 1a172e8..d8df9f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -29,8 +29,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
/**
* Node managers information on available resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 6b0e87b..8448287 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -57,7 +58,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.ClusterMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.NodesListManagerEventType;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
index afc417d..ba6ac9b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
@@ -25,11 +25,11 @@ import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
public class RMNodeStatusEvent extends RMNodeEvent {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
index e8e1238..03edd40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerNode.java
@@ -33,9 +33,9 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerState;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
index 67d0d64..3f3847e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
@@ -22,7 +22,7 @@ import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 96207f3..89aff29 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -30,11 +30,11 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 462ce50..3fd1fd5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -34,9 +34,9 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ed9eb26/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
index 77f6492..cf83e67 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestMiniYarnClusterNodeUtilization.java
@@ -28,6 +28,7 @@ import java.util.ArrayList;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.MiniYARNCluster.CustomNodeManager;
@@ -36,7 +37,6 @@ import org.apache.hadoop.yarn.server.api.ServerRMProxy;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
-import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.nodemanager.NodeStatusUpdater;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
[02/20] hadoop git commit: YARN-4292. ResourceUtilization should be a
part of NodeInfo REST API. (Sunil G via wangda)
Posted by wa...@apache.org.
YARN-4292. ResourceUtilization should be a part of NodeInfo REST API. (Sunil G via wangda)
(cherry picked from commit a2c3bfc8c1349102a7f2bc4ea96b80b429ac227b)
(cherry picked from commit 51e448d295f6d2214c1d6455446a8e05e8a967b1)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/15b3ff19
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/15b3ff19
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/15b3ff19
Branch: refs/heads/branch-2.8
Commit: 15b3ff19715cd671ed782ce2e193eaf139f768fd
Parents: 9c22910
Author: Wangda Tan <wa...@apache.org>
Authored: Thu Dec 3 14:28:00 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:33:53 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../resourcemanager/webapp/dao/NodeInfo.java | 8 ++
.../webapp/dao/ResourceUtilizationInfo.java | 89 ++++++++++++++++++++
.../webapp/TestRMWebServicesNodes.java | 79 +++++++++++++++--
4 files changed, 174 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 02dbf3d..165fa47 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -536,6 +536,9 @@ Release 2.8.0 - UNRELEASED
YARN-4248. REST API for submit/update/delete Reservations. (curino)
+ YARN-4292. ResourceUtilization should be a part of NodeInfo REST API.
+ (Sunil G via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
index 3104117..0f877f8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/NodeInfo.java
@@ -50,6 +50,7 @@ public class NodeInfo {
protected long usedVirtualCores;
protected long availableVirtualCores;
protected ArrayList<String> nodeLabels = new ArrayList<String>();
+ protected ResourceUtilizationInfo resourceUtilization;
public NodeInfo() {
} // JAXB needs this
@@ -82,6 +83,9 @@ public class NodeInfo {
nodeLabels.addAll(labelSet);
Collections.sort(nodeLabels);
}
+
+ // update node and containers resource utilization
+ this.resourceUtilization = new ResourceUtilizationInfo(ni);
}
public String getRack() {
@@ -139,4 +143,8 @@ public class NodeInfo {
public ArrayList<String> getNodeLabels() {
return this.nodeLabels;
}
+
+ public ResourceUtilizationInfo getResourceUtilization() {
+ return this.resourceUtilization;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
new file mode 100644
index 0000000..67d0d64
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/ResourceUtilizationInfo.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.webapp.dao;
+
+import javax.xml.bind.annotation.XmlAccessType;
+import javax.xml.bind.annotation.XmlAccessorType;
+import javax.xml.bind.annotation.XmlRootElement;
+
+import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+
+/**
+ * DAO object represents resource utilization of node and containers.
+ */
+@XmlRootElement
+@XmlAccessorType(XmlAccessType.FIELD)
+public class ResourceUtilizationInfo {
+
+ protected int nodePhysicalMemoryMB;
+ protected int nodeVirtualMemoryMB;
+ protected double nodeCPUUsage;
+ protected int aggregatedContainersPhysicalMemoryMB;
+ protected int aggregatedContainersVirtualMemoryMB;
+ protected double containersCPUUsage;
+
+ public ResourceUtilizationInfo() {
+ } // JAXB needs this
+
+ public ResourceUtilizationInfo(RMNode ni) {
+
+ // update node and containers resource utilization
+ ResourceUtilization nodeUtilization = ni.getNodeUtilization();
+ if (nodeUtilization != null) {
+ this.nodePhysicalMemoryMB = nodeUtilization.getPhysicalMemory();
+ this.nodeVirtualMemoryMB = nodeUtilization.getVirtualMemory();
+ this.nodeCPUUsage = nodeUtilization.getCPU();
+ }
+
+ ResourceUtilization containerAggrUtilization = ni
+ .getAggregatedContainersUtilization();
+ if (containerAggrUtilization != null) {
+ this.aggregatedContainersPhysicalMemoryMB = containerAggrUtilization
+ .getPhysicalMemory();
+ this.aggregatedContainersVirtualMemoryMB = containerAggrUtilization
+ .getVirtualMemory();
+ this.containersCPUUsage = containerAggrUtilization.getCPU();
+ }
+ }
+
+ public int getNodePhysicalMemoryMB() {
+ return nodePhysicalMemoryMB;
+ }
+
+ public int getNodeVirtualMemoryMB() {
+ return nodeVirtualMemoryMB;
+ }
+
+ public int getAggregatedContainersPhysicalMemoryMB() {
+ return aggregatedContainersPhysicalMemoryMB;
+ }
+
+ public int getAggregatedContainersVirtualMemoryMB() {
+ return aggregatedContainersVirtualMemoryMB;
+ }
+
+ public double getNodeCPUUsage() {
+ return nodeCPUUsage;
+ }
+
+ public double getContainersCPUUsage() {
+ return containersCPUUsage;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/15b3ff19/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
index 206edb1..462ce50 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
@@ -639,6 +640,44 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
assertEquals("incorrect number of elements", 3, nodeArray.length());
}
+ @Test
+ public void testNodesResourceUtilization() throws JSONException, Exception {
+ WebResource r = resource();
+ MockNM nm1 = rm.registerNode("h1:1234", 5120);
+ rm.sendNodeStarted(nm1);
+ rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+
+ RMNodeImpl node = (RMNodeImpl) rm.getRMContext().getRMNodes()
+ .get(nm1.getNodeId());
+ NodeHealthStatus nodeHealth = NodeHealthStatus.newInstance(true,
+ "test health report", System.currentTimeMillis());
+ ResourceUtilization nodeResource = ResourceUtilization.newInstance(4096, 0,
+ (float) 10.5);
+ ResourceUtilization containerResource = ResourceUtilization.newInstance(
+ 2048, 0, (float) 5.05);
+ NodeStatus nodeStatus = NodeStatus.newInstance(nm1.getNodeId(), 0,
+ new ArrayList<ContainerStatus>(), null, nodeHealth, containerResource,
+ nodeResource, null);
+ node.handle(new RMNodeStatusEvent(nm1.getNodeId(), nodeStatus, null));
+ rm.NMwaitForState(nm1.getNodeId(), NodeState.RUNNING);
+
+ ClientResponse response = r.path("ws").path("v1").path("cluster")
+ .path("nodes").accept(MediaType.APPLICATION_JSON)
+ .get(ClientResponse.class);
+
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ JSONObject json = response.getEntity(JSONObject.class);
+ assertEquals("incorrect number of elements", 1, json.length());
+ JSONObject nodes = json.getJSONObject("nodes");
+ assertEquals("incorrect number of elements", 1, nodes.length());
+ JSONArray nodeArray = nodes.getJSONArray("node");
+ assertEquals("incorrect number of elements", 1, nodeArray.length());
+ JSONObject info = nodeArray.getJSONObject(0);
+
+ // verify the resource utilization
+ verifyNodeInfo(info, nm1);
+ }
+
public void verifyNodesXML(NodeList nodes, MockNM nm) throws JSONException,
Exception {
for (int i = 0; i < nodes.getLength(); i++) {
@@ -656,14 +695,23 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
WebServicesTestUtils.getXmlLong(element, "availMemoryMB"),
WebServicesTestUtils.getXmlLong(element, "usedVirtualCores"),
WebServicesTestUtils.getXmlLong(element, "availableVirtualCores"),
- WebServicesTestUtils.getXmlString(element, "version"));
+ WebServicesTestUtils.getXmlString(element, "version"),
+ WebServicesTestUtils.getXmlInt(element, "nodePhysicalMemoryMB"),
+ WebServicesTestUtils.getXmlInt(element, "nodeVirtualMemoryMB"),
+ WebServicesTestUtils.getXmlFloat(element, "nodeCPUUsage"),
+ WebServicesTestUtils.getXmlInt(element,
+ "aggregatedContainersPhysicalMemoryMB"),
+ WebServicesTestUtils.getXmlInt(element,
+ "aggregatedContainersVirtualMemoryMB"),
+ WebServicesTestUtils.getXmlFloat(element, "containersCPUUsage"));
}
}
public void verifyNodeInfo(JSONObject nodeInfo, MockNM nm)
throws JSONException, Exception {
- assertEquals("incorrect number of elements", 13, nodeInfo.length());
+ assertEquals("incorrect number of elements", 14, nodeInfo.length());
+ JSONObject resourceInfo = nodeInfo.getJSONObject("resourceUtilization");
verifyNodeInfoGeneric(nm, nodeInfo.getString("state"),
nodeInfo.getString("rack"),
nodeInfo.getString("id"), nodeInfo.getString("nodeHostName"),
@@ -672,15 +720,23 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
nodeInfo.getString("healthReport"), nodeInfo.getInt("numContainers"),
nodeInfo.getLong("usedMemoryMB"), nodeInfo.getLong("availMemoryMB"),
nodeInfo.getLong("usedVirtualCores"), nodeInfo.getLong("availableVirtualCores"),
- nodeInfo.getString("version"));
-
+ nodeInfo.getString("version"),
+ resourceInfo.getInt("nodePhysicalMemoryMB"),
+ resourceInfo.getInt("nodeVirtualMemoryMB"),
+ resourceInfo.getDouble("nodeCPUUsage"),
+ resourceInfo.getInt("aggregatedContainersPhysicalMemoryMB"),
+ resourceInfo.getInt("aggregatedContainersVirtualMemoryMB"),
+ resourceInfo.getDouble("containersCPUUsage"));
}
public void verifyNodeInfoGeneric(MockNM nm, String state, String rack,
String id, String nodeHostName,
String nodeHTTPAddress, long lastHealthUpdate, String healthReport,
int numContainers, long usedMemoryMB, long availMemoryMB, long usedVirtualCores,
- long availVirtualCores, String version)
+ long availVirtualCores, String version, int nodePhysicalMemoryMB,
+ int nodeVirtualMemoryMB, double nodeCPUUsage,
+ int containersPhysicalMemoryMB, int containersVirtualMemoryMB,
+ double containersCPUUsage)
throws JSONException, Exception {
RMNode node = rm.getRMContext().getRMNodes().get(nm.getNodeId());
@@ -701,6 +757,19 @@ public class TestRMWebServicesNodes extends JerseyTestBase {
expectedHttpAddress, nodeHTTPAddress);
WebServicesTestUtils.checkStringMatch("version",
node.getNodeManagerVersion(), version);
+ if (node.getNodeUtilization() != null) {
+ ResourceUtilization nodeResource = ResourceUtilization.newInstance(
+ nodePhysicalMemoryMB, nodeVirtualMemoryMB, (float) nodeCPUUsage);
+ assertEquals("nodeResourceUtilization doesn't match",
+ node.getNodeUtilization(), nodeResource);
+ }
+ if (node.getAggregatedContainersUtilization() != null) {
+ ResourceUtilization containerResource = ResourceUtilization.newInstance(
+ containersPhysicalMemoryMB, containersVirtualMemoryMB,
+ (float) containersCPUUsage);
+ assertEquals("containerResourceUtilization doesn't match",
+ node.getAggregatedContainersUtilization(), containerResource);
+ }
long expectedHealthUpdate = node.getLastHealthReportTime();
assertEquals("lastHealthUpdate doesn't match, got: " + lastHealthUpdate
[19/20] hadoop git commit: YARN-4225. Add preemption status to yarn
queue -status for capacity scheduler. (Eric Payne via wangda)
Posted by wa...@apache.org.
YARN-4225. Add preemption status to yarn queue -status for capacity scheduler. (Eric Payne via wangda)
(cherry picked from commit 7faa406f27f687844c941346f59a27a375af3233)
(cherry picked from commit 2f5435acbda53470724d833cfa1c58dc3c3c9bb1)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4b1386b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4b1386b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4b1386b
Branch: refs/heads/branch-2.8
Commit: c4b1386b4ed9e1afb50aa10755bc99fc333cf4cd
Parents: 9ed9eb2
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Dec 16 13:19:40 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:45 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../dev-support/findbugs-exclude.xml | 5 +
.../hadoop/yarn/api/records/QueueInfo.java | 16 +++-
.../src/main/proto/yarn_protos.proto | 1 +
.../apache/hadoop/yarn/client/cli/QueueCLI.java | 6 ++
.../hadoop/yarn/client/ProtocolHATestBase.java | 2 +-
.../hadoop/yarn/client/cli/TestYarnCLI.java | 99 +++++++++++++++++++-
.../api/records/impl/pb/QueueInfoPBImpl.java | 13 +++
.../hadoop/yarn/api/TestPBImplRecords.java | 2 +-
.../scheduler/capacity/AbstractCSQueue.java | 1 +
.../TestSchedulerApplicationAttempt.java | 3 +-
11 files changed, 145 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 37e707b..67a9c54 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1080,6 +1080,9 @@ Release 2.8.0 - UNRELEASED
YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the
lastScheduler time. (Lin Yiqun via zxu)
+ YARN-4225. Add preemption status to yarn queue -status for capacity scheduler.
+ (Eric Payne via wangda)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 133e95d..2d0d5d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -500,4 +500,9 @@
</Or>
<Bug pattern="URF_UNREAD_PUBLIC_OR_PROTECTED_FIELD" />
</Match>
+
+ <Match>
+ <Package name="org.apache.hadoop.yarn.api.records.impl.pb" />
+ <Bug pattern="NP_BOOLEAN_RETURN_NULL" />
+ </Match>
</FindBugsFilter>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
index 176af84..7816feb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/QueueInfo.java
@@ -55,7 +55,8 @@ public abstract class QueueInfo {
float maximumCapacity, float currentCapacity,
List<QueueInfo> childQueues, List<ApplicationReport> applications,
QueueState queueState, Set<String> accessibleNodeLabels,
- String defaultNodeLabelExpression, QueueStatistics queueStatistics) {
+ String defaultNodeLabelExpression, QueueStatistics queueStatistics,
+ boolean preemptionDisabled) {
QueueInfo queueInfo = Records.newRecord(QueueInfo.class);
queueInfo.setQueueName(queueName);
queueInfo.setCapacity(capacity);
@@ -67,6 +68,7 @@ public abstract class QueueInfo {
queueInfo.setAccessibleNodeLabels(accessibleNodeLabels);
queueInfo.setDefaultNodeLabelExpression(defaultNodeLabelExpression);
queueInfo.setQueueStatistics(queueStatistics);
+ queueInfo.setPreemptionDisabled(preemptionDisabled);
return queueInfo;
}
@@ -205,4 +207,16 @@ public abstract class QueueInfo {
@Unstable
public abstract void setQueueStatistics(QueueStatistics queueStatistics);
+ /**
+ * Get the <em>preemption status</em> of the queue.
+ * @return if property is not in proto, return null;
+ * otherwise, return <em>preemption status</em> of the queue
+ */
+ @Public
+ @Stable
+ public abstract Boolean getPreemptionDisabled();
+
+ @Private
+ @Unstable
+ public abstract void setPreemptionDisabled(boolean preemptionDisabled);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 3c9877b..3c208e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -416,6 +416,7 @@ message QueueInfoProto {
repeated string accessibleNodeLabels = 8;
optional string defaultNodeLabelExpression = 9;
optional QueueStatisticsProto queueStatistics = 10;
+ optional bool preemptionDisabled = 11;
}
enum QueueACLProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
index b5db536..330b081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/QueueCLI.java
@@ -152,5 +152,11 @@ public class QueueCLI extends YarnCLI {
labelList.append(nodeLabel);
}
writer.println(labelList.toString());
+
+ Boolean preemptStatus = queueInfo.getPreemptionDisabled();
+ if (preemptStatus != null) {
+ writer.print("\tPreemption : ");
+ writer.println(preemptStatus ? "disabled" : "enabled");
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
index 75e6cee..45629b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java
@@ -665,7 +665,7 @@ public abstract class ProtocolHATestBase extends ClientBaseWithFixes {
public QueueInfo createFakeQueueInfo() {
return QueueInfo.newInstance("root", 100f, 100f, 50f, null,
- createFakeAppReports(), QueueState.RUNNING, null, null, null);
+ createFakeAppReports(), QueueState.RUNNING, null, null, null, false);
}
public List<QueueUserACLInfo> createFakeQueueUserACLInfoList() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
index 4e66600..f30bd40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java
@@ -46,6 +46,7 @@ import java.util.regex.Pattern;
import org.apache.commons.cli.Options;
import org.apache.commons.lang.time.DateFormatUtils;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptReport;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -69,9 +70,15 @@ import org.apache.hadoop.yarn.api.records.SignalContainerCommand;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
import org.apache.hadoop.yarn.client.api.YarnClient;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
import org.apache.hadoop.yarn.exceptions.ContainerNotFoundException;
+import org.apache.hadoop.yarn.server.MiniYARNCluster;
+import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSystemTestUtil;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacitySchedulerConfiguration;
import org.apache.hadoop.yarn.util.Records;
import org.junit.Assert;
import org.junit.Before;
@@ -1324,7 +1331,7 @@ public class TestYarnCLI {
nodeLabels.add("GPU");
nodeLabels.add("JDK_7");
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
- null, null, QueueState.RUNNING, nodeLabels, "GPU", null);
+ null, null, QueueState.RUNNING, nodeLabels, "GPU", null, false);
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "-status", "queueA" });
assertEquals(0, result);
@@ -1339,16 +1346,103 @@ public class TestYarnCLI {
pw.println("\tMaximum Capacity : " + "80.0%");
pw.println("\tDefault Node Label expression : " + "GPU");
pw.println("\tAccessible Node Labels : " + "JDK_7,GPU");
+ pw.println("\tPreemption : " + "enabled");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
}
+
+ @Test
+ public void testGetQueueInfoPreemptionEnabled() throws Exception {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+ conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+ "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity."
+ + "ProportionalCapacityPreemptionPolicy");
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
+
+ YarnClient yarnClient = null;
+ try {
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(yarnConf);
+ yarnClient.start();
+
+ QueueCLI cli = new QueueCLI();
+ cli.setClient(yarnClient);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+ sysOutStream.reset();
+ int result = cli.run(new String[] { "-status", "a1" });
+ assertEquals(0, result);
+ Assert.assertTrue(sysOutStream.toString()
+ .contains("Preemption : enabled"));
+ } finally {
+ // clean-up
+ if (yarnClient != null) {
+ yarnClient.stop();
+ }
+ cluster.stop();
+ cluster.close();
+ }
+ }
+
+ @Test
+ public void testGetQueueInfoPreemptionDisabled() throws Exception {
+ CapacitySchedulerConfiguration conf = new CapacitySchedulerConfiguration();
+ ReservationSystemTestUtil.setupQueueConfiguration(conf);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+ conf.set(YarnConfiguration.RM_SCHEDULER_MONITOR_POLICIES,
+ "org.apache.hadoop.yarn.server.resourcemanager.monitor.capacity."
+ + "ProportionalCapacityPreemptionPolicy");
+ conf.setBoolean(YarnConfiguration.RM_SCHEDULER_ENABLE_MONITORS, true);
+ conf.setBoolean(
+ "yarn.scheduler.capacity.root.a.a1.disable_preemption", true);
+ MiniYARNCluster cluster =
+ new MiniYARNCluster("testReservationAPIs", 2, 1, 1);
+
+ YarnClient yarnClient = null;
+ try {
+ cluster.init(conf);
+ cluster.start();
+ final Configuration yarnConf = cluster.getConfig();
+ yarnClient = YarnClient.createYarnClient();
+ yarnClient.init(yarnConf);
+ yarnClient.start();
+
+ QueueCLI cli = new QueueCLI();
+ cli.setClient(yarnClient);
+ cli.setSysOutPrintStream(sysOut);
+ cli.setSysErrPrintStream(sysErr);
+ sysOutStream.reset();
+ int result = cli.run(new String[] { "-status", "a1" });
+ assertEquals(0, result);
+ Assert.assertTrue(sysOutStream.toString()
+ .contains("Preemption : disabled"));
+ } finally {
+ // clean-up
+ if (yarnClient != null) {
+ yarnClient.stop();
+ }
+ cluster.stop();
+ cluster.close();
+ }
+ }
@Test
public void testGetQueueInfoWithEmptyNodeLabel() throws Exception {
QueueCLI cli = createAndGetQueueCLI();
QueueInfo queueInfo = QueueInfo.newInstance("queueA", 0.4f, 0.8f, 0.5f,
- null, null, QueueState.RUNNING, null, null, null);
+ null, null, QueueState.RUNNING, null, null, null, true);
when(client.getQueueInfo(any(String.class))).thenReturn(queueInfo);
int result = cli.run(new String[] { "-status", "queueA" });
assertEquals(0, result);
@@ -1364,6 +1458,7 @@ public class TestYarnCLI {
pw.println("\tDefault Node Label expression : "
+ NodeLabel.DEFAULT_NODE_LABEL_PARTITION);
pw.println("\tAccessible Node Labels : ");
+ pw.println("\tPreemption : " + "disabled");
pw.close();
String queueInfoStr = baos.toString("UTF-8");
Assert.assertEquals(queueInfoStr, sysOutStream.toString());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
index 90aecf4..605cab1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/QueueInfoPBImpl.java
@@ -395,4 +395,17 @@ public class QueueInfoPBImpl extends QueueInfo {
}
builder.setQueueStatistics(convertToProtoFormat(queueStatistics));
}
+
+ @Override
+ public Boolean getPreemptionDisabled() {
+ QueueInfoProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.hasPreemptionDisabled()) ? p
+ .getPreemptionDisabled() : null;
+ }
+
+ @Override
+ public void setPreemptionDisabled(boolean preemptionDisabled) {
+ maybeInitBuilder();
+ builder.setPreemptionDisabled(preemptionDisabled);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index db33672..b7f5ff7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -488,7 +488,7 @@ public class TestPBImplRecords {
// it is recursive(has sub queues)
typeValueCache.put(QueueInfo.class, QueueInfo.newInstance("root", 1.0f,
1.0f, 0.1f, null, null, QueueState.RUNNING, ImmutableSet.of("x", "y"),
- "x && y", null));
+ "x && y", null, false));
generateByNewInstance(QueueStatistics.class);
generateByNewInstance(QueueUserACLInfo.class);
generateByNewInstance(YarnClusterMetrics.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index b40ac27..0d70e90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -303,6 +303,7 @@ public abstract class AbstractCSQueue implements CSQueue {
queueInfo.setDefaultNodeLabelExpression(defaultLabelExpression);
queueInfo.setCurrentCapacity(getUsedCapacity());
queueInfo.setQueueStatistics(getQueueStatistics());
+ queueInfo.setPreemptionDisabled(preemptionDisabled);
return queueInfo;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4b1386b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
index 95d7129..ddf3ccf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestSchedulerApplicationAttempt.java
@@ -149,7 +149,8 @@ public class TestSchedulerApplicationAttempt {
private Queue createQueue(String name, Queue parent, float capacity) {
QueueMetrics metrics = QueueMetrics.forQueue(name, parent, false, conf);
- QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null, null, QueueState.RUNNING, null, "", null);
+ QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
+ null, QueueState.RUNNING, null, "", null, false);
ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
Queue queue = mock(Queue.class);
when(queue.getMetrics()).thenReturn(metrics);
[10/20] hadoop git commit: HDFS-8831. Trash Support for deletion in
HDFS encryption zone. Contributed by Xiaoyu Yao.
Posted by wa...@apache.org.
HDFS-8831. Trash Support for deletion in HDFS encryption zone. Contributed by Xiaoyu Yao.
(cherry picked from commit cbc7b6bf97a80c39d4bbb3005e42dacae6726baf)
(cherry picked from commit eb7f9901b4355c5488172dc93f734ee496b3e4dc)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f3ee0e78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f3ee0e78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f3ee0e78
Branch: refs/heads/branch-2.8
Commit: f3ee0e7857027930a4019bf5c0388bc4332a88b6
Parents: 134195d
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri Dec 4 10:39:45 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:43 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileSystem.java | 51 ++++-
.../org/apache/hadoop/fs/FilterFileSystem.java | 11 ++
.../main/java/org/apache/hadoop/fs/FsShell.java | 12 +-
.../main/java/org/apache/hadoop/fs/Trash.java | 14 +-
.../java/org/apache/hadoop/fs/TrashPolicy.java | 56 +++++-
.../apache/hadoop/fs/TrashPolicyDefault.java | 193 +++++++++++--------
.../org/apache/hadoop/fs/TestHarFileSystem.java | 4 +
.../java/org/apache/hadoop/fs/TestTrash.java | 9 +
.../hadoop/hdfs/DistributedFileSystem.java | 61 ++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 47 +++++
11 files changed, 367 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index ba82329..8ed96bf 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -105,6 +105,8 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 10;
+ public static final String TRASH_PREFIX = ".Trash";
+
/** FileSystem cache */
static final Cache CACHE = new Cache();
@@ -2649,6 +2651,53 @@ public abstract class FileSystem extends Configured implements Closeable {
+ " doesn't support getAllStoragePolicies");
}
+ /**
+ * Get the root directory of Trash for current user when the path specified
+ * is deleted.
+ *
+ * @param path the trash root of the path to be determined.
+ * @return the default implementation returns "/user/$USER/.Trash".
+ * @throws IOException
+ */
+ public Path getTrashRoot(Path path) throws IOException {
+ return this.makeQualified(new Path(getHomeDirectory().toUri().getPath(),
+ TRASH_PREFIX));
+ }
+
+ /**
+ * Get all the trash roots for current user or all users.
+ *
+ * @param allUsers return trash roots for all users if true.
+ * @return all the trash root directories.
+ * Default FileSystem returns .Trash under users' home directories if
+ * /user/$USER/.Trash exists.
+ * @throws IOException
+ */
+ public Collection<FileStatus> getTrashRoots(boolean allUsers)
+ throws IOException {
+ Path userHome = new Path(getHomeDirectory().toUri().getPath());
+ List<FileStatus> ret = new ArrayList<FileStatus>();
+ if (!allUsers) {
+ Path userTrash = new Path(userHome, TRASH_PREFIX);
+ if (exists(userTrash)) {
+ ret.add(getFileStatus(userTrash));
+ }
+ } else {
+ Path homeParent = userHome.getParent();
+ if (exists(homeParent)) {
+ FileStatus[] candidates = listStatus(homeParent);
+ for (FileStatus candidate : candidates) {
+ Path userTrash = new Path(candidate.getPath(), TRASH_PREFIX);
+ if (exists(userTrash)) {
+ candidate.setPath(userTrash);
+ ret.add(candidate);
+ }
+ }
+ }
+ }
+ return ret;
+ }
+
// making it volatile to be able to do a double checked locking
private volatile static boolean FILE_SYSTEMS_LOADED = false;
@@ -3169,7 +3218,7 @@ public abstract class FileSystem extends Configured implements Closeable {
* For each StatisticsData object, we will call accept on the visitor.
* Finally, at the end, we will call aggregate to get the final total.
*
- * @param The visitor to use.
+ * @param visitor to use.
* @return The total.
*/
private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
index 00f6778..53678e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FilterFileSystem.java
@@ -643,4 +643,15 @@ public class FilterFileSystem extends FileSystem {
throws IOException {
return fs.getAllStoragePolicies();
}
+
+ @Override
+ public Path getTrashRoot(Path path) throws IOException {
+ return fs.getTrashRoot(path);
+ }
+
+ @Override
+ public Collection<FileStatus> getTrashRoots(boolean allUsers)
+ throws IOException {
+ return fs.getTrashRoots(allUsers);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index d91866e..706f6a7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -121,11 +121,21 @@ public class FsShell extends Configured implements Tool {
return getTrash().getCurrentTrashDir();
}
+ /**
+ * Returns the current trash location for the path specified
+ * @param path to be deleted
+ * @return path to the trash
+ * @throws IOException
+ */
+ public Path getCurrentTrashDir(Path path) throws IOException {
+ return getTrash().getCurrentTrashDir(path);
+ }
+
// NOTE: Usage/Help are inner classes to allow access to outer methods
// that access commandFactory
/**
- * Display help for commands with their short usage and long description
+ * Display help for commands with their short usage and long description.
*/
protected class Usage extends FsCommand {
public static final String NAME = "usage";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
index 13d0ec3..b771812 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Trash.java
@@ -54,7 +54,7 @@ public class Trash extends Configured {
*/
public Trash(FileSystem fs, Configuration conf) throws IOException {
super(conf);
- trashPolicy = TrashPolicy.getInstance(conf, fs, fs.getHomeDirectory());
+ trashPolicy = TrashPolicy.getInstance(conf, fs);
}
/**
@@ -92,11 +92,7 @@ public class Trash extends Configured {
throw new IOException("Failed to get server trash configuration", e);
}
Trash trash = new Trash(fullyResolvedFs, conf);
- boolean success = trash.moveToTrash(fullyResolvedPath);
- if (success) {
- LOG.info("Moved: '" + p + "' to trash at: " + trash.getCurrentTrashDir());
- }
- return success;
+ return trash.moveToTrash(fullyResolvedPath);
}
/**
@@ -124,7 +120,7 @@ public class Trash extends Configured {
}
/** get the current working directory */
- Path getCurrentTrashDir() {
+ Path getCurrentTrashDir() throws IOException {
return trashPolicy.getCurrentTrashDir();
}
@@ -139,4 +135,8 @@ public class Trash extends Configured {
public Runnable getEmptier() throws IOException {
return trashPolicy.getEmptier();
}
+
+ public Path getCurrentTrashDir(Path path) throws IOException {
+ return trashPolicy.getCurrentTrashDir(path);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
index eab83b3..1d901c1 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicy.java
@@ -38,7 +38,7 @@ public abstract class TrashPolicy extends Configured {
/**
* Used to setup the trash policy. Must be implemented by all TrashPolicy
- * implementations
+ * implementations.
* @param conf the configuration to be used
* @param fs the filesystem to be used
* @param home the home directory
@@ -46,7 +46,19 @@ public abstract class TrashPolicy extends Configured {
public abstract void initialize(Configuration conf, FileSystem fs, Path home);
/**
- * Returns whether the Trash Policy is enabled for this filesystem
+ * Used to setup the trash policy. Must be implemented by all TrashPolicy
+ * implementations. Different from initialize(conf, fs, home), this one does
+ * not assume trash always under /user/$USER due to HDFS encryption zone.
+ * @param conf the configuration to be used
+ * @param fs the filesystem to be used
+ * @throws IOException
+ */
+ public void initialize(Configuration conf, FileSystem fs) throws IOException{
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Returns whether the Trash Policy is enabled for this filesystem.
*/
public abstract boolean isEnabled();
@@ -68,8 +80,27 @@ public abstract class TrashPolicy extends Configured {
/**
* Get the current working directory of the Trash Policy
+ * This API does not work with files deleted from encryption zone when HDFS
+ * data encryption at rest feature is enabled as rename file between
+ * encryption zones or encryption zone and non-encryption zone is not allowed.
+ *
+ * The caller is recommend to use the new API
+ * TrashPolicy#getCurrentTrashDir(Path path).
+ * It returns the trash location correctly for the path specified no matter
+ * the path is in encryption zone or not.
+ */
+ public abstract Path getCurrentTrashDir() throws IOException;
+
+ /**
+ * Get the current trash directory for path specified based on the Trash
+ * Policy
+ * @param path path to be deleted
+ * @return current trash directory for the path to be deleted
+ * @throws IOException
*/
- public abstract Path getCurrentTrashDir();
+ public Path getCurrentTrashDir(Path path) throws IOException {
+ throw new UnsupportedOperationException();
+ }
/**
* Return a {@link Runnable} that periodically empties the trash of all
@@ -78,7 +109,7 @@ public abstract class TrashPolicy extends Configured {
public abstract Runnable getEmptier() throws IOException;
/**
- * Get an instance of the configured TrashPolicy based on the value
+ * Get an instance of the configured TrashPolicy based on the value
* of the configuration parameter fs.trash.classname.
*
* @param conf the configuration to be used
@@ -93,4 +124,21 @@ public abstract class TrashPolicy extends Configured {
trash.initialize(conf, fs, home); // initialize TrashPolicy
return trash;
}
+
+ /**
+ * Get an instance of the configured TrashPolicy based on the value
+ * of the configuration parameter fs.trash.classname.
+ *
+ * @param conf the configuration to be used
+ * @param fs the file system to be used
+ * @return an instance of TrashPolicy
+ */
+ public static TrashPolicy getInstance(Configuration conf, FileSystem fs)
+ throws IOException {
+ Class<? extends TrashPolicy> trashClass = conf.getClass(
+ "fs.trash.classname", TrashPolicyDefault.class, TrashPolicy.class);
+ TrashPolicy trash = ReflectionUtils.newInstance(trashClass, conf);
+ trash.initialize(conf, fs); // initialize TrashPolicy
+ return trash;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
index 1ed8a46..b5d245b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/TrashPolicyDefault.java
@@ -27,6 +27,7 @@ import java.io.IOException;
import java.text.DateFormat;
import java.text.ParseException;
import java.text.SimpleDateFormat;
+import java.util.Collection;
import java.util.Date;
import org.apache.commons.logging.Log;
@@ -66,23 +67,18 @@ public class TrashPolicyDefault extends TrashPolicy {
new SimpleDateFormat("yyMMddHHmm");
private static final int MSECS_PER_MINUTE = 60*1000;
- private Path current;
- private Path homesParent;
private long emptierInterval;
public TrashPolicyDefault() { }
- private TrashPolicyDefault(FileSystem fs, Path home, Configuration conf)
+ private TrashPolicyDefault(FileSystem fs, Configuration conf)
throws IOException {
- initialize(conf, fs, home);
+ initialize(conf, fs);
}
@Override
public void initialize(Configuration conf, FileSystem fs, Path home) {
this.fs = fs;
- this.trash = new Path(home, TRASH);
- this.homesParent = home.getParent();
- this.current = new Path(trash, CURRENT);
this.deletionInterval = (long)(conf.getFloat(
FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT)
* MSECS_PER_MINUTE);
@@ -91,6 +87,17 @@ public class TrashPolicyDefault extends TrashPolicy {
* MSECS_PER_MINUTE);
}
+ @Override
+ public void initialize(Configuration conf, FileSystem fs) {
+ this.fs = fs;
+ this.deletionInterval = (long)(conf.getFloat(
+ FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT)
+ * MSECS_PER_MINUTE);
+ this.emptierInterval = (long)(conf.getFloat(
+ FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT)
+ * MSECS_PER_MINUTE);
+ }
+
private Path makeTrashRelativePath(Path basePath, Path rmFilePath) {
return Path.mergePaths(basePath, rmFilePath);
}
@@ -113,17 +120,19 @@ public class TrashPolicyDefault extends TrashPolicy {
String qpath = fs.makeQualified(path).toString();
- if (qpath.startsWith(trash.toString())) {
+ Path trashRoot = fs.getTrashRoot(path);
+ Path trashCurrent = new Path(trashRoot, CURRENT);
+ if (qpath.startsWith(trashRoot.toString())) {
return false; // already in trash
}
- if (trash.getParent().toString().startsWith(qpath)) {
+ if (trashRoot.getParent().toString().startsWith(qpath)) {
throw new IOException("Cannot move \"" + path +
"\" to the trash, as it contains the trash");
}
- Path trashPath = makeTrashRelativePath(current, path);
- Path baseTrashPath = makeTrashRelativePath(current, path.getParent());
+ Path trashPath = makeTrashRelativePath(trashCurrent, path);
+ Path baseTrashPath = makeTrashRelativePath(trashCurrent, path.getParent());
IOException cause = null;
@@ -148,14 +157,16 @@ public class TrashPolicyDefault extends TrashPolicy {
trashPath = new Path(orig + Time.now());
}
- if (fs.rename(path, trashPath)) // move to current trash
+ if (fs.rename(path, trashPath)) { // move to current trash
+ LOG.info("Moved: '" + path + "' to trash at: " + trashPath);
return true;
+ }
} catch (IOException e) {
cause = e;
}
}
throw (IOException)
- new IOException("Failed to move to trash: "+path).initCause(cause);
+ new IOException("Failed to move to trash: " + path).initCause(cause);
}
@SuppressWarnings("deprecation")
@@ -166,72 +177,32 @@ public class TrashPolicyDefault extends TrashPolicy {
@SuppressWarnings("deprecation")
public void createCheckpoint(Date date) throws IOException {
-
- if (!fs.exists(current)) // no trash, no checkpoint
- return;
-
- Path checkpointBase;
- synchronized (CHECKPOINT) {
- checkpointBase = new Path(trash, CHECKPOINT.format(date));
-
+ Collection<FileStatus> trashRoots = fs.getTrashRoots(false);
+ for (FileStatus trashRoot: trashRoots) {
+ LOG.info("TrashPolicyDefault#createCheckpoint for trashRoot: " +
+ trashRoot.getPath());
+ createCheckpoint(trashRoot.getPath(), date);
}
- Path checkpoint = checkpointBase;
-
- int attempt = 0;
- while (true) {
- try {
- fs.rename(current, checkpoint, Rename.NONE);
- break;
- } catch (FileAlreadyExistsException e) {
- if (++attempt > 1000) {
- throw new IOException("Failed to checkpoint trash: "+checkpoint);
- }
- checkpoint = checkpointBase.suffix("-" + attempt);
- }
- }
-
- LOG.info("Created trash checkpoint: "+checkpoint.toUri().getPath());
}
@Override
public void deleteCheckpoint() throws IOException {
- FileStatus[] dirs = null;
-
- try {
- dirs = fs.listStatus(trash); // scan trash sub-directories
- } catch (FileNotFoundException fnfe) {
- return;
+ Collection<FileStatus> trashRoots = fs.getTrashRoots(false);
+ for (FileStatus trashRoot : trashRoots) {
+ LOG.info("TrashPolicyDefault#deleteCheckpoint for trashRoot: " +
+ trashRoot.getPath());
+ deleteCheckpoint(trashRoot.getPath());
}
+ }
- long now = Time.now();
- for (int i = 0; i < dirs.length; i++) {
- Path path = dirs[i].getPath();
- String dir = path.toUri().getPath();
- String name = path.getName();
- if (name.equals(CURRENT.getName())) // skip current
- continue;
-
- long time;
- try {
- time = getTimeFromCheckpoint(name);
- } catch (ParseException e) {
- LOG.warn("Unexpected item in trash: "+dir+". Ignoring.");
- continue;
- }
-
- if ((now - deletionInterval) > time) {
- if (fs.delete(path, true)) {
- LOG.info("Deleted trash checkpoint: "+dir);
- } else {
- LOG.warn("Couldn't delete checkpoint: "+dir+" Ignoring.");
- }
- }
- }
+ @Override
+ public Path getCurrentTrashDir() throws IOException {
+ return new Path(fs.getTrashRoot(null), CURRENT);
}
@Override
- public Path getCurrentTrashDir() {
- return current;
+ public Path getCurrentTrashDir(Path path) throws IOException {
+ return new Path(fs.getTrashRoot(path), CURRENT);
}
@Override
@@ -278,25 +249,24 @@ public class TrashPolicyDefault extends TrashPolicy {
try {
now = Time.now();
if (now >= end) {
-
- FileStatus[] homes = null;
+ Collection<FileStatus> trashRoots;
try {
- homes = fs.listStatus(homesParent); // list all home dirs
+ trashRoots = fs.getTrashRoots(true); // list all home dirs
} catch (IOException e) {
- LOG.warn("Trash can't list homes: "+e+" Sleeping.");
+ LOG.warn("Trash can't list all trash roots: "+e+" Sleeping.");
continue;
}
- for (FileStatus home : homes) { // dump each trash
- if (!home.isDirectory())
+ for (FileStatus trashRoot : trashRoots) { // dump each trash
+ if (!trashRoot.isDirectory())
continue;
try {
- TrashPolicyDefault trash = new TrashPolicyDefault(
- fs, home.getPath(), conf);
- trash.deleteCheckpoint();
- trash.createCheckpoint(new Date(now));
+ TrashPolicyDefault trash = new TrashPolicyDefault(fs, conf);
+ trash.deleteCheckpoint(trashRoot.getPath());
+ trash.createCheckpoint(trashRoot.getPath(), new Date(now));
} catch (IOException e) {
- LOG.warn("Trash caught: "+e+". Skipping "+home.getPath()+".");
+ LOG.warn("Trash caught: "+e+". Skipping " +
+ trashRoot.getPath() + ".");
}
}
}
@@ -319,6 +289,69 @@ public class TrashPolicyDefault extends TrashPolicy {
}
}
+ private void createCheckpoint(Path trashRoot, Date date) throws IOException {
+ if (!fs.exists(new Path(trashRoot, CURRENT))) {
+ return;
+ }
+ Path checkpointBase;
+ synchronized (CHECKPOINT) {
+ checkpointBase = new Path(trashRoot, CHECKPOINT.format(date));
+ }
+ Path checkpoint = checkpointBase;
+ Path current = new Path(trashRoot, CURRENT);
+
+ int attempt = 0;
+ while (true) {
+ try {
+ fs.rename(current, checkpoint, Rename.NONE);
+ LOG.info("Created trash checkpoint: " + checkpoint.toUri().getPath());
+ break;
+ } catch (FileAlreadyExistsException e) {
+ if (++attempt > 1000) {
+ throw new IOException("Failed to checkpoint trash: " + checkpoint);
+ }
+ checkpoint = checkpointBase.suffix("-" + attempt);
+ }
+ }
+ }
+
+ private void deleteCheckpoint(Path trashRoot) throws IOException {
+ LOG.info("TrashPolicyDefault#deleteCheckpoint for trashRoot: " + trashRoot);
+
+ FileStatus[] dirs = null;
+ try {
+ dirs = fs.listStatus(trashRoot); // scan trash sub-directories
+ } catch (FileNotFoundException fnfe) {
+ return;
+ }
+
+ long now = Time.now();
+ for (int i = 0; i < dirs.length; i++) {
+ Path path = dirs[i].getPath();
+ String dir = path.toUri().getPath();
+ String name = path.getName();
+ if (name.equals(CURRENT.getName())) { // skip current
+ continue;
+ }
+
+ long time;
+ try {
+ time = getTimeFromCheckpoint(name);
+ } catch (ParseException e) {
+ LOG.warn("Unexpected item in trash: "+dir+". Ignoring.");
+ continue;
+ }
+
+ if ((now - deletionInterval) > time) {
+ if (fs.delete(path, true)) {
+ LOG.info("Deleted trash checkpoint: "+dir);
+ } else {
+ LOG.warn("Couldn't delete checkpoint: " + dir + " Ignoring.");
+ }
+ }
+ }
+ }
+
private long getTimeFromCheckpoint(String name) throws ParseException {
long time;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
index b179c36..8ec9d80 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestHarFileSystem.java
@@ -214,6 +214,10 @@ public class TestHarFileSystem {
public Collection<? extends BlockStoragePolicySpi> getAllStoragePolicies()
throws IOException;
+
+ public Path getTrashRoot(Path path) throws IOException;
+
+ public Collection<FileStatus> getTrashRoots(boolean allUsers) throws IOException;
}
@Test
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index 9a91733..88194fd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -696,6 +696,10 @@ public class TestTrash extends TestCase {
}
@Override
+ public void initialize(Configuration conf, FileSystem fs) {
+ }
+
+ @Override
public boolean isEnabled() {
return false;
}
@@ -719,6 +723,11 @@ public class TestTrash extends TestCase {
}
@Override
+ public Path getCurrentTrashDir(Path path) throws IOException {
+ return null;
+ }
+
+ @Override
public Runnable getEmptier() throws IOException {
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 9150593..366e4e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2301,4 +2301,65 @@ public class DistributedFileSystem extends FileSystem {
throws IOException {
return dfs.getInotifyEventStream(lastReadTxid);
}
+
+ /**
+ * Get the root directory of Trash for a path in HDFS.
+ * 1. File in encryption zone returns /ez1/.Trash/username
+ * 2. File not in encryption zone returns /users/username/.Trash
+ * Caller appends either Current or checkpoint timestamp for trash destination
+ * @param path the trash root of the path to be determined.
+ * @return trash root
+ * @throws IOException
+ */
+ @Override
+ public Path getTrashRoot(Path path) throws IOException {
+ if ((path == null) || !dfs.isHDFSEncryptionEnabled()) {
+ return super.getTrashRoot(path);
+ }
+
+ String absSrc = path.toUri().getPath();
+ EncryptionZone ez = dfs.getEZForPath(absSrc);
+ if ((ez != null) && !ez.getPath().equals(absSrc)) {
+ return this.makeQualified(
+ new Path(ez.getPath() + "/" + FileSystem.TRASH_PREFIX +
+ dfs.ugi.getShortUserName()));
+ } else {
+ return super.getTrashRoot(path);
+ }
+ }
+
+ /**
+ * Get all the trash roots of HDFS for current user or for all the users.
+ * 1. File deleted from non-encryption zone /user/username/.Trash
+ * 2. File deleted from encryption zones
+ * e.g., ez1 rooted at /ez1 has its trash root at /ez1/.Trash/$USER
+ * @allUsers return trashRoots of all users if true, used by emptier
+ * @return trash roots of HDFS
+ * @throws IOException
+ */
+ @Override
+ public Collection<FileStatus> getTrashRoots(boolean allUsers) throws IOException {
+ List<FileStatus> ret = new ArrayList<FileStatus>();
+ // Get normal trash roots
+ ret.addAll(super.getTrashRoots(allUsers));
+
+ // Get EZ Trash roots
+ final RemoteIterator<EncryptionZone> it = dfs.listEncryptionZones();
+ while (it.hasNext()) {
+ Path ezTrashRoot = new Path(it.next().getPath(), FileSystem.TRASH_PREFIX);
+ if (allUsers) {
+ for (FileStatus candidate : listStatus(ezTrashRoot)) {
+ if (exists(candidate.getPath())) {
+ ret.add(candidate);
+ }
+ }
+ } else {
+ Path userTrash = new Path(ezTrashRoot, System.getProperty("user.name"));
+ if (exists(userTrash)) {
+ ret.add(getFileStatus(userTrash));
+ }
+ }
+ }
+ return ret;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8c421ee..d1bfd42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -835,7 +835,6 @@ Release 2.8.0 - UNRELEASED
HDFS-9214. Support reconfiguring dfs.datanode.balance.max.concurrent.moves
without DN restart. (Xiaobing Zhou via Arpit Agarwal)
-
HDFS-9527. The return type of FSNamesystem.getBlockCollection should be
changed to INodeFile. (szetszwo)
@@ -868,6 +867,8 @@ Release 2.8.0 - UNRELEASED
HDFS-9490. MiniDFSCluster should change block generation stamp via
FsDatasetTestUtils. (Tony Wu via lei)
+ HDFS-8831. Trash Support for deletion in HDFS encryption zone. (xyao)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f3ee0e78/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index c1e4f6e..de696b8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileSystemTestHelper;
import org.apache.hadoop.fs.FileSystemTestWrapper;
+import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -96,6 +97,7 @@ import static org.mockito.Matchers.anyShort;
import static org.mockito.Mockito.withSettings;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.apache.hadoop.test.MetricsAsserts.assertGauge;
@@ -1408,4 +1410,49 @@ public class TestEncryptionZones {
assertExceptionContains("Path not found: " + zoneFile, e);
}
}
+
+ @Test(timeout = 120000)
+ public void testEncryptionZoneWithTrash() throws Exception {
+ // Create the encryption zone1
+ final HdfsAdmin dfsAdmin =
+ new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ final Path zone1 = new Path("/zone1");
+ fs.mkdirs(zone1);
+ dfsAdmin.createEncryptionZone(zone1, TEST_KEY);
+
+ // Create the encrypted file in zone1
+ final Path encFile1 = new Path(zone1, "encFile1");
+ final int len = 8192;
+ DFSTestUtil.createFile(fs, encFile1, len, (short) 1, 0xFEED);
+
+ Configuration clientConf = new Configuration(conf);
+ clientConf.setLong(FS_TRASH_INTERVAL_KEY, 1);
+ FsShell shell = new FsShell(clientConf);
+
+ // Delete encrypted file from the shell with trash enabled
+ // Verify the file is moved to appropriate trash within the zone
+ verifyShellDeleteWithTrash(shell, encFile1);
+
+ // Delete encryption zone from the shell with trash enabled
+ // Verify the zone is moved to appropriate trash location in user's home dir
+ verifyShellDeleteWithTrash(shell, zone1);
+ }
+
+ private void verifyShellDeleteWithTrash(FsShell shell, Path path)
+ throws Exception{
+ try {
+ final Path trashFile =
+ new Path(shell.getCurrentTrashDir(path) + "/" + path);
+ String[] argv = new String[]{"-rm", "-r", path.toString()};
+ int res = ToolRunner.run(shell, argv);
+ assertEquals("rm failed", 0, res);
+ assertTrue("File not in trash : " + trashFile, fs.exists(trashFile));
+ } catch (IOException ioe) {
+ fail(ioe.getMessage());
+ } finally {
+ if (fs.exists(path)) {
+ fs.delete(path, true);
+ }
+ }
+ }
}
[20/20] hadoop git commit: YARN-4440.
FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler
time. Contributed by Lin Yiqun
Posted by wa...@apache.org.
YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the lastScheduler time. Contributed by Lin Yiqun
(cherry picked from commit 2aaed1032719e978660d1f14a8ed299e83b5f65e)
(cherry picked from commit be50df687bf800d35f418128d0c3f05ab6c429c8)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e4f638d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e4f638d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e4f638d8
Branch: refs/heads/branch-2.8
Commit: e4f638d81524ef4ecf69ba809c5ac6ca10c52014
Parents: 1d1c6c1
Author: Zhihai Xu <zx...@apache.org>
Authored: Tue Dec 15 00:17:21 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:45 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/fair/FSAppAttempt.java | 7 +++
.../scheduler/fair/TestFairScheduler.java | 60 ++++++++++++++++++++
3 files changed, 70 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f638d8/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 155ba2a..4c171d1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1075,6 +1075,9 @@ Release 2.8.0 - UNRELEASED
YARN-4392. ApplicationCreatedEvent event time resets after RM restart/failover.
(Naganarasimha G R and Xuan Gong via xgong)
+ YARN-4440. FSAppAttempt#getAllowedLocalityLevelByTime should init the
+ lastScheduler time. (Lin Yiqun via zxu)
+
Release 2.7.3 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f638d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
index abbf77a..3778cba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSAppAttempt.java
@@ -286,6 +286,13 @@ public class FSAppAttempt extends SchedulerApplicationAttempt
// default level is NODE_LOCAL
if (! allowedLocalityLevel.containsKey(priority)) {
+ // add the initial time of priority to prevent comparing with FsApp
+ // startTime and allowedLocalityLevel degrade
+ lastScheduledContainer.put(priority, currentTimeMs);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Init the lastScheduledContainer time, priority: " + priority
+ + ", time: " + currentTimeMs);
+ }
allowedLocalityLevel.put(priority, NodeType.NODE_LOCAL);
return NodeType.NODE_LOCAL;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e4f638d8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index e5402d0..f37d7dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -5151,4 +5151,64 @@ public class TestFairScheduler extends FairSchedulerTestBase {
assertEquals("root.user1", resourceManager.getRMContext().getRMApps()
.get(attId3.getApplicationId()).getQueue());
}
+
+ @Test
+ public void testFairSchedulerContinuousSchedulingInitTime() throws Exception {
+ int DELAY_THRESHOLD_TIME_MS = 1000;
+ conf.set(FairSchedulerConfiguration.CONTINUOUS_SCHEDULING_ENABLED, "true");
+ conf.set(FairSchedulerConfiguration.LOCALITY_DELAY_NODE_MS,
+ String.valueOf(DELAY_THRESHOLD_TIME_MS));
+ conf.set(FairSchedulerConfiguration.LOCALITY_DELAY_RACK_MS,
+ String.valueOf(DELAY_THRESHOLD_TIME_MS));
+
+ ControlledClock clock = new ControlledClock();
+ scheduler.setClock(clock);
+ scheduler.init(conf);
+ scheduler.start();
+
+ int priorityValue;
+ Priority priority;
+ FSAppAttempt fsAppAttempt;
+ ResourceRequest request1;
+ ResourceRequest request2;
+ ApplicationAttemptId id11;
+
+ priorityValue = 1;
+ id11 = createAppAttemptId(1, 1);
+ createMockRMApp(id11);
+ priority = Priority.newInstance(priorityValue);
+ scheduler.addApplication(id11.getApplicationId(), "root.queue1", "user1",
+ false);
+ scheduler.addApplicationAttempt(id11, false, false);
+ fsAppAttempt = scheduler.getApplicationAttempt(id11);
+
+ String hostName = "127.0.0.1";
+ RMNode node1 =
+ MockNodes.newNodeInfo(1, Resources.createResource(16 * 1024, 16), 1,
+ hostName);
+ List<ResourceRequest> ask1 = new ArrayList<>();
+ request1 =
+ createResourceRequest(1024, 8, node1.getRackName(), priorityValue, 1,
+ true);
+ request2 =
+ createResourceRequest(1024, 8, ResourceRequest.ANY, priorityValue, 1,
+ true);
+ ask1.add(request1);
+ ask1.add(request2);
+ scheduler.allocate(id11, ask1, new ArrayList<ContainerId>(), null, null,
+ null, null);
+
+ NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
+ scheduler.handle(nodeEvent1);
+ FSSchedulerNode node =
+ (FSSchedulerNode) scheduler.getSchedulerNode(node1.getNodeID());
+ // Tick the time and let the fsApp startTime different from initScheduler
+ // time
+ clock.tickSec(DELAY_THRESHOLD_TIME_MS / 1000);
+ scheduler.attemptScheduling(node);
+ Map<Priority, Long> lastScheduledContainer =
+ fsAppAttempt.getLastScheduledContainer();
+ long initSchedulerTime = lastScheduledContainer.get(priority);
+ assertEquals(DELAY_THRESHOLD_TIME_MS, initSchedulerTime);
+ }
}
[05/20] hadoop git commit: HDFS-9490. MiniDFSCluster should change
block generation stamp via FsDatasetTestUtils. (Tony Wu via lei)
Posted by wa...@apache.org.
HDFS-9490. MiniDFSCluster should change block generation stamp via FsDatasetTestUtils. (Tony Wu via lei)
(cherry picked from commit 0ac8fb4b336dd03a037092c9712b962e7ed8f852)
(cherry picked from commit 58714f3b7357d05604987e1db8f6734a7d2827d9)
Conflicts:
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/134195df
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/134195df
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/134195df
Branch: refs/heads/branch-2.8
Commit: 134195df55c6b8d69840f423d61c12245bb2aad5
Parents: 12772f5
Author: Lei Xu <le...@apache.org>
Authored: Fri Dec 4 10:24:55 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:42 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/MiniDFSCluster.java | 8 +++-----
.../hdfs/server/datanode/FsDatasetTestUtils.java | 9 +++++++++
.../fsdataset/impl/FsDatasetImplTestUtils.java | 14 ++++++++++++++
.../namenode/ha/TestPendingCorruptDnMessages.java | 2 +-
5 files changed, 30 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index baaa58d..8c421ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -865,6 +865,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9552. Document types of permission checks performed for HDFS
operations. (cnauroth)
+ HDFS-9490. MiniDFSCluster should change block generation stamp via
+ FsDatasetTestUtils. (Tony Wu via lei)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
index 0f4816e..092d9f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSCluster.java
@@ -2026,12 +2026,10 @@ public class MiniDFSCluster {
getMaterializedReplica(i, blk).truncateMeta(newSize);
}
- public boolean changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
+ public void changeGenStampOfBlock(int dnIndex, ExtendedBlock blk,
long newGenStamp) throws IOException {
- File blockFile = getBlockFile(dnIndex, blk);
- File metaFile = FsDatasetUtil.findMetaFile(blockFile);
- return metaFile.renameTo(new File(DatanodeUtil.getMetaName(
- blockFile.getAbsolutePath(), newGenStamp)));
+ getFsDatasetTestUtils(dnIndex)
+ .changeStoredGenerationStamp(blk, newGenStamp);
}
/*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
index 07fb7ce..fd47705 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/FsDatasetTestUtils.java
@@ -242,4 +242,13 @@ public interface FsDatasetTestUtils {
* Get the persistently stored generation stamp.
*/
long getStoredGenerationStamp(ExtendedBlock block) throws IOException;
+
+ /**
+ * Change the persistently stored generation stamp.
+ * @param block the block whose generation stamp will be changed
+ * @param newGenStamp the new generation stamp
+ * @throws IOException
+ */
+ void changeStoredGenerationStamp(ExtendedBlock block, long newGenStamp)
+ throws IOException;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
index 8fce163..320ae9f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImplTestUtils.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
+import org.apache.hadoop.hdfs.server.datanode.DatanodeUtil;
import org.apache.hadoop.hdfs.server.datanode.FinalizedReplica;
import org.apache.hadoop.hdfs.server.datanode.FsDatasetTestUtils;
import org.apache.hadoop.hdfs.server.datanode.Replica;
@@ -47,6 +48,7 @@ import java.io.IOException;
import java.io.RandomAccessFile;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
+import java.nio.file.StandardCopyOption;
import java.util.Random;
/**
@@ -363,4 +365,16 @@ public class FsDatasetImplTestUtils implements FsDatasetTestUtils {
File[] files = FileUtil.listFiles(dir);
return FsDatasetUtil.getGenerationStampFromFile(files, f);
}
+
+ @Override
+ public void changeStoredGenerationStamp(
+ ExtendedBlock block, long newGenStamp) throws IOException {
+ File blockFile =
+ dataset.getBlockFile(block.getBlockPoolId(), block.getBlockId());
+ File metaFile = FsDatasetUtil.findMetaFile(blockFile);
+ File newMetaFile = new File(
+ DatanodeUtil.getMetaName(blockFile.getAbsolutePath(), newGenStamp));
+ Files.move(metaFile.toPath(), newMetaFile.toPath(),
+ StandardCopyOption.ATOMIC_MOVE);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/134195df/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
index 443500c..5f116d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/ha/TestPendingCorruptDnMessages.java
@@ -68,7 +68,7 @@ public class TestPendingCorruptDnMessages {
// Change the gen stamp of the block on datanode to go back in time (gen
// stamps start at 1000)
ExtendedBlock block = DFSTestUtil.getFirstBlock(fs, filePath);
- assertTrue(cluster.changeGenStampOfBlock(0, block, 900));
+ cluster.changeGenStampOfBlock(0, block, 900);
// Run directory dsscanner to update Datanode's volumeMap
DataNodeTestUtils.runDirectoryScanner(cluster.getDataNodes().get(0));
[13/20] hadoop git commit: YARN-3946. Update exact reason as to why a
submitted app is in ACCEPTED state to app's diagnostic message.
(Naganarasimha G R via wangda)
Posted by wa...@apache.org.
YARN-3946. Update exact reason as to why a submitted app is in ACCEPTED state to app's diagnostic message. (Naganarasimha G R via wangda)
(cherry picked from commit 6cb0af3c39a5d49cb2f7911ee21363a9542ca2d7)
(cherry picked from commit a1a723fdffee44be9a5d2a1ced896442a5318375)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8ee83765
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8ee83765
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8ee83765
Branch: refs/heads/branch-2.8
Commit: 8ee83765f4a07307ca655e8152211c42ec563cba
Parents: 4f21f94
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Dec 14 10:52:46 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:44 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/resourcemanager/rmapp/RMAppImpl.java | 10 ++-
.../rmapp/attempt/RMAppAttempt.java | 6 ++
.../rmapp/attempt/RMAppAttemptImpl.java | 22 ++++-
.../scheduler/SchedulerApplicationAttempt.java | 94 +++++++++++++++++---
...CSAMContainerLaunchDiagnosticsConstants.java | 35 ++++++++
.../scheduler/capacity/CapacityScheduler.java | 2 +-
.../scheduler/capacity/LeafQueue.java | 15 +++-
.../allocator/RegularContainerAllocator.java | 16 ++--
.../scheduler/common/fica/FiCaSchedulerApp.java | 93 ++++++++++++++++++-
.../scheduler/fair/FairScheduler.java | 4 +-
.../scheduler/fifo/FifoScheduler.java | 2 +-
.../TestCapacitySchedulerPlanFollower.java | 6 +-
.../capacity/TestApplicationLimits.java | 9 +-
.../TestApplicationLimitsByPartition.java | 44 +++++++--
.../TestNodeLabelContainerAllocation.java | 17 +++-
.../scheduler/capacity/TestReservations.java | 39 ++++++++
.../webapp/TestRMWebServicesApps.java | 4 +-
18 files changed, 378 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 412999f..9f0e8e1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -542,6 +542,9 @@ Release 2.8.0 - UNRELEASED
YARN-4405. Support node label store in non-appendable file system. (Wangda
Tan via jianhe)
+ YARN-3946. Update exact reason as to why a submitted app is in ACCEPTED state to
+ app's diagnostic message. (Naganarasimha G R via wangda)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 7a2b717..c4c8d2e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -645,7 +645,8 @@ public class RMAppImpl implements RMApp, Recoverable {
progress = currentAttempt.getProgress();
logAggregationStatus = this.getLogAggregationStatusForAppReport();
}
- diags = this.diagnostics.toString();
+ //if the diagnostics is not already set get it from attempt
+ diags = getDiagnostics().toString();
if (currentAttempt != null &&
currentAttempt.getAppAttemptState() == RMAppAttemptState.LAUNCHED) {
@@ -750,8 +751,13 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public StringBuilder getDiagnostics() {
this.readLock.lock();
-
try {
+ if (diagnostics.length() == 0 && getCurrentAppAttempt() != null) {
+ String appAttemptDiagnostics = getCurrentAppAttempt().getDiagnostics();
+ if (appAttemptDiagnostics != null) {
+ return new StringBuilder(appAttemptDiagnostics);
+ }
+ }
return this.diagnostics;
} finally {
this.readLock.unlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
index 4dd8345..91aee40 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttempt.java
@@ -246,4 +246,10 @@ public interface RMAppAttempt extends EventHandler<RMAppAttemptEvent> {
* @return the finish time of the application attempt.
*/
long getFinishTime();
+
+ /**
+ * To capture Launch diagnostics of the app.
+ * @param amLaunchDiagnostics
+ */
+ void updateAMLaunchDiagnostics(String amLaunchDiagnostics);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 88a89b5..8aefe9f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -73,11 +73,11 @@ import org.apache.hadoop.yarn.server.resourcemanager.amlauncher.AMLauncherEventT
import org.apache.hadoop.yarn.server.resourcemanager.blacklist.BlacklistManager;
import org.apache.hadoop.yarn.server.resourcemanager.blacklist.BlacklistUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.blacklist.DisabledBlacklistManager;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
-import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.Recoverable;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationAttemptStateData;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.records.ApplicationStateData;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -91,6 +91,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAt
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeFinishedContainersPulledByAMEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptAddedSchedulerEvent;
@@ -187,6 +188,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
private ResourceRequest amReq = null;
private BlacklistManager blacklistedNodesForAM = null;
+ private String amLaunchDiagnostics;
+
private static final StateMachineFactory<RMAppAttemptImpl,
RMAppAttemptState,
RMAppAttemptEventType,
@@ -703,8 +706,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
@Override
public String getDiagnostics() {
this.readLock.lock();
-
try {
+ if (diagnostics.length() == 0 && amLaunchDiagnostics != null) {
+ return amLaunchDiagnostics;
+ }
return this.diagnostics.toString();
} finally {
this.readLock.unlock();
@@ -1409,6 +1414,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.launchAMStartTime;
ClusterMetrics.getMetrics().addAMLaunchDelay(delay);
}
+
+ appAttempt
+ .updateAMLaunchDiagnostics(AMState.LAUNCHED.getDiagnosticMessage());
// Register with AMLivelinessMonitor
appAttempt.attemptLaunched();
@@ -1507,6 +1515,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.originalTrackingUrl =
sanitizeTrackingUrl(registrationEvent.getTrackingurl());
+ // reset AMLaunchDiagnostics once AM Registers with RM
+ appAttempt.updateAMLaunchDiagnostics(null);
+
// Let the app know
appAttempt.eventHandler.handle(new RMAppEvent(appAttempt
.getAppAttemptId().getApplicationId(),
@@ -2097,4 +2108,9 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.writeLock.unlock();
}
}
+
+ @Override
+ public void updateAMLaunchDiagnostics(String amLaunchDiagnostics) {
+ this.amLaunchDiagnostics = amLaunchDiagnostics;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index c5f8def..22c346d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
@@ -30,6 +29,7 @@ import java.util.Set;
import java.util.concurrent.atomic.AtomicLong;
import org.apache.commons.lang.time.DateUtils;
+import org.apache.commons.lang.time.FastDateFormat;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.api.ContainerType;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.AggregateAppResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
@@ -85,6 +86,9 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
private static final Log LOG = LogFactory
.getLog(SchedulerApplicationAttempt.class);
+ private FastDateFormat fdf =
+ FastDateFormat.getInstance("EEE MMM dd HH:mm:ss Z yyyy");
+
private static final long MEM_AGGREGATE_ALLOCATION_CACHE_MSECS = 3000;
protected long lastMemoryAggregateAllocationUpdateTime = 0;
private long lastMemorySeconds = 0;
@@ -148,10 +152,12 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
protected Queue queue;
protected boolean isStopped = false;
- private String appAMNodePartitionName = CommonNodeLabelsManager.NO_LABEL;
+ protected String appAMNodePartitionName = CommonNodeLabelsManager.NO_LABEL;
protected final RMContext rmContext;
-
+
+ private RMAppAttempt appAttempt;
+
public SchedulerApplicationAttempt(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
RMContext rmContext) {
@@ -166,9 +172,11 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
if (rmContext.getRMApps() != null &&
rmContext.getRMApps()
.containsKey(applicationAttemptId.getApplicationId())) {
+ RMApp rmApp = rmContext.getRMApps().get(applicationAttemptId.getApplicationId());
ApplicationSubmissionContext appSubmissionContext =
- rmContext.getRMApps().get(applicationAttemptId.getApplicationId())
+ rmApp
.getApplicationSubmissionContext();
+ appAttempt = rmApp.getCurrentAppAttempt();
if (appSubmissionContext != null) {
unmanagedAM = appSubmissionContext.getUnmanagedAM();
this.logAggregationContext =
@@ -489,7 +497,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
ContainerType containerType = ContainerType.TASK;
// The working knowledge is that masterContainer for AM is null as it
// itself is the master container.
- if (isWaitingForAMContainer(getApplicationId())) {
+ if (isWaitingForAMContainer()) {
containerType = ContainerType.APPLICATION_MASTER;
}
try {
@@ -575,13 +583,10 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
return returnList;
}
- public boolean isWaitingForAMContainer(ApplicationId applicationId) {
+ public boolean isWaitingForAMContainer() {
// The working knowledge is that masterContainer for AM is null as it
// itself is the master container.
- RMAppAttempt appAttempt =
- rmContext.getRMApps().get(applicationId).getCurrentAppAttempt();
- return (appAttempt != null && appAttempt.getMasterContainer() == null
- && appAttempt.getSubmissionContext().getUnmanagedAM() == false);
+ return (!unmanagedAM && appAttempt.getMasterContainer() == null);
}
// Blacklist used for user containers
@@ -603,7 +608,7 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
}
public boolean isBlacklisted(String resourceName) {
- boolean useAMBlacklist = isWaitingForAMContainer(getApplicationId());
+ boolean useAMBlacklist = isWaitingForAMContainer();
return this.appSchedulingInfo.isBlacklisted(resourceName, useAMBlacklist);
}
@@ -905,4 +910,71 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
public String getAppAMNodePartitionName() {
return appAMNodePartitionName;
}
+
+ public void updateAMContainerDiagnostics(AMState state,
+ String diagnosticMessage) {
+ if (!isWaitingForAMContainer()) {
+ return;
+ }
+ StringBuilder diagnosticMessageBldr = new StringBuilder();
+ diagnosticMessageBldr.append("[");
+ diagnosticMessageBldr.append(fdf.format(System.currentTimeMillis()));
+ diagnosticMessageBldr.append("] ");
+ switch (state) {
+ case INACTIVATED:
+ diagnosticMessageBldr.append(state.diagnosticMessage);
+ if (diagnosticMessage != null) {
+ diagnosticMessageBldr.append(diagnosticMessage);
+ }
+ getPendingAppDiagnosticMessage(diagnosticMessageBldr);
+ break;
+ case ACTIVATED:
+ diagnosticMessageBldr.append(state.diagnosticMessage);
+ if (diagnosticMessage != null) {
+ diagnosticMessageBldr.append(diagnosticMessage);
+ }
+ getActivedAppDiagnosticMessage(diagnosticMessageBldr);
+ break;
+ default:
+ // UNMANAGED , ASSIGNED
+ diagnosticMessageBldr.append(state.diagnosticMessage);
+ break;
+ }
+ appAttempt.updateAMLaunchDiagnostics(diagnosticMessageBldr.toString());
+ }
+
+ protected void getPendingAppDiagnosticMessage(
+ StringBuilder diagnosticMessage) {
+ // Give the specific information which might be applicable for the
+ // respective scheduler
+ // like partitionAMResourcelimit,UserAMResourceLimit, queue'AMResourceLimit
+ }
+
+ protected void getActivedAppDiagnosticMessage(
+ StringBuilder diagnosticMessage) {
+ // Give the specific information which might be applicable for the
+ // respective scheduler
+ // queue's resource usage for specific partition
+ }
+
+ public static enum AMState {
+ UNMANAGED("User launched the Application Master, since it's unmanaged. "),
+ INACTIVATED("Application is added to the scheduler and is not yet activated. "),
+ ACTIVATED("Application is Activated, waiting for resources to be assigned for AM. "),
+ ASSIGNED("Scheduler has assigned a container for AM, waiting for AM "
+ + "container to be launched"),
+ LAUNCHED("AM container is launched, waiting for AM container to Register "
+ + "with RM")
+ ;
+
+ private String diagnosticMessage;
+
+ AMState(String diagnosticMessage) {
+ this.diagnosticMessage = diagnosticMessage;
+ }
+
+ public String getDiagnosticMessage() {
+ return diagnosticMessage;
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAMContainerLaunchDiagnosticsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAMContainerLaunchDiagnosticsConstants.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAMContainerLaunchDiagnosticsConstants.java
new file mode 100644
index 0000000..4cc062c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSAMContainerLaunchDiagnosticsConstants.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
+
+/**
+ * diagnostic messages for AMcontainer launching
+ */
+public interface CSAMContainerLaunchDiagnosticsConstants {
+ String SKIP_AM_ALLOCATION_IN_IGNORE_EXCLUSIVE_MODE =
+ "Skipping assigning to Node in Ignore Exclusivity mode. ";
+ String SKIP_AM_ALLOCATION_IN_BLACK_LISTED_NODE =
+ "Skipped scheduling for this Node as its black listed. ";
+ String SKIP_AM_ALLOCATION_DUE_TO_LOCALITY =
+ "Skipping assigning to Node as request locality is not matching. ";
+ String QUEUE_AM_RESOURCE_LIMIT_EXCEED =
+ "Queue's AM resource limit exceeded. ";
+ String USER_AM_RESOURCE_LIMIT_EXCEED = "User's AM resource limit exceeded. ";
+ String LAST_NODE_PROCESSED_MSG =
+ " Last Node which was processed for the application : ";
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 782ed03..159c7a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -960,7 +960,7 @@ public class CapacityScheduler extends
updateDemandForQueue = (LeafQueue) application.getQueue();
}
- if (application.isWaitingForAMContainer(application.getApplicationId())) {
+ if (application.isWaitingForAMContainer()) {
// Allocate is for AM and update AM blacklist for this
application.updateAMBlacklist(
blacklistAdditions, blacklistRemovals);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index a89707f..f58aba2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -60,6 +60,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
@@ -634,7 +635,7 @@ public class LeafQueue extends AbstractCSQueue {
queueUsage.getAMUsed(partitionName));
if (LOG.isDebugEnabled()) {
- LOG.debug("application AMResource "
+ LOG.debug("application "+application.getId() +" AMResource "
+ application.getAMResource(partitionName)
+ " maxAMResourcePerQueuePercent " + maxAMResourcePerQueuePercent
+ " amLimit " + amLimit + " lastClusterResource "
@@ -653,6 +654,8 @@ public class LeafQueue extends AbstractCSQueue {
+ " skipping enforcement to allow at least one application"
+ " to start");
} else {
+ application.updateAMContainerDiagnostics(AMState.INACTIVATED,
+ CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED);
LOG.info("Not activating application " + applicationId
+ " as amIfStarted: " + amIfStarted + " exceeds amLimit: "
+ amLimit);
@@ -685,6 +688,8 @@ public class LeafQueue extends AbstractCSQueue {
+ " low. skipping enforcement to allow at least one application"
+ " to start");
} else {
+ application.updateAMContainerDiagnostics(AMState.INACTIVATED,
+ CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED);
LOG.info("Not activating application " + applicationId
+ " for user: " + user + " as userAmIfStarted: "
+ userAmIfStarted + " exceeds userAmLimit: " + userAMLimit);
@@ -693,6 +698,8 @@ public class LeafQueue extends AbstractCSQueue {
}
user.activateApplication();
orderingPolicy.addSchedulableEntity(application);
+ application.updateAMContainerDiagnostics(AMState.ACTIVATED, null);
+
queueUsage.incAMUsed(partitionName,
application.getAMResource(partitionName));
user.getResourceUsage().incAMUsed(partitionName,
@@ -868,6 +875,8 @@ public class LeafQueue extends AbstractCSQueue {
// Check user limit
if (!canAssignToUser(clusterResource, application.getUser(), userLimit,
application, node.getPartition(), currentResourceLimits)) {
+ application.updateAMContainerDiagnostics(AMState.ACTIVATED,
+ "User capacity has reached its maximum limit.");
continue;
}
@@ -903,7 +912,9 @@ public class LeafQueue extends AbstractCSQueue {
// Done
return assignment;
- } else if (!assignment.getSkipped()) {
+ } else if (assignment.getSkipped()) {
+ application.updateNodeInfoForAMDiagnostics(node);
+ } else {
// If we don't allocate anything, and it is not skipped by application,
// we will return to respect FIFO of applications
return CSAssignment.NULL_ASSIGNMENT;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
index 6bd4d17..820cccd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/allocator/RegularContainerAllocator.java
@@ -28,12 +28,12 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
-import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMContainerLaunchDiagnosticsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -79,6 +79,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
FiCaSchedulerNode node, SchedulingMode schedulingMode,
ResourceLimits resourceLimits, Priority priority) {
if (SchedulerAppUtils.isBlacklisted(application, node, LOG)) {
+ application.updateAppSkipNodeDiagnostics(
+ CSAMContainerLaunchDiagnosticsConstants.SKIP_AM_ALLOCATION_IN_BLACK_LISTED_NODE);
return ContainerAllocation.APP_SKIPPED;
}
@@ -99,16 +101,14 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// AM container allocation doesn't support non-exclusive allocation to
// avoid painful of preempt an AM container
if (schedulingMode == SchedulingMode.IGNORE_PARTITION_EXCLUSIVITY) {
- RMAppAttempt rmAppAttempt =
- rmContext.getRMApps().get(application.getApplicationId())
- .getCurrentAppAttempt();
- if (rmAppAttempt.getSubmissionContext().getUnmanagedAM() == false
- && null == rmAppAttempt.getMasterContainer()) {
+ if (application.isWaitingForAMContainer()) {
if (LOG.isDebugEnabled()) {
LOG.debug("Skip allocating AM container to app_attempt="
+ application.getApplicationAttemptId()
+ ", don't allow to allocate AM container in non-exclusive mode");
}
+ application.updateAppSkipNodeDiagnostics(
+ "Skipping assigning to Node in Ignore Exclusivity mode. ");
return ContainerAllocation.APP_SKIPPED;
}
}
@@ -318,6 +318,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
schedulingMode, currentResoureLimits);
}
+ application.updateAppSkipNodeDiagnostics(
+ CSAMContainerLaunchDiagnosticsConstants.SKIP_AM_ALLOCATION_DUE_TO_LOCALITY);
return ContainerAllocation.APP_SKIPPED;
}
@@ -621,6 +623,8 @@ public class RegularContainerAllocator extends AbstractContainerAllocator {
// something went wrong getting/creating the container
if (container == null) {
+ application
+ .updateAppSkipNodeDiagnostics("Scheduling of container failed. ");
LOG.warn("Couldn't get container for allocation!");
return ContainerAllocation.APP_SKIPPED;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 6f4bfe5..c9c792e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NMToken;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -54,13 +55,15 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMContainerLaunchDiagnosticsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.LeafQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.QueueCapacities;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocator;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.AbstractContainerAllocator;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.allocator.ContainerAllocator;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -87,6 +90,11 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
private AbstractContainerAllocator containerAllocator;
+ /**
+ * to hold the message if its app doesn't not get container from a node
+ */
+ private String appSkipNodeDiagnostics;
+
public FiCaSchedulerApp(ApplicationAttemptId applicationAttemptId,
String user, Queue queue, ActiveUsersManager activeUsersManager,
RMContext rmContext) {
@@ -187,6 +195,8 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
node.getNodeID(), appSchedulingInfo.getUser(), this.rmContext,
request.getNodeLabelExpression());
+ updateAMContainerDiagnostics(AMState.ASSIGNED, null);
+
// Add it to allContainers list.
newlyAllocatedContainers.add(rmContainer);
liveContainers.put(container.getId(), rmContainer);
@@ -502,4 +512,85 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
getCSLeafQueue().incAMUsedResource(newPartition, containerResource, this);
}
}
+
+ protected void getPendingAppDiagnosticMessage(
+ StringBuilder diagnosticMessage) {
+ LeafQueue queue = getCSLeafQueue();
+ diagnosticMessage.append(" Details : AM Partition = ");
+ diagnosticMessage.append(appAMNodePartitionName.isEmpty()
+ ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
+ diagnosticMessage.append("; ");
+ diagnosticMessage.append("AM Resource Request = ");
+ diagnosticMessage.append(getAMResource(appAMNodePartitionName));
+ diagnosticMessage.append("; ");
+ diagnosticMessage.append("Queue Resource Limit for AM = ");
+ diagnosticMessage
+ .append(queue.getAMResourceLimitPerPartition(appAMNodePartitionName));
+ diagnosticMessage.append("; ");
+ diagnosticMessage.append("User AM Resource Limit of the queue = ");
+ diagnosticMessage.append(
+ queue.getUserAMResourceLimitPerPartition(appAMNodePartitionName));
+ diagnosticMessage.append("; ");
+ diagnosticMessage.append("Queue AM Resource Usage = ");
+ diagnosticMessage.append(
+ queue.getQueueResourceUsage().getAMUsed(appAMNodePartitionName));
+ diagnosticMessage.append("; ");
+ }
+
+ protected void getActivedAppDiagnosticMessage(
+ StringBuilder diagnosticMessage) {
+ LeafQueue queue = getCSLeafQueue();
+ QueueCapacities queueCapacities = queue.getQueueCapacities();
+ diagnosticMessage.append(" Details : AM Partition = ");
+ diagnosticMessage.append(appAMNodePartitionName.isEmpty()
+ ? NodeLabel.DEFAULT_NODE_LABEL_PARTITION : appAMNodePartitionName);
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Partition Resource = ");
+ diagnosticMessage.append(rmContext.getNodeLabelManager()
+ .getResourceByLabel(appAMNodePartitionName, Resources.none()));
+ diagnosticMessage.append(" ; ");
+ diagnosticMessage.append("Queue's Absolute capacity = ");
+ diagnosticMessage.append(
+ queueCapacities.getAbsoluteCapacity(appAMNodePartitionName) * 100);
+ diagnosticMessage.append(" % ; ");
+ diagnosticMessage.append("Queue's Absolute used capacity = ");
+ diagnosticMessage.append(
+ queueCapacities.getAbsoluteUsedCapacity(appAMNodePartitionName) * 100);
+ diagnosticMessage.append(" % ; ");
+ diagnosticMessage.append("Queue's Absolute max capacity = ");
+ diagnosticMessage.append(
+ queueCapacities.getAbsoluteMaximumCapacity(appAMNodePartitionName)
+ * 100);
+ diagnosticMessage.append(" % ; ");
+ }
+
+ /**
+ * Set the message temporarily if the reason is known for why scheduling did
+ * not happen for a given node, if not message will be over written
+ * @param message
+ */
+ public void updateAppSkipNodeDiagnostics(String message) {
+ this.appSkipNodeDiagnostics = message;
+ }
+
+ public void updateNodeInfoForAMDiagnostics(FiCaSchedulerNode node) {
+ if (isWaitingForAMContainer()) {
+ StringBuilder diagnosticMessageBldr = new StringBuilder();
+ if (appSkipNodeDiagnostics != null) {
+ diagnosticMessageBldr.append(appSkipNodeDiagnostics);
+ appSkipNodeDiagnostics = null;
+ }
+ diagnosticMessageBldr.append(
+ CSAMContainerLaunchDiagnosticsConstants.LAST_NODE_PROCESSED_MSG);
+ diagnosticMessageBldr.append(node.getNodeID());
+ diagnosticMessageBldr.append(" ( Partition : ");
+ diagnosticMessageBldr.append(node.getLabels());
+ diagnosticMessageBldr.append(", Total resource : ");
+ diagnosticMessageBldr.append(node.getTotalResource());
+ diagnosticMessageBldr.append(", Available resource : ");
+ diagnosticMessageBldr.append(node.getAvailableResource());
+ diagnosticMessageBldr.append(" ).");
+ updateAMContainerDiagnostics(AMState.ACTIVATED, diagnosticMessageBldr.toString());
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 04977a6..9c16e49 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -75,9 +75,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnSched
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntitlement;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAddedSchedulerEvent;
@@ -1008,7 +1008,7 @@ public class FairScheduler extends
preemptionContainerIds.add(container.getContainerId());
}
- if (application.isWaitingForAMContainer(application.getApplicationId())) {
+ if (application.isWaitingForAMContainer()) {
// Allocate is for AM and update AM blacklist for this
application.updateAMBlacklist(
blacklistAdditions, blacklistRemovals);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index cfae3a2..8e75d11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -359,7 +359,7 @@ public class FifoScheduler extends
" #ask=" + ask.size());
}
- if (application.isWaitingForAMContainer(application.getApplicationId())) {
+ if (application.isWaitingForAMContainer()) {
// Allocate is for AM and update AM blacklist for this
application.updateAMBlacklist(
blacklistAdditions, blacklistRemovals);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
index 9152d12..bccd5c3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/reservation/TestCapacitySchedulerPlanFollower.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.exceptions.PlanningException;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.planning.ReservationAgent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler;
@@ -80,9 +81,12 @@ public class TestCapacitySchedulerPlanFollower extends
ConcurrentMap<ApplicationId, RMApp> spyApps =
spy(new ConcurrentHashMap<ApplicationId, RMApp>());
RMApp rmApp = mock(RMApp.class);
+ RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any()))
- .thenReturn(null);
+ .thenReturn(rmAppAttempt);
+ when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
+ Mockito.doReturn(true).when(spyApps).containsKey((ApplicationId) Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
when(spyRMContext.getScheduler()).thenReturn(scheduler);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
index 9f4b9f5..3e65013 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimits.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -602,7 +603,13 @@ public class TestApplicationLimits {
when(rmApp.getAMResourceRequest()).thenReturn(amResourceRequest);
Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId)Matchers.any());
when(spyRMContext.getRMApps()).thenReturn(spyApps);
-
+ RMAppAttempt rmAppAttempt = mock(RMAppAttempt.class);
+ when(rmApp.getRMAppAttempt((ApplicationAttemptId) Matchers.any()))
+ .thenReturn(rmAppAttempt);
+ when(rmApp.getCurrentAppAttempt()).thenReturn(rmAppAttempt);
+ Mockito.doReturn(rmApp).when(spyApps).get((ApplicationId) Matchers.any());
+ Mockito.doReturn(true).when(spyApps)
+ .containsKey((ApplicationId) Matchers.any());
Priority priority_1 = TestUtils.createMockPriority(1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
index 267abaf..89fcb16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestApplicationLimitsByPartition.java
@@ -32,6 +32,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.NullRMNodeLabels
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -120,15 +121,13 @@ public class TestApplicationLimitsByPartition {
// Submit app1 with 1Gb AM resource to Queue A1 for label X
RMApp app1 = rm1.submitApp(GB, "app", "user", null, "a1", "x");
- MockRM.launchAndRegisterAM(app1, rm1, nm1);
// Submit app2 with 1Gb AM resource to Queue A1 for label X
RMApp app2 = rm1.submitApp(GB, "app", "user", null, "a1", "x");
- MockRM.launchAndRegisterAM(app2, rm1, nm1);
// Submit 3rd app to Queue A1 for label X, and this will be pending as
// AM limit is already crossed for label X. (2GB)
- rm1.submitApp(GB, "app", "user", null, "a1", "x");
+ RMApp pendingApp = rm1.submitApp(GB, "app", "user", null, "a1", "x");
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1");
@@ -138,6 +137,16 @@ public class TestApplicationLimitsByPartition {
// pending.
Assert.assertEquals(2, leafQueue.getNumActiveApplications());
Assert.assertEquals(1, leafQueue.getNumPendingApplications());
+ Assert.assertTrue("AM diagnostics not set properly", app1.getDiagnostics()
+ .toString().contains(AMState.ACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly", app2.getDiagnostics()
+ .toString().contains(AMState.ACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString()
+ .contains(AMState.INACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString().contains(
+ CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED));
// Now verify the same test case in Queue C1 where label is not configured.
// Submit an app to Queue C1 with empty label
@@ -150,7 +159,7 @@ public class TestApplicationLimitsByPartition {
// Submit 3rd app to Queue C1. This will be pending as Queue's am-limit
// is reached.
- rm1.submitApp(GB, "app", "user", null, "c1");
+ pendingApp = rm1.submitApp(GB, "app", "user", null, "c1");
leafQueue = (LeafQueue) cs.getQueue("c1");
Assert.assertNotNull(leafQueue);
@@ -159,6 +168,12 @@ public class TestApplicationLimitsByPartition {
// is reached.
Assert.assertEquals(2, leafQueue.getNumActiveApplications());
Assert.assertEquals(1, leafQueue.getNumPendingApplications());
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString()
+ .contains(AMState.INACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString().contains(
+ CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED));
rm1.killApp(app3.getApplicationId());
Thread.sleep(1000);
@@ -288,11 +303,10 @@ public class TestApplicationLimitsByPartition {
// Submit app1 (2 GB) to Queue A1 and label X
RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "x");
- MockRM.launchAndRegisterAM(app1, rm1, nm1);
// Submit 2nd app to label "X" with one GB. Since queue am-limit is 2GB,
// 2nd app will be pending and first one will get activated.
- rm1.submitApp(GB, "app", "user", null, "a1", "x");
+ RMApp pendingApp = rm1.submitApp(GB, "app", "user", null, "a1", "x");
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1");
@@ -302,6 +316,14 @@ public class TestApplicationLimitsByPartition {
// used for partition "x" also.
Assert.assertEquals(1, leafQueue.getNumActiveApplications());
Assert.assertEquals(1, leafQueue.getNumPendingApplications());
+ Assert.assertTrue("AM diagnostics not set properly", app1.getDiagnostics()
+ .toString().contains(AMState.ACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString()
+ .contains(AMState.INACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString()
+ .contains(CSAMContainerLaunchDiagnosticsConstants.QUEUE_AM_RESOURCE_LIMIT_EXCEED));
rm1.close();
}
@@ -381,7 +403,7 @@ public class TestApplicationLimitsByPartition {
// 4Gb -> 40% of label "X" in queue A1
// Since we have 2 users, 50% of 4Gb will be max for each user. Here user1
// has already crossed this 2GB limit, hence this app will be pending.
- rm1.submitApp(GB, "app", user_1, null, "a1", "x");
+ RMApp pendingApp = rm1.submitApp(GB, "app", user_1, null, "a1", "x");
// Verify active applications count per user and also in queue level.
Assert.assertEquals(3, leafQueue.getNumActiveApplications());
@@ -389,6 +411,14 @@ public class TestApplicationLimitsByPartition {
Assert.assertEquals(2, leafQueue.getNumActiveApplications(user_1));
Assert.assertEquals(1, leafQueue.getNumPendingApplications(user_1));
Assert.assertEquals(1, leafQueue.getNumPendingApplications());
+
+ //verify Diagnostic messages
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString()
+ .contains(AMState.INACTIVATED.getDiagnosticMessage()));
+ Assert.assertTrue("AM diagnostics not set properly",
+ pendingApp.getDiagnostics().toString().contains(
+ CSAMContainerLaunchDiagnosticsConstants.USER_AM_RESOURCE_LIMIT_EXCEED));
rm1.close();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
index 91666df..dff82ca 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestNodeLabelContainerAllocation.java
@@ -1028,10 +1028,11 @@ public class TestNodeLabelContainerAllocation {
rm1.getRMContext().setNodeLabelManager(mgr);
rm1.start();
- MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB); // label = x
+ String nodeIdStr = "h1:1234";
+ MockNM nm1 = rm1.registerNode(nodeIdStr, 8 * GB); // label = x
// launch an app to queue b1 (label = y), AM container should be launched in nm3
- rm1.submitApp(1 * GB, "app", "user", null, "b1");
+ RMApp app = rm1.submitApp(1 * GB, "app", "user", null, "b1");
CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
@@ -1040,7 +1041,17 @@ public class TestNodeLabelContainerAllocation {
for (int i = 0; i < 50; i++) {
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
}
-
+
+ Assert.assertTrue(
+ "Scheduler diagnostics should have reason for not assigning the node",
+ app.getDiagnostics().toString().contains(
+ CSAMContainerLaunchDiagnosticsConstants.SKIP_AM_ALLOCATION_IN_IGNORE_EXCLUSIVE_MODE));
+
+ Assert.assertTrue(
+ "Scheduler diagnostics should have last processed node information",
+ app.getDiagnostics().toString().contains(
+ CSAMContainerLaunchDiagnosticsConstants.LAST_NODE_PROCESSED_MSG
+ + nodeIdStr + " ( Partition : [x]"));
Assert.assertEquals(0, cs.getSchedulerNode(nm1.getNodeId())
.getNumContainers());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
index 884de2a..49de478 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestReservations.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
@@ -53,6 +54,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt.AMState;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
@@ -65,6 +67,7 @@ import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
public class TestReservations {
@@ -188,6 +191,9 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
a.submitApplicationAttempt(app_0, user_0);
@@ -196,6 +202,9 @@ public class TestReservations {
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -348,6 +357,9 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
a.submitApplicationAttempt(app_0, user_0);
@@ -356,6 +368,9 @@ public class TestReservations {
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -502,6 +517,9 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
a.submitApplicationAttempt(app_0, user_0);
@@ -510,6 +528,9 @@ public class TestReservations {
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -758,6 +779,9 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
a.submitApplicationAttempt(app_0, user_0);
@@ -766,6 +790,9 @@ public class TestReservations {
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -927,6 +954,9 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
a.submitApplicationAttempt(app_0, user_0);
@@ -934,6 +964,9 @@ public class TestReservations {
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
@@ -1068,6 +1101,9 @@ public class TestReservations {
.getMockApplicationAttemptId(0, 0);
FiCaSchedulerApp app_0 = new FiCaSchedulerApp(appAttemptId_0, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_0 = spy(app_0);
+ Mockito.doNothing().when(app_0).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
rmContext.getRMApps().put(app_0.getApplicationId(), mock(RMApp.class));
a.submitApplicationAttempt(app_0, user_0);
@@ -1076,6 +1112,9 @@ public class TestReservations {
.getMockApplicationAttemptId(1, 0);
FiCaSchedulerApp app_1 = new FiCaSchedulerApp(appAttemptId_1, user_0, a,
mock(ActiveUsersManager.class), spyRMContext);
+ app_1 = spy(app_1);
+ Mockito.doNothing().when(app_1).updateAMContainerDiagnostics(any(AMState.class),
+ any(String.class));
a.submitApplicationAttempt(app_1, user_0);
// Setup some nodes
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8ee83765/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
index d7e80e8..d0a8c27 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesApps.java
@@ -1387,8 +1387,8 @@ public class TestRMWebServicesApps extends JerseyTestBase {
assertEquals("progress doesn't match", 0, progress, 0.0);
WebServicesTestUtils.checkStringMatch("trackingUI", "UNASSIGNED",
trackingUI);
- WebServicesTestUtils.checkStringMatch("diagnostics", app.getDiagnostics()
- .toString(), diagnostics);
+ WebServicesTestUtils.checkStringEqual("diagnostics",
+ app.getDiagnostics().toString(), diagnostics);
assertEquals("clusterId doesn't match",
ResourceManager.getClusterTimeStamp(), clusterId);
assertEquals("startedTime doesn't match", app.getStartTime(), startedTime);
[06/20] hadoop git commit: YARN-4405. Support node label store in
non-appendable file system. Contributed by Wangda Tan (cherry picked from
commit 755dda8dd8bb23864abc752bad506f223fcac010)
Posted by wa...@apache.org.
YARN-4405. Support node label store in non-appendable file system. Contributed by Wangda Tan
(cherry picked from commit 755dda8dd8bb23864abc752bad506f223fcac010)
(cherry picked from commit be404e168060c03a33a35c8d30925e317a64cad7)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f355645c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f355645c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f355645c
Branch: refs/heads/branch-2.8
Commit: f355645ca44aaac1bd0b8dbafb4576b495c1878f
Parents: 0b88a7f
Author: Jian He <ji...@apache.org>
Authored: Thu Dec 3 17:45:31 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:42 2015 -0800
----------------------------------------------------------------------
.../conf/TestConfigurationFieldsBase.java | 14 ++---
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 6 ++
.../yarn/conf/TestYarnConfigurationFields.java | 2 +
.../nodelabels/CommonNodeLabelsManager.java | 15 ++++-
.../nodelabels/FileSystemNodeLabelsStore.java | 65 +++++++++++---------
.../hadoop/yarn/nodelabels/NodeLabelsStore.java | 16 ++---
.../src/main/resources/yarn-default.xml | 8 +++
.../DummyCommonNodeLabelsManager.java | 6 +-
.../TestFileSystemNodeLabelsStore.java | 28 +++++++--
.../nodelabels/NullRMNodeLabelsManager.java | 4 +-
11 files changed, 106 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
index 2e4d8b1..e528602 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfigurationFieldsBase.java
@@ -18,27 +18,22 @@
package org.apache.hadoop.conf;
-import java.lang.Class;
+import org.junit.Before;
+import org.junit.Ignore;
+import org.junit.Test;
+
import java.lang.reflect.Field;
import java.lang.reflect.Modifier;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
-import java.util.List;
import java.util.Map;
-import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
-import org.junit.Before;
-import org.junit.Ignore;
-import org.junit.Test;
import static org.junit.Assert.assertTrue;
-import org.apache.hadoop.conf.Configuration;
-
/**
* Base class for comparing fields in one or more Configuration classes
* against a corresponding .xml file. Usage is intended as follows:
@@ -331,6 +326,7 @@ public abstract class TestConfigurationFieldsBase {
private static Set<String> compareConfigurationToXmlFields(Map<String,String> keyMap1, Map<String,String> keyMap2) {
Set<String> retVal = new HashSet<String>(keyMap1.keySet());
retVal.removeAll(keyMap2.keySet());
+
return retVal;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 165fa47..727868d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -539,6 +539,9 @@ Release 2.8.0 - UNRELEASED
YARN-4292. ResourceUtilization should be a part of NodeInfo REST API.
(Sunil G via wangda)
+ YARN-4405. Support node label store in non-appendable file system. (Wangda
+ Tan via jianhe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 1b2565f..521bb3d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2075,6 +2075,12 @@ public class YarnConfiguration extends Configuration {
*/
public static final String NODE_LABELS_PREFIX = YARN_PREFIX + "node-labels.";
+ /** Node label store implementation class */
+ public static final String FS_NODE_LABELS_STORE_IMPL_CLASS = NODE_LABELS_PREFIX
+ + "fs-store.impl.class";
+ public static final String DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS =
+ "org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore";
+
/** URI for NodeLabelManager */
public static final String FS_NODE_LABELS_STORE_ROOT_DIR = NODE_LABELS_PREFIX
+ "fs-store.root-dir";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 97fcfa1..dec5cfb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -49,6 +49,8 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
// Specific properties to skip
configurationPropsToSkipCompare
+ .add(YarnConfiguration.DEFAULT_FS_NODE_LABELS_STORE_IMPL_CLASS);
+ configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS);
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_CLIENT_FAILOVER_PROXY_PROVIDER);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index a00c49d..8b26cc5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -42,6 +42,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -224,10 +225,20 @@ public class CommonNodeLabelsManager extends AbstractService {
labelCollections.put(NO_LABEL, new RMNodeLabel(NO_LABEL));
}
+ boolean isCentralizedConfiguration() {
+ return isCentralizedNodeLabelConfiguration;
+ }
+
protected void initNodeLabelStore(Configuration conf) throws Exception {
- this.store = new FileSystemNodeLabelsStore(this);
+ this.store =
+ ReflectionUtils
+ .newInstance(
+ conf.getClass(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS,
+ FileSystemNodeLabelsStore.class, NodeLabelsStore.class),
+ conf);
+ this.store.setNodeLabelsManager(this);
this.store.init(conf);
- this.store.recover(!isCentralizedNodeLabelConfiguration);
+ this.store.recover();
}
// for UT purpose
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
index c9727a2..a65349b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/FileSystemNodeLabelsStore.java
@@ -52,11 +52,6 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.ReplaceLabelsOn
import com.google.common.collect.Sets;
public class FileSystemNodeLabelsStore extends NodeLabelsStore {
-
- public FileSystemNodeLabelsStore(CommonNodeLabelsManager mgr) {
- super(mgr);
- }
-
protected static final Log LOG = LogFactory.getLog(FileSystemNodeLabelsStore.class);
protected static final String DEFAULT_DIR_NAME = "node-labels";
@@ -69,8 +64,8 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
Path fsWorkingPath;
FileSystem fs;
- FSDataOutputStream editlogOs;
- Path editLogPath;
+ private FSDataOutputStream editlogOs;
+ private Path editLogPath;
private String getDefaultFSNodeLabelsRootDir() throws IOException {
// default is in local: /tmp/hadoop-yarn-${user}/node-labels/
@@ -160,12 +155,40 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
ensureCloseEditlogFile();
}
}
+
+ protected void loadFromMirror(Path newMirrorPath, Path oldMirrorPath)
+ throws IOException {
+ // If mirror.new exists, read from mirror.new,
+ FSDataInputStream is = null;
+ if (fs.exists(newMirrorPath)) {
+ is = fs.open(newMirrorPath);
+ } else if (fs.exists(oldMirrorPath)) {
+ is = fs.open(oldMirrorPath);
+ }
+
+ if (null != is) {
+ List<NodeLabel> labels = new AddToClusterNodeLabelsRequestPBImpl(
+ AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is))
+ .getNodeLabels();
+ mgr.addToCluserNodeLabels(labels);
+
+ if (mgr.isCentralizedConfiguration()) {
+ // Only load node to labels mapping while using centralized configuration
+ Map<NodeId, Set<String>> nodeToLabels =
+ new ReplaceLabelsOnNodeRequestPBImpl(
+ ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
+ .getNodeToLabels();
+ mgr.replaceLabelsOnNode(nodeToLabels);
+ }
+ is.close();
+ }
+ }
/* (non-Javadoc)
* @see org.apache.hadoop.yarn.nodelabels.NodeLabelsStore#recover(boolean)
*/
@Override
- public void recover(boolean ignoreNodeToLabelsMappings) throws YarnException,
+ public void recover() throws YarnException,
IOException {
/*
* Steps of recover
@@ -181,31 +204,13 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
// Open mirror from serialized file
Path mirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME);
Path oldMirrorPath = new Path(fsWorkingPath, MIRROR_FILENAME + ".old");
-
- FSDataInputStream is = null;
- if (fs.exists(mirrorPath)) {
- is = fs.open(mirrorPath);
- } else if (fs.exists(oldMirrorPath)) {
- is = fs.open(oldMirrorPath);
- }
-
- if (null != is) {
- List<NodeLabel> labels =
- new AddToClusterNodeLabelsRequestPBImpl(
- AddToClusterNodeLabelsRequestProto.parseDelimitedFrom(is)).getNodeLabels();
- Map<NodeId, Set<String>> nodeToLabels =
- new ReplaceLabelsOnNodeRequestPBImpl(
- ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
- .getNodeToLabels();
- mgr.addToCluserNodeLabels(labels);
- mgr.replaceLabelsOnNode(nodeToLabels);
- is.close();
- }
+
+ loadFromMirror(mirrorPath, oldMirrorPath);
// Open and process editlog
editLogPath = new Path(fsWorkingPath, EDITLOG_FILENAME);
if (fs.exists(editLogPath)) {
- is = fs.open(editLogPath);
+ FSDataInputStream is = fs.open(editLogPath);
while (true) {
try {
@@ -233,7 +238,7 @@ public class FileSystemNodeLabelsStore extends NodeLabelsStore {
new ReplaceLabelsOnNodeRequestPBImpl(
ReplaceLabelsOnNodeRequestProto.parseDelimitedFrom(is))
.getNodeToLabels();
- if (!ignoreNodeToLabelsMappings) {
+ if (mgr.isCentralizedConfiguration()) {
/*
* In case of Distributed NodeLabels setup,
* ignoreNodeToLabelsMappings will be set to true and recover will
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
index 46b94fd..aacb920 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/NodeLabelsStore.java
@@ -31,11 +31,7 @@ import org.apache.hadoop.yarn.api.records.NodeLabel;
import org.apache.hadoop.yarn.exceptions.YarnException;
public abstract class NodeLabelsStore implements Closeable {
- protected final CommonNodeLabelsManager mgr;
-
- public NodeLabelsStore(CommonNodeLabelsManager mgr) {
- this.mgr = mgr;
- }
+ protected CommonNodeLabelsManager mgr;
/**
* Store node {@literal ->} label
@@ -62,16 +58,14 @@ public abstract class NodeLabelsStore implements Closeable {
* ignoreNodeToLabelsMappings will be set to true and recover will be invoked
* as RM will collect the node labels from NM through registration/HB
*
- * @param ignoreNodeToLabelsMappings
* @throws IOException
* @throws YarnException
*/
- public abstract void recover(boolean ignoreNodeToLabelsMappings)
- throws IOException, YarnException;
+ public abstract void recover() throws IOException, YarnException;
public void init(Configuration conf) throws Exception {}
-
- public CommonNodeLabelsManager getNodeLabelsManager() {
- return mgr;
+
+ public void setNodeLabelsManager(CommonNodeLabelsManager mgr) {
+ this.mgr = mgr;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f86eab9..0917366 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2465,4 +2465,12 @@
<name>yarn.am.blacklisting.disable-failure-threshold</name>
<value>0.8f</value>
</property>
+
+ <property>
+ <description>
+ Choose different implementation of node label's storage
+ </description>
+ <name>yarn.node-labels.fs-store.impl.class</name>
+ <value>org.apache.hadoop.yarn.nodelabels.FileSystemNodeLabelsStore</value>
+ </property>
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
index fce663a..64c74c2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/DummyCommonNodeLabelsManager.java
@@ -36,10 +36,10 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager {
@Override
public void initNodeLabelStore(Configuration conf) {
- this.store = new NodeLabelsStore(this) {
+ this.store = new NodeLabelsStore() {
@Override
- public void recover(boolean ignoreNodeToLabelsMappings)
+ public void recover()
throws IOException {
}
@@ -65,6 +65,8 @@ public class DummyCommonNodeLabelsManager extends CommonNodeLabelsManager {
// do nothing
}
};
+
+ this.store.setNodeLabelsManager(this);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
index 4929f95..82e4e11 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/nodelabels/TestFileSystemNodeLabelsStore.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.nodelabels;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Map;
import org.apache.hadoop.conf.Configuration;
@@ -33,13 +34,17 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.mockito.Mockito;
import com.google.common.collect.ImmutableMap;
-import org.mockito.Mockito;
+@RunWith(Parameterized.class)
public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
MockNodeLabelManager mgr = null;
Configuration conf = null;
+ String storeClassName = null;
private static class MockNodeLabelManager extends
CommonNodeLabelsManager {
@@ -59,8 +64,15 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
}
}
- private FileSystemNodeLabelsStore getStore() {
- return (FileSystemNodeLabelsStore) mgr.store;
+ public TestFileSystemNodeLabelsStore(String className) {
+ this.storeClassName = className;
+ }
+
+ @Parameterized.Parameters
+ public static Collection<String[]> getParameters() {
+ return Arrays.asList(
+ new String[][] { { FileSystemNodeLabelsStore.class.getCanonicalName() },
+ { NonAppendableFSNodeLabelStore.class.getCanonicalName() } });
}
@Before
@@ -68,6 +80,7 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
mgr = new MockNodeLabelManager();
conf = new Configuration();
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
+ conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_IMPL_CLASS, storeClassName);
File tempDir = File.createTempFile("nlb", ".tmp");
tempDir.delete();
tempDir.mkdirs();
@@ -80,7 +93,11 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
@After
public void after() throws IOException {
- getStore().fs.delete(getStore().fsWorkingPath, true);
+ if (mgr.store instanceof FileSystemNodeLabelsStore) {
+ FileSystemNodeLabelsStore fsStore =
+ ((FileSystemNodeLabelsStore) mgr.store);
+ fsStore.fs.delete(fsStore.fsWorkingPath, true);
+ }
mgr.stop();
}
@@ -324,11 +341,12 @@ public class TestFileSystemNodeLabelsStore extends NodeLabelTestBase {
@Test
public void testRootMkdirOnInitStore() throws Exception {
final FileSystem mockFs = Mockito.mock(FileSystem.class);
- FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore(mgr) {
+ FileSystemNodeLabelsStore mockStore = new FileSystemNodeLabelsStore() {
void setFileSystem(Configuration conf) throws IOException {
fs = mockFs;
}
};
+ mockStore.setNodeLabelsManager(mgr);
mockStore.fs = mockFs;
verifyMkdirsCount(mockStore, true, 0);
verifyMkdirsCount(mockStore, false, 1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f355645c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
index 2e21d26..bb0b45f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/NullRMNodeLabelsManager.java
@@ -37,10 +37,10 @@ public class NullRMNodeLabelsManager extends RMNodeLabelsManager {
@Override
public void initNodeLabelStore(Configuration conf) {
- this.store = new NodeLabelsStore(this) {
+ this.store = new NodeLabelsStore() {
@Override
- public void recover(boolean ignoreNodeToLabelsMappings)
+ public void recover()
throws IOException {
// do nothing
}
[16/20] hadoop git commit: HDFS-8326. Documentation about when
checkpoints are run is out of date. (Misty Stanley-Jones via xyao)
Posted by wa...@apache.org.
HDFS-8326. Documentation about when checkpoints are run is out of date. (Misty Stanley-Jones via xyao)
(cherry picked from commit d0e75e60fb16ffd6c95648a06ff3958722f71e4d)
(cherry picked from commit 5595b912e417286594ec0703d3e6062b00985fcc)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4f21f940
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4f21f940
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4f21f940
Branch: refs/heads/branch-2.8
Commit: 4f21f9408e6e434f79d677ab361263eb1a2b8dd0
Parents: 41986d5
Author: Xiaoyu Yao <xy...@apache.org>
Authored: Fri May 8 14:46:25 2015 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:44 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++-
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md | 5 +++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f21f940/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d1bfd42..e53a974 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1739,7 +1739,8 @@ Release 2.7.2 - UNRELEASED
HDFS-9445. Datanode may deadlock while handling a bad volume.
(Wlater Su via Kihwal)
-Release 2.7.1 - 2015-07-06
+ HDFS-8326. Documentation about when checkpoints are run is out of date.
+ (Misty Stanley-Jones via xyao)
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4f21f940/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
index 7471666..3872f57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsDesign.md
@@ -155,9 +155,10 @@ The Persistence of File System Metadata
The HDFS namespace is stored by the NameNode. The NameNode uses a transaction log called the EditLog to persistently record every change that occurs to file system metadata. For example, creating a new file in HDFS causes the NameNode to insert a record into the EditLog indicating this. Similarly, changing the replication factor of a file causes a new record to be inserted into the EditLog. The NameNode uses a file in its local host OS file system to store the EditLog. The entire file system namespace, including the mapping of blocks to files and file system properties, is stored in a file called the FsImage. The FsImage is stored as a file in the NameNode’s local file system too.
-The NameNode keeps an image of the entire file system namespace and file Blockmap in memory. This key metadata item is designed to be compact, such that a NameNode with 4 GB of RAM is plenty to support a huge number of files and directories. When the NameNode starts up, it reads the FsImage and EditLog from disk, applies all the transactions from the EditLog to the in-memory representation of the FsImage, and flushes out this new version into a new FsImage on disk. It can then truncate the old EditLog because its transactions have been applied to the persistent FsImage. This process is called a checkpoint. In the current implementation, a checkpoint only occurs when the NameNode starts up. Work is in progress to support periodic checkpointing in the near future.
+The NameNode keeps an image of the entire file system namespace and file Blockmap in memory. When the NameNode starts up, or a checkpoint is triggered by a configurable threshold, it reads the FsImage and EditLog from disk, applies all the transactions from the EditLog to the in-memory representation of the FsImage, and flushes out this new version into a new FsImage on disk. It can then truncate the old EditLog because its transactions have been applied to the persistent FsImage. This process is called a checkpoint. The purpose of a checkpoint is to make sure that HDFS has a consistent view of the file system metadata by taking a snapshot of the file system metadata and saving it to FsImage. Even though it is efficient to read a FsImage, it is not efficient to make incremental edits directly to a FsImage. Instead of modifying FsImage for each edit, we persist the edits in the Editlog. During the checkpoint the changes from Editlog are applied to the FsImage. A checkpoint can be tri
ggered at a given time interval (`dfs.namenode.checkpoint.period`) expressed in seconds, or after a given number of filesystem transactions have accumulated (`dfs.namenode.checkpoint.txns`). If both of these properties are set, the first threshold to be reached triggers a checkpoint.
+
+The DataNode stores HDFS data in files in its local file system. The DataNode has no knowledge about HDFS files. It stores each block of HDFS data in a separate file in its local file system. The DataNode does not create all files in the same directory. Instead, it uses a heuristic to determine the optimal number of files per directory and creates subdirectories appropriately. It is not optimal to create all local files in the same directory because the local file system might not be able to efficiently support a huge number of files in a single directory. When a DataNode starts up, it scans through its local file system, generates a list of all HDFS data blocks that correspond to each of these local files, and sends this report to the NameNode. The report is called the _Blockreport_.
-The DataNode stores HDFS data in files in its local file system. The DataNode has no knowledge about HDFS files. It stores each block of HDFS data in a separate file in its local file system. The DataNode does not create all files in the same directory. Instead, it uses a heuristic to determine the optimal number of files per directory and creates subdirectories appropriately. It is not optimal to create all local files in the same directory because the local file system might not be able to efficiently support a huge number of files in a single directory. When a DataNode starts up, it scans through its local file system, generates a list of all HDFS data blocks that correspond to each of these local files and sends this report to the NameNode: this is the Blockreport.
The Communication Protocols
---------------------------
[12/20] hadoop git commit: YARN-4424. Fix deadlock in RMAppImpl.
(Jian he via wangda)
Posted by wa...@apache.org.
YARN-4424. Fix deadlock in RMAppImpl. (Jian he via wangda)
(cherry picked from commit 7e4715186d31ac889fba26d453feedcebb11fc70)
Conflicts:
hadoop-yarn-project/CHANGES.txt
(cherry picked from commit 7013f9d6cda88e72a839b1c55757615b55101beb)
Conflicts:
hadoop-yarn-project/CHANGES.txt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4ba6c85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4ba6c85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4ba6c85
Branch: refs/heads/branch-2.8
Commit: a4ba6c85255dbb1d7dae4072db581c043e494dec
Parents: c4ae007
Author: Wangda Tan <wa...@apache.org>
Authored: Tue Dec 8 14:25:16 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:43 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 ++
.../server/resourcemanager/rmapp/RMAppImpl.java | 22 +++++++-------------
2 files changed, 10 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4ba6c85/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1764fc3..412999f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -2000,6 +2000,8 @@ Release 2.6.3 - UNRELEASED
YARN-4434. NodeManager Disk Checker parameter documentation is not correct.
(Weiwei Yang via aajisaka)
+ YARN-4424. Fix deadlock in RMAppImpl. (Jian he via wangda)
+
Release 2.6.2 - 2015-10-28
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4ba6c85/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index b1a5814..7a2b717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -490,21 +490,15 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public FinalApplicationStatus getFinalApplicationStatus() {
- this.readLock.lock();
- try {
- // finish state is obtained based on the state machine's current state
- // as a fall-back in case the application has not been unregistered
- // ( or if the app never unregistered itself )
- // when the report is requested
- if (currentAttempt != null
- && currentAttempt.getFinalApplicationStatus() != null) {
- return currentAttempt.getFinalApplicationStatus();
- }
- return
- createFinalApplicationStatus(this.stateMachine.getCurrentState());
- } finally {
- this.readLock.unlock();
+ // finish state is obtained based on the state machine's current state
+ // as a fall-back in case the application has not been unregistered
+ // ( or if the app never unregistered itself )
+ // when the report is requested
+ if (currentAttempt != null
+ && currentAttempt.getFinalApplicationStatus() != null) {
+ return currentAttempt.getFinalApplicationStatus();
}
+ return createFinalApplicationStatus(this.stateMachine.getCurrentState());
}
@Override
[11/20] hadoop git commit: YARN-4422. Generic AHS sometimes doesn't
show started, node, or logs on App page (Eric Payne via jeagles)
Posted by wa...@apache.org.
YARN-4422. Generic AHS sometimes doesn't show started, node, or logs on App page (Eric Payne via jeagles)
(cherry picked from commit 4ff973f96ae7f77cda3b52b38427e2991819ad31)
(cherry picked from commit 7c604c3d065ab15927ef7e14be6acf7772f6077f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c4ae007f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c4ae007f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c4ae007f
Branch: refs/heads/branch-2.8
Commit: c4ae007fd4a3c86a7d03ee7756efb36740880400
Parents: 46cb47c
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Mon Dec 7 15:04:48 2015 -0600
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Dec 18 16:34:43 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../ApplicationHistoryManagerOnTimelineStore.java | 7 +++++++
.../resourcemanager/metrics/AppAttemptFinishedEvent.java | 10 +++++++++-
.../resourcemanager/metrics/SystemMetricsPublisher.java | 6 +++++-
4 files changed, 24 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4ae007f/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cb86de2..1764fc3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -1110,6 +1110,9 @@ Release 2.7.3 - UNRELEASED
YARN-4398. Remove unnecessary synchronization in RMStateStore. (Ning Ding via jianhe)
+ YARN-4422. Generic AHS sometimes doesn't show started, node, or logs on App page
+ (Eric Payne via jeagles)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4ae007f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
index 9b4e78e..6e6b9fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java
@@ -488,6 +488,13 @@ public class ApplicationHistoryManagerOnTimelineStore extends AbstractService
AppAttemptMetricsConstants.STATE_EVENT_INFO)
.toString());
}
+ if (eventInfo
+ .containsKey(AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)) {
+ amContainerId =
+ ConverterUtils.toContainerId(eventInfo.get(
+ AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO)
+ .toString());
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4ae007f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
index 71d9363..fc1d10f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/AppAttemptFinishedEvent.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.resourcemanager.metrics;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
@@ -31,6 +32,7 @@ public class AppAttemptFinishedEvent extends
private String diagnosticsInfo;
private FinalApplicationStatus appStatus;
private YarnApplicationAttemptState state;
+ private ContainerId masterContainerId;
public AppAttemptFinishedEvent(
ApplicationAttemptId appAttemptId,
@@ -39,7 +41,8 @@ public class AppAttemptFinishedEvent extends
String diagnosticsInfo,
FinalApplicationStatus appStatus,
YarnApplicationAttemptState state,
- long finishedTime) {
+ long finishedTime,
+ ContainerId masterContainerId) {
super(SystemMetricsEventType.APP_ATTEMPT_FINISHED, finishedTime);
this.appAttemptId = appAttemptId;
// This is the tracking URL after the application attempt is finished
@@ -48,6 +51,7 @@ public class AppAttemptFinishedEvent extends
this.diagnosticsInfo = diagnosticsInfo;
this.appStatus = appStatus;
this.state = state;
+ this.masterContainerId = masterContainerId;
}
@Override
@@ -79,4 +83,8 @@ public class AppAttemptFinishedEvent extends
return state;
}
+ public ContainerId getMasterContainerId() {
+ return masterContainerId;
+ }
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c4ae007f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
index 71153c2..56d1a68 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/metrics/SystemMetricsPublisher.java
@@ -188,7 +188,8 @@ public class SystemMetricsPublisher extends CompositeService {
// based on app state if it doesn't exist
app.getFinalApplicationStatus(),
RMServerUtils.createApplicationAttemptState(appAttemtpState),
- finishedTime));
+ finishedTime,
+ appAttempt.getMasterContainer().getId()));
}
}
@@ -419,6 +420,9 @@ public class SystemMetricsPublisher extends CompositeService {
event.getFinalApplicationStatus().toString());
eventInfo.put(AppAttemptMetricsConstants.STATE_EVENT_INFO,
event.getYarnApplicationAttemptState().toString());
+ eventInfo.put(
+ AppAttemptMetricsConstants.MASTER_CONTAINER_EVENT_INFO,
+ event.getMasterContainerId().toString());
tEvent.setEventInfo(eventInfo);
entity.addEvent(tEvent);
putEntity(entity);