You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/02/18 22:18:59 UTC
[01/50] [abbrv] hadoop git commit: HADOOP-11589.
NetUtils.createSocketAddr should trim the input URI. Contributed by Rakesh R.
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 e2786151d -> 5a3d0af56
HADOOP-11589. NetUtils.createSocketAddr should trim the input URI. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/965ce9e2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/965ce9e2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/965ce9e2
Branch: refs/heads/YARN-2928
Commit: 965ce9e24ed051c6705defc0f85dcfaa4fc9a3ef
Parents: 6804d68
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Sun Feb 15 00:13:10 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Sun Feb 15 00:13:10 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 1 +
.../java/org/apache/hadoop/conf/TestConfiguration.java | 10 ++++++++++
3 files changed, 14 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/965ce9e2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 99320cb..522ec47 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -602,6 +602,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-9869. Configuration.getSocketAddr()/getEnum() should use
getTrimmed(). (Tsuyoshi Ozawa via aajisaka)
+ HADOOP-11589. NetUtils.createSocketAddr should trim the input URI.
+ (Rakesh R via ozawa)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/965ce9e2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index ef1092b..e475149 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -189,6 +189,7 @@ public class NetUtils {
throw new IllegalArgumentException("Target address cannot be null." +
helpText);
}
+ target = target.trim();
boolean hasScheme = target.contains("://");
URI uri = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/965ce9e2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index a367553..b7c76eb 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -1480,6 +1480,16 @@ public class TestConfiguration extends TestCase {
// it's expected behaviour.
}
+ public void testTrimCreateSocketAddress() {
+ Configuration conf = new Configuration();
+ NetUtils.addStaticResolution("host", "127.0.0.1");
+ final String defaultAddr = "host:1 ";
+
+ InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);
+ conf.setSocketAddr("myAddress", addr);
+ assertEquals(defaultAddr.trim(), NetUtils.getHostPortString(addr));
+ }
+
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()
[02/50] [abbrv] hadoop git commit: Rvert "HADOOP-11589.
NetUtils.createSocketAddr should trim the input URI. Contributed by Rakesh
R."
Posted by zj...@apache.org.
Rvert "HADOOP-11589. NetUtils.createSocketAddr should trim the input URI. Contributed by Rakesh R."
This reverts commit 965ce9e24ed051c6705defc0f85dcfaa4fc9a3ef.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b7d48286
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b7d48286
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b7d48286
Branch: refs/heads/YARN-2928
Commit: b7d48286a94d469483f0af0ff52fb7341552b00c
Parents: 965ce9e
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Sun Feb 15 00:28:59 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Sun Feb 15 00:28:59 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ---
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 1 -
.../java/org/apache/hadoop/conf/TestConfiguration.java | 10 ----------
3 files changed, 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d48286/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 522ec47..99320cb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -602,9 +602,6 @@ Release 2.7.0 - UNRELEASED
HADOOP-9869. Configuration.getSocketAddr()/getEnum() should use
getTrimmed(). (Tsuyoshi Ozawa via aajisaka)
- HADOOP-11589. NetUtils.createSocketAddr should trim the input URI.
- (Rakesh R via ozawa)
-
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d48286/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index e475149..ef1092b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -189,7 +189,6 @@ public class NetUtils {
throw new IllegalArgumentException("Target address cannot be null." +
helpText);
}
- target = target.trim();
boolean hasScheme = target.contains("://");
URI uri = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b7d48286/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index b7c76eb..a367553 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -1480,16 +1480,6 @@ public class TestConfiguration extends TestCase {
// it's expected behaviour.
}
- public void testTrimCreateSocketAddress() {
- Configuration conf = new Configuration();
- NetUtils.addStaticResolution("host", "127.0.0.1");
- final String defaultAddr = "host:1 ";
-
- InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);
- conf.setSocketAddr("myAddress", addr);
- assertEquals(defaultAddr.trim(), NetUtils.getHostPortString(addr));
- }
-
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()
[27/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
MAPREDUCE-6260. Convert site documentation to markdown (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8b787e2f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8b787e2f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8b787e2f
Branch: refs/heads/YARN-2928
Commit: 8b787e2fdbd0050c0345cf14b26af9d61049068f
Parents: 34b78d5
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 17 06:52:14 2015 -1000
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 17 06:52:14 2015 -1000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../src/site/apt/DistributedCacheDeploy.apt.vm | 151 -
.../src/site/apt/EncryptedShuffle.apt.vm | 320 ---
.../src/site/apt/MapReduceTutorial.apt.vm | 1605 -----------
...pReduce_Compatibility_Hadoop1_Hadoop2.apt.vm | 114 -
.../src/site/apt/MapredAppMasterRest.apt.vm | 2709 ------------------
.../src/site/apt/MapredCommands.apt.vm | 233 --
.../apt/PluggableShuffleAndPluggableSort.apt.vm | 98 -
.../site/markdown/DistributedCacheDeploy.md.vm | 119 +
.../src/site/markdown/EncryptedShuffle.md | 255 ++
.../src/site/markdown/MapReduceTutorial.md | 1156 ++++++++
.../MapReduce_Compatibility_Hadoop1_Hadoop2.md | 69 +
.../src/site/markdown/MapredAppMasterRest.md | 2397 ++++++++++++++++
.../src/site/markdown/MapredCommands.md | 153 +
.../PluggableShuffleAndPluggableSort.md | 73 +
.../src/site/apt/HistoryServerRest.apt.vm | 2672 -----------------
.../src/site/markdown/HistoryServerRest.md | 2361 +++++++++++++++
17 files changed, 6586 insertions(+), 7902 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 9ef7a32..aebc71e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -96,6 +96,9 @@ Trunk (Unreleased)
MAPREDUCE-6250. deprecate sbin/mr-jobhistory-daemon.sh (aw)
+ MAPREDUCE-6260. Convert site documentation to markdown (Masatake Iwasaki
+ via aw)
+
BUG FIXES
MAPREDUCE-6191. Improve clearing stale state of Java serialization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm
deleted file mode 100644
index 2195e10..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/DistributedCacheDeploy.apt.vm
+++ /dev/null
@@ -1,151 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Map Reduce Next Generation-${project.version} - Distributed Cache Deploy
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop MapReduce Next Generation - Distributed Cache Deploy
-
-* Introduction
-
- The MapReduce application framework has rudimentary support for deploying a
- new version of the MapReduce framework via the distributed cache. By setting
- the appropriate configuration properties, users can run a different version
- of MapReduce than the one initially deployed to the cluster. For example,
- cluster administrators can place multiple versions of MapReduce in HDFS and
- configure <<<mapred-site.xml>>> to specify which version jobs will use by
- default. This allows the administrators to perform a rolling upgrade of the
- MapReduce framework under certain conditions.
-
-* Preconditions and Limitations
-
- The support for deploying the MapReduce framework via the distributed cache
- currently does not address the job client code used to submit and query
- jobs. It also does not address the <<<ShuffleHandler>>> code that runs as an
- auxilliary service within each NodeManager. As a result the following
- limitations apply to MapReduce versions that can be successfully deployed via
- the distributed cache in a rolling upgrade fashion:
-
- * The MapReduce version must be compatible with the job client code used to
- submit and query jobs. If it is incompatible then the job client must be
- upgraded separately on any node from which jobs using the new MapReduce
- version will be submitted or queried.
-
- * The MapReduce version must be compatible with the configuration files used
- by the job client submitting the jobs. If it is incompatible with that
- configuration (e.g.: a new property must be set or an existing property
- value changed) then the configuration must be updated first.
-
- * The MapReduce version must be compatible with the <<<ShuffleHandler>>>
- version running on the nodes in the cluster. If it is incompatible then the
- new <<<ShuffleHandler>>> code must be deployed to all the nodes in the
- cluster, and the NodeManagers must be restarted to pick up the new
- <<<ShuffleHandler>>> code.
-
-* Deploying a New MapReduce Version via the Distributed Cache
-
- Deploying a new MapReduce version consists of three steps:
-
- [[1]] Upload the MapReduce archive to a location that can be accessed by the
- job submission client. Ideally the archive should be on the cluster's default
- filesystem at a publicly-readable path. See the archive location discussion
- below for more details.
-
- [[2]] Configure <<<mapreduce.application.framework.path>>> to point to the
- location where the archive is located. As when specifying distributed cache
- files for a job, this is a URL that also supports creating an alias for the
- archive if a URL fragment is specified. For example,
- <<<hdfs:/mapred/framework/hadoop-mapreduce-${project.version}.tar.gz#mrframework>>>
- will be localized as <<<mrframework>>> rather than
- <<<hadoop-mapreduce-${project.version}.tar.gz>>>.
-
- [[3]] Configure <<<mapreduce.application.classpath>>> to set the proper
- classpath to use with the MapReduce archive configured above. NOTE: An error
- occurs if <<<mapreduce.application.framework.path>>> is configured but
- <<<mapreduce.application.classpath>>> does not reference the base name of the
- archive path or the alias if an alias was specified.
-
-** Location of the MapReduce Archive and How It Affects Job Performance
-
- Note that the location of the MapReduce archive can be critical to job
- submission and job startup performance. If the archive is not located on the
- cluster's default filesystem then it will be copied to the job staging
- directory for each job and localized to each node where the job's tasks
- run. This will slow down job submission and task startup performance.
-
- If the archive is located on the default filesystem then the job client will
- not upload the archive to the job staging directory for each job
- submission. However if the archive path is not readable by all cluster users
- then the archive will be localized separately for each user on each node
- where tasks execute. This can cause unnecessary duplication in the
- distributed cache.
-
- When working with a large cluster it can be important to increase the
- replication factor of the archive to increase its availability. This will
- spread the load when the nodes in the cluster localize the archive for the
- first time.
-
-* MapReduce Archives and Classpath Configuration
-
- Setting a proper classpath for the MapReduce archive depends upon the
- composition of the archive and whether it has any additional dependencies.
- For example, the archive can contain not only the MapReduce jars but also the
- necessary YARN, HDFS, and Hadoop Common jars and all other dependencies. In
- that case, <<<mapreduce.application.classpath>>> would be configured to
- something like the following example, where the archive basename is
- hadoop-mapreduce-${project.version}.tar.gz and the archive is organized
- internally similar to the standard Hadoop distribution archive:
-
- <<<$HADOOP_CONF_DIR,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/lib/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/common/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/common/lib/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/yarn/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/yarn/lib/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/hdfs/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/hdfs/lib/*>>>
-
- Another possible approach is to have the archive consist of just the
- MapReduce jars and have the remaining dependencies picked up from the Hadoop
- distribution installed on the nodes. In that case, the above example would
- change to something like the following:
-
- <<<$HADOOP_CONF_DIR,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/lib/*,$HADOOP_COMMON_HOME/share/hadoop/common/*,$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,$HADOOP_YARN_HOME/share/hadoop/yarn/*,$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*>>>
-
-** NOTE:
-
- If shuffle encryption is also enabled in the cluster, then we could meet the problem that MR job get failed with exception like below:
-
-+---+
-2014-10-10 02:17:16,600 WARN [fetcher#1] org.apache.hadoop.mapreduce.task.reduce.Fetcher: Failed to connect to junpingdu-centos5-3.cs1cloud.internal:13562 with 1 map outputs
-javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
- at com.sun.net.ssl.internal.ssl.Alerts.getSSLException(Alerts.java:174)
- at com.sun.net.ssl.internal.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1731)
- at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:241)
- at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:235)
- at com.sun.net.ssl.internal.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1206)
- at com.sun.net.ssl.internal.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:136)
- at com.sun.net.ssl.internal.ssl.Handshaker.processLoop(Handshaker.java:593)
- at com.sun.net.ssl.internal.ssl.Handshaker.process_record(Handshaker.java:529)
- at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:925)
- at com.sun.net.ssl.internal.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1170)
- at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1197)
- at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1181)
- at sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:434)
- at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.setNewClient(AbstractDelegateHttpsURLConnection.java:81)
- at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.setNewClient(AbstractDelegateHttpsURLConnection.java:61)
- at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:584)
- at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1193)
- at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:379)
- at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:318)
- at org.apache.hadoop.mapreduce.task.reduce.Fetcher.verifyConnection(Fetcher.java:427)
-....
-
-+---+
-
- This is because MR client (deployed from HDFS) cannot access ssl-client.xml in local FS under directory of $HADOOP_CONF_DIR. To fix the problem, we can add the directory with ssl-client.xml to the classpath of MR which is specified in "mapreduce.application.classpath" as mentioned above. To avoid MR application being affected by other local configurations, it is better to create a dedicated directory for putting ssl-client.xml, e.g. a sub-directory under $HADOOP_CONF_DIR, like: $HADOOP_CONF_DIR/security.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
deleted file mode 100644
index 1761ad8..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/EncryptedShuffle.apt.vm
+++ /dev/null
@@ -1,320 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Map Reduce Next Generation-${project.version} - Encrypted Shuffle
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop MapReduce Next Generation - Encrypted Shuffle
-
-* {Introduction}
-
- The Encrypted Shuffle capability allows encryption of the MapReduce shuffle
- using HTTPS and with optional client authentication (also known as
- bi-directional HTTPS, or HTTPS with client certificates). It comprises:
-
- * A Hadoop configuration setting for toggling the shuffle between HTTP and
- HTTPS.
-
- * A Hadoop configuration settings for specifying the keystore and truststore
- properties (location, type, passwords) used by the shuffle service and the
- reducers tasks fetching shuffle data.
-
- * A way to re-load truststores across the cluster (when a node is added or
- removed).
-
-* {Configuration}
-
-** <<core-site.xml>> Properties
-
- To enable encrypted shuffle, set the following properties in core-site.xml of
- all nodes in the cluster:
-
-*--------------------------------------+---------------------+-----------------+
-| <<Property>> | <<Default Value>> | <<Explanation>> |
-*--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.require.client.cert>>> | <<<false>>> | Whether client certificates are required |
-*--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.hostname.verifier>>> | <<<DEFAULT>>> | The hostname verifier to provide for HttpsURLConnections. Valid values are: <<DEFAULT>>, <<STRICT>>, <<STRICT_I6>>, <<DEFAULT_AND_LOCALHOST>> and <<ALLOW_ALL>> |
-*--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.keystores.factory.class>>> | <<<org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory>>> | The KeyStoresFactory implementation to use |
-*--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.server.conf>>> | <<<ssl-server.xml>>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
-*--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.client.conf>>> | <<<ssl-client.xml>>> | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
-*--------------------------------------+---------------------+-----------------+
-| <<<hadoop.ssl.enabled.protocols>>> | <<<TLSv1>>> | The supported SSL protocols (JDK6 can use <<TLSv1>>, JDK7+ can use <<TLSv1,TLSv1.1,TLSv1.2>>) |
-*--------------------------------------+---------------------+-----------------+
-
- <<IMPORTANT:>> Currently requiring client certificates should be set to false.
- Refer the {{{ClientCertificates}Client Certificates}} section for details.
-
- <<IMPORTANT:>> All these properties should be marked as final in the cluster
- configuration files.
-
-*** Example:
-
-------
- ...
- <property>
- <name>hadoop.ssl.require.client.cert</name>
- <value>false</value>
- <final>true</final>
- </property>
-
- <property>
- <name>hadoop.ssl.hostname.verifier</name>
- <value>DEFAULT</value>
- <final>true</final>
- </property>
-
- <property>
- <name>hadoop.ssl.keystores.factory.class</name>
- <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
- <final>true</final>
- </property>
-
- <property>
- <name>hadoop.ssl.server.conf</name>
- <value>ssl-server.xml</value>
- <final>true</final>
- </property>
-
- <property>
- <name>hadoop.ssl.client.conf</name>
- <value>ssl-client.xml</value>
- <final>true</final>
- </property>
- ...
-------
-
-** <<<mapred-site.xml>>> Properties
-
- To enable encrypted shuffle, set the following property in mapred-site.xml
- of all nodes in the cluster:
-
-*--------------------------------------+---------------------+-----------------+
-| <<Property>> | <<Default Value>> | <<Explanation>> |
-*--------------------------------------+---------------------+-----------------+
-| <<<mapreduce.shuffle.ssl.enabled>>> | <<<false>>> | Whether encrypted shuffle is enabled |
-*--------------------------------------+---------------------+-----------------+
-
- <<IMPORTANT:>> This property should be marked as final in the cluster
- configuration files.
-
-*** Example:
-
-------
- ...
- <property>
- <name>mapreduce.shuffle.ssl.enabled</name>
- <value>true</value>
- <final>true</final>
- </property>
- ...
-------
-
- The Linux container executor should be set to prevent job tasks from
- reading the server keystore information and gaining access to the shuffle
- server certificates.
-
- Refer to Hadoop Kerberos configuration for details on how to do this.
-
-* {Keystore and Truststore Settings}
-
- Currently <<<FileBasedKeyStoresFactory>>> is the only <<<KeyStoresFactory>>>
- implementation. The <<<FileBasedKeyStoresFactory>>> implementation uses the
- following properties, in the <<ssl-server.xml>> and <<ssl-client.xml>> files,
- to configure the keystores and truststores.
-
-** <<<ssl-server.xml>>> (Shuffle server) Configuration:
-
- The mapred user should own the <<ssl-server.xml>> file and have exclusive
- read access to it.
-
-*---------------------------------------------+---------------------+-----------------+
-| <<Property>> | <<Default Value>> | <<Explanation>> |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.keystore.type>>> | <<<jks>>> | Keystore file type |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.keystore.location>>> | NONE | Keystore file location. The mapred user should own this file and have exclusive read access to it. |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.keystore.password>>> | NONE | Keystore file password |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.truststore.type>>> | <<<jks>>> | Truststore file type |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.truststore.location>>> | NONE | Truststore file location. The mapred user should own this file and have exclusive read access to it. |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.truststore.password>>> | NONE | Truststore file password |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.server.truststore.reload.interval>>> | 10000 | Truststore reload interval, in milliseconds |
-*--------------------------------------+----------------------------+-----------------+
-
-*** Example:
-
-------
-<configuration>
-
- <!-- Server Certificate Store -->
- <property>
- <name>ssl.server.keystore.type</name>
- <value>jks</value>
- </property>
- <property>
- <name>ssl.server.keystore.location</name>
- <value>${user.home}/keystores/server-keystore.jks</value>
- </property>
- <property>
- <name>ssl.server.keystore.password</name>
- <value>serverfoo</value>
- </property>
-
- <!-- Server Trust Store -->
- <property>
- <name>ssl.server.truststore.type</name>
- <value>jks</value>
- </property>
- <property>
- <name>ssl.server.truststore.location</name>
- <value>${user.home}/keystores/truststore.jks</value>
- </property>
- <property>
- <name>ssl.server.truststore.password</name>
- <value>clientserverbar</value>
- </property>
- <property>
- <name>ssl.server.truststore.reload.interval</name>
- <value>10000</value>
- </property>
-</configuration>
-------
-
-** <<<ssl-client.xml>>> (Reducer/Fetcher) Configuration:
-
- The mapred user should own the <<ssl-client.xml>> file and it should have
- default permissions.
-
-*---------------------------------------------+---------------------+-----------------+
-| <<Property>> | <<Default Value>> | <<Explanation>> |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.keystore.type>>> | <<<jks>>> | Keystore file type |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.keystore.location>>> | NONE | Keystore file location. The mapred user should own this file and it should have default permissions. |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.keystore.password>>> | NONE | Keystore file password |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.truststore.type>>> | <<<jks>>> | Truststore file type |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.truststore.location>>> | NONE | Truststore file location. The mapred user should own this file and it should have default permissions. |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.truststore.password>>> | NONE | Truststore file password |
-*---------------------------------------------+---------------------+-----------------+
-| <<<ssl.client.truststore.reload.interval>>> | 10000 | Truststore reload interval, in milliseconds |
-*--------------------------------------+----------------------------+-----------------+
-
-*** Example:
-
-------
-<configuration>
-
- <!-- Client certificate Store -->
- <property>
- <name>ssl.client.keystore.type</name>
- <value>jks</value>
- </property>
- <property>
- <name>ssl.client.keystore.location</name>
- <value>${user.home}/keystores/client-keystore.jks</value>
- </property>
- <property>
- <name>ssl.client.keystore.password</name>
- <value>clientfoo</value>
- </property>
-
- <!-- Client Trust Store -->
- <property>
- <name>ssl.client.truststore.type</name>
- <value>jks</value>
- </property>
- <property>
- <name>ssl.client.truststore.location</name>
- <value>${user.home}/keystores/truststore.jks</value>
- </property>
- <property>
- <name>ssl.client.truststore.password</name>
- <value>clientserverbar</value>
- </property>
- <property>
- <name>ssl.client.truststore.reload.interval</name>
- <value>10000</value>
- </property>
-</configuration>
-------
-
-* Activating Encrypted Shuffle
-
- When you have made the above configuration changes, activate Encrypted
- Shuffle by re-starting all NodeManagers.
-
- <<IMPORTANT:>> Using encrypted shuffle will incur in a significant
- performance impact. Users should profile this and potentially reserve
- 1 or more cores for encrypted shuffle.
-
-* {ClientCertificates} Client Certificates
-
- Using Client Certificates does not fully ensure that the client is a
- reducer task for the job. Currently, Client Certificates (their private key)
- keystore files must be readable by all users submitting jobs to the cluster.
- This means that a rogue job could read such those keystore files and use
- the client certificates in them to establish a secure connection with a
- Shuffle server. However, unless the rogue job has a proper JobToken, it won't
- be able to retrieve shuffle data from the Shuffle server. A job, using its
- own JobToken, can only retrieve shuffle data that belongs to itself.
-
-* Reloading Truststores
-
- By default the truststores will reload their configuration every 10 seconds.
- If a new truststore file is copied over the old one, it will be re-read,
- and its certificates will replace the old ones. This mechanism is useful for
- adding or removing nodes from the cluster, or for adding or removing trusted
- clients. In these cases, the client or NodeManager certificate is added to
- (or removed from) all the truststore files in the system, and the new
- configuration will be picked up without you having to restart the NodeManager
- daemons.
-
-* Debugging
-
- <<NOTE:>> Enable debugging only for troubleshooting, and then only for jobs
- running on small amounts of data. It is very verbose and slows down jobs by
- several orders of magnitude. (You might need to increase mapred.task.timeout
- to prevent jobs from failing because tasks run so slowly.)
-
- To enable SSL debugging in the reducers, set <<<-Djavax.net.debug=all>>> in
- the <<<mapreduce.reduce.child.java.opts>>> property; for example:
-
-------
- <property>
- <name>mapred.reduce.child.java.opts</name>
- <value>-Xmx-200m -Djavax.net.debug=all</value>
- </property>
-------
-
- You can do this on a per-job basis, or by means of a cluster-wide setting in
- the <<<mapred-site.xml>>> file.
-
- To set this property in NodeManager, set it in the <<<yarn-env.sh>>> file:
-
-------
- YARN_NODEMANAGER_OPTS="-Djavax.net.debug=all $YARN_NODEMANAGER_OPTS"
-------
[06/50] [abbrv] hadoop git commit: YARN-3203. Correct a log message
in AuxServices. Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
YARN-3203. Correct a log message in AuxServices. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/447bd7b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/447bd7b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/447bd7b5
Branch: refs/heads/YARN-2928
Commit: 447bd7b5a61a5788dc2a5d29cedfc19f0e99c0f5
Parents: ab0b958
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Mon Feb 16 23:55:58 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Mon Feb 16 23:55:58 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/nodemanager/containermanager/AuxServices.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/447bd7b5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e00e447..7161dc3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -295,6 +295,9 @@ Release 2.7.0 - UNRELEASED
YARN-3182. Cleanup switch statement in ApplicationMasterLauncher#handle().
(Ray Chiang via ozawa)
+ YARN-3203. Correct a log message in AuxServices. (Brahma Reddy Battula
+ via ozawa)
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/447bd7b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index bf02679..fb6f79b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -128,7 +128,7 @@ public class AuxServices extends AbstractService
// TODO better use s.getName()?
if(!sName.equals(s.getName())) {
LOG.warn("The Auxilurary Service named '"+sName+"' in the "
- +"configuration is for class "+sClass+" which has "
+ +"configuration is for "+sClass+" which has "
+"a name of '"+s.getName()+"'. Because these are "
+"not the same tools trying to send ServiceData and read "
+"Service Meta Data may have issues unless the refer to "
[33/50] [abbrv] hadoop git commit: HADOOP-11521. Make connection
timeout configurable in s3a. (Thomas Demoor via stevel)
Posted by zj...@apache.org.
HADOOP-11521. Make connection timeout configurable in s3a. (Thomas Demoor via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00b80958
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00b80958
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00b80958
Branch: refs/heads/YARN-2928
Commit: 00b80958d862dbcc08d6f186c09963d3351ba0fd
Parents: 6be5670
Author: Steve Loughran <st...@apache.org>
Authored: Tue Feb 17 20:00:00 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Feb 17 20:06:27 2015 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop-common/src/main/resources/core-default.xml | 6 ++++++
.../src/main/java/org/apache/hadoop/fs/s3a/Constants.java | 4 ++++
.../src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 ++
.../hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md | 6 ++++++
5 files changed, 21 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 405e821..c1caf5f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -615,6 +615,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
+ HADOOP-11521. Make connection timeout configurable in s3a.
+ (Thomas Demoor via stevel)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 5eb301a..41121f1 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -758,6 +758,12 @@ for ldap providers in the same way as above does.
</property>
<property>
+ <name>fs.s3a.connection.establish.timeout</name>
+ <value>5000</value>
+ <description>Socket connection setup timeout in seconds.</description>
+</property>
+
+<property>
<name>fs.s3a.connection.timeout</name>
<value>50000</value>
<description>Socket connection timeout in seconds.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index b6863bb..1d4f67b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -42,6 +42,10 @@ public class Constants {
// number of times we should retry errors
public static final String MAX_ERROR_RETRIES = "fs.s3a.attempts.maximum";
public static final int DEFAULT_MAX_ERROR_RETRIES = 10;
+
+ // seconds until we give up trying to establish a connection to s3
+ public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
+ public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
// seconds until we give up on a connection to s3
public static final String SOCKET_TIMEOUT = "fs.s3a.connection.timeout";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index 22350bc..4de5c13 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -174,6 +174,8 @@ public class S3AFileSystem extends FileSystem {
awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
awsConf.setMaxErrorRetry(conf.getInt(MAX_ERROR_RETRIES,
DEFAULT_MAX_ERROR_RETRIES));
+ awsConf.setConnectionTimeout(conf.getInt(ESTABLISH_TIMEOUT,
+ DEFAULT_ESTABLISH_TIMEOUT));
awsConf.setSocketTimeout(conf.getInt(SOCKET_TIMEOUT,
DEFAULT_SOCKET_TIMEOUT));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00b80958/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 8f511dd..1e44864 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -211,6 +211,12 @@ If you do any of these: change your credentials immediately!
</property>
<property>
+ <name>fs.s3a.connection.establish.timeout</name>
+ <value>5000</value>
+ <description>Socket connection setup timeout in seconds.</description>
+ </property>
+
+ <property>
<name>fs.s3a.connection.timeout</name>
<value>50000</value>
<description>Socket connection timeout in seconds.</description>
[04/50] [abbrv] hadoop git commit: MAPREDUCE-6256. Removed unused
private methods in o.a.h.mapreduce.Job.java. Contributed by Naganarasimha G
R.
Posted by zj...@apache.org.
MAPREDUCE-6256. Removed unused private methods in o.a.h.mapreduce.Job.java. Contributed by Naganarasimha G R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3338f6d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3338f6d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3338f6d9
Branch: refs/heads/YARN-2928
Commit: 3338f6d90795dafd445891cbd2f4d46b53012382
Parents: ef950ea
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Sun Feb 15 10:09:42 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Sun Feb 15 10:09:42 2015 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../java/org/apache/hadoop/mapreduce/Job.java | 47 --------------------
2 files changed, 3 insertions(+), 47 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3338f6d9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 38982e4..bb9e105 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -291,6 +291,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6255. Fix JobCounter's format to use grouping separator.
(Ryu Kobayashi via ozawa)
+ MAPREDUCE-6256. Removed unused private methods in o.a.h.mapreduce.Job.java.
+ (Naganarasimha G R via ozawa)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3338f6d9/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
index 493ca5f..470290c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java
@@ -98,9 +98,6 @@ public class Job extends JobContextImpl implements JobContext {
"mapreduce.client.genericoptionsparser.used";
public static final String SUBMIT_REPLICATION =
"mapreduce.client.submit.file.replication";
- private static final String TASKLOG_PULL_TIMEOUT_KEY =
- "mapreduce.client.tasklog.timeout";
- private static final int DEFAULT_TASKLOG_TIMEOUT = 60000;
@InterfaceStability.Evolving
public static enum TaskStatusFilter { NONE, KILLED, FAILED, SUCCEEDED, ALL }
@@ -340,10 +337,6 @@ public class Job extends JobContextImpl implements JobContext {
updateStatus();
return status;
}
-
- private void setStatus(JobStatus status) {
- this.status = status;
- }
/**
* Returns the current state of the Job.
@@ -1393,46 +1386,6 @@ public class Job extends JobContextImpl implements JobContext {
return success;
}
- /**
- * @return true if the profile parameters indicate that this is using
- * hprof, which generates profile files in a particular location
- * that we can retrieve to the client.
- */
- private boolean shouldDownloadProfile() {
- // Check the argument string that was used to initialize profiling.
- // If this indicates hprof and file-based output, then we're ok to
- // download.
- String profileParams = getProfileParams();
-
- if (null == profileParams) {
- return false;
- }
-
- // Split this on whitespace.
- String [] parts = profileParams.split("[ \\t]+");
-
- // If any of these indicate hprof, and the use of output files, return true.
- boolean hprofFound = false;
- boolean fileFound = false;
- for (String p : parts) {
- if (p.startsWith("-agentlib:hprof") || p.startsWith("-Xrunhprof")) {
- hprofFound = true;
-
- // This contains a number of comma-delimited components, one of which
- // may specify the file to write to. Make sure this is present and
- // not empty.
- String [] subparts = p.split(",");
- for (String sub : subparts) {
- if (sub.startsWith("file=") && sub.length() != "file=".length()) {
- fileFound = true;
- }
- }
- }
- }
-
- return hprofFound && fileFound;
- }
-
private void printTaskEvents(TaskCompletionEvent[] events,
Job.TaskStatusFilter filter, boolean profiling, IntegerRanges mapRanges,
IntegerRanges reduceRanges) throws IOException, InterruptedException {
[48/50] [abbrv] hadoop git commit: YARN-3132. RMNodeLabelsManager
should remove node from node-to-label mapping when node becomes deactivated.
Contributed by Wangda Tan
Posted by zj...@apache.org.
YARN-3132. RMNodeLabelsManager should remove node from node-to-label mapping when node becomes deactivated. Contributed by Wangda Tan
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f5da5566
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f5da5566
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f5da5566
Branch: refs/heads/YARN-2928
Commit: f5da5566d9c392a5df71a2dce4c2d0d50eea51ee
Parents: 4981d08
Author: Jian He <ji...@apache.org>
Authored: Wed Feb 18 11:51:51 2015 -0800
Committer: Jian He <ji...@apache.org>
Committed: Wed Feb 18 11:51:51 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodelabels/CommonNodeLabelsManager.java | 3 +-
.../nodelabels/RMNodeLabelsManager.java | 20 ++++++-
.../nodelabels/TestRMNodeLabelsManager.java | 63 ++++++++++++++++++--
4 files changed, 78 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da5566/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cbba046..884b506 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -611,6 +611,9 @@ Release 2.7.0 - UNRELEASED
YARN-3207. Secondary filter matches entites which do not have the key being
filtered for. (Zhijie Shen via xgong)
+ YARN-3132. RMNodeLabelsManager should remove node from node-to-label mapping
+ when node becomes deactivated. (Wangda Tan via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da5566/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
index cb6f1f3..e2da664 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/nodelabels/CommonNodeLabelsManager.java
@@ -45,7 +45,6 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.AsyncDispatcher;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEvent;
import org.apache.hadoop.yarn.nodelabels.event.NodeLabelsStoreEventType;
@@ -496,7 +495,7 @@ public class CommonNodeLabelsManager extends AbstractService {
}
}
- private void removeNodeFromLabels(NodeId node, Set<String> labels) {
+ protected void removeNodeFromLabels(NodeId node, Set<String> labels) {
for(String l : labels) {
labelCollections.get(l).removeNodeId(node);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da5566/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 9942d80..e5abdc9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -228,9 +228,23 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
Map<String, Host> before = cloneNodeMap(ImmutableSet.of(nodeId));
Node nm = getNMInNodeSet(nodeId);
if (null != nm) {
- // set nm is not running, and its resource = 0
- nm.running = false;
- nm.resource = Resource.newInstance(0, 0);
+ if (null == nm.labels) {
+ // When node deactivated, remove the nm from node collection if no
+ // labels explicitly set for this particular nm
+
+ // Save labels first, we need to remove label->nodes relation later
+ Set<String> savedNodeLabels = getLabelsOnNode(nodeId);
+
+ // Remove this node in nodes collection
+ nodeCollections.get(nodeId.getHost()).nms.remove(nodeId);
+
+ // Remove this node in labels->node
+ removeNodeFromLabels(nodeId, savedNodeLabels);
+ } else {
+ // set nm is not running, and its resource = 0
+ nm.running = false;
+ nm.resource = Resource.newInstance(0, 0);
+ }
}
// get the node after edition
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f5da5566/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
index a91947f..8a37c24 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/TestRMNodeLabelsManager.java
@@ -62,7 +62,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
}
@Test(timeout = 5000)
- public void testNodeActiveDeactiveUpdate() throws Exception {
+ public void testGetLabelResourceWhenNodeActiveDeactive() throws Exception {
mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3")));
@@ -119,7 +119,7 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
@SuppressWarnings({ "unchecked", "rawtypes" })
@Test(timeout = 5000)
- public void testUpdateNodeLabelWithActiveNode() throws Exception {
+ public void testGetLabelResource() throws Exception {
mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3")));
@@ -430,6 +430,52 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
}
}
+ @Test(timeout = 5000)
+ public void testGetLabelsOnNodesWhenNodeActiveDeactive() throws Exception {
+ mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(
+ toNodeId("n1"), toSet("p2")));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1:1"), toSet("p1")));
+
+ // Active/Deactive a node directly assigned label, should not remove from
+ // node->label map
+ mgr.activateNode(toNodeId("n1:1"), SMALL_RESOURCE);
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:1")),
+ toSet("p1"));
+ mgr.deactivateNode(toNodeId("n1:1"));
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:1")),
+ toSet("p1"));
+ // Host will not affected
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1")),
+ toSet("p2"));
+
+ // Active/Deactive a node doesn't directly assigned label, should remove
+ // from node->label map
+ mgr.activateNode(toNodeId("n1:2"), SMALL_RESOURCE);
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:2")),
+ toSet("p2"));
+ mgr.deactivateNode(toNodeId("n1:2"));
+ Assert.assertNull(mgr.getNodeLabels().get(toNodeId("n1:2")));
+ // Host will not affected too
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1")),
+ toSet("p2"));
+
+ // When we change label on the host after active a node without directly
+ // assigned label, such node will still be removed after deactive
+ // Active/Deactive a node doesn't directly assigned label, should remove
+ // from node->label map
+ mgr.activateNode(toNodeId("n1:2"), SMALL_RESOURCE);
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p3")));
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1:2")),
+ toSet("p3"));
+ mgr.deactivateNode(toNodeId("n1:2"));
+ Assert.assertNull(mgr.getNodeLabels().get(toNodeId("n1:2")));
+ // Host will not affected too
+ assertCollectionEquals(mgr.getNodeLabels().get(toNodeId("n1")),
+ toSet("p3"));
+
+ }
+
private void checkNodeLabelInfo(List<NodeLabel> infos, String labelName, int activeNMs, int memory) {
for (NodeLabel info : infos) {
if (info.getLabelName().equals(labelName)) {
@@ -470,19 +516,24 @@ public class TestRMNodeLabelsManager extends NodeLabelTestBase {
mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
// Add labels and replace labels on node
- mgr.addToCluserNodeLabels(toSet("p1", "p2", "p3"));
- mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1"),
- toNodeId("n2"), toSet("p2"), toNodeId("n3"), toSet("p3")));
+ mgr.addToCluserNodeLabels(toSet("p1"));
+ mgr.replaceLabelsOnNode(ImmutableMap.of(toNodeId("n1"), toSet("p1")));
+ // p1 -> n1, n1:1
+ Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
assertLabelsToNodesEquals(
mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
// Activate a node for which host to label mapping exists
mgr.activateNode(NodeId.newInstance("n1", 2), Resource.newInstance(10, 0));
+ // p1 -> n1, n1:1, n1:2
+ Assert.assertEquals(3, mgr.getLabelsToNodes().get("p1").size());
assertLabelsToNodesEquals(
mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
- // Deactivate a node. Label mapping should still exist.
+ // Deactivate a node. n1:1 will be removed from the map
mgr.deactivateNode(NodeId.newInstance("n1", 1));
+ // p1 -> n1, n1:2
+ Assert.assertEquals(2, mgr.getLabelsToNodes().get("p1").size());
assertLabelsToNodesEquals(
mgr.getLabelsToNodes(), transposeNodeToLabels(mgr.getNodeLabels()));
}
[29/50] [abbrv] hadoop git commit: HADOOP-11522. Update S3A
Documentation. (Thomas Demoor via stevel)
Posted by zj...@apache.org.
HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78a7e8d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78a7e8d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78a7e8d3
Branch: refs/heads/YARN-2928
Commit: 78a7e8d3a639a740ed8f2f6340b3921cd16ed16b
Parents: 72389c7
Author: Steve Loughran <st...@apache.org>
Authored: Tue Feb 17 18:14:31 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Feb 17 18:15:02 2015 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
.../src/main/resources/core-default.xml | 71 +++++++++++++++++-
.../src/site/markdown/tools/hadoop-aws/index.md | 77 +++++++++++++++++++-
3 files changed, 146 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78a7e8d3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1256ae5..6e43872 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -610,6 +610,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11600. Fix up source codes to be compiled with Guava 17.0. (ozawa)
+ HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78a7e8d3/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index 7135ab8..5eb301a 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -690,12 +690,12 @@ for ldap providers in the same way as above does.
</property>
<property>
- <name>fs.s3a.access.key</name>
+ <name>fs.s3a.awsAccessKeyId</name>
<description>AWS access key ID. Omit for Role-based authentication.</description>
</property>
<property>
- <name>fs.s3a.secret.key</name>
+ <name>fs.s3a.awsSecretAccessKey</name>
<description>AWS secret key. Omit for Role-based authentication.</description>
</property>
@@ -712,6 +712,46 @@ for ldap providers in the same way as above does.
</property>
<property>
+ <name>fs.s3a.endpoint</name>
+ <description>AWS S3 endpoint to connect to. An up-to-date list is
+ provided in the AWS Documentation: regions and endpoints. Without this
+ property, the standard region (s3.amazonaws.com) is assumed.
+ </description>
+</property>
+
+<property>
+ <name>fs.s3a.proxy.host</name>
+ <description>Hostname of the (optional) proxy server for S3 connections.</description>
+</property>
+
+<property>
+ <name>fs.s3a.proxy.port</name>
+ <description>Proxy server port. If this property is not set
+ but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
+ the value of fs.s3a.connection.ssl.enabled).</description>
+</property>
+
+<property>
+ <name>fs.s3a.proxy.username</name>
+ <description>Username for authenticating with proxy server.</description>
+</property>
+
+<property>
+ <name>fs.s3a.proxy.password</name>
+ <description>Password for authenticating with proxy server.</description>
+</property>
+
+<property>
+ <name>fs.s3a.proxy.domain</name>
+ <description>Domain for authenticating with proxy server.</description>
+</property>
+
+<property>
+ <name>fs.s3a.proxy.workstation</name>
+ <description>Workstation for authenticating with proxy server.</description>
+</property>
+
+<property>
<name>fs.s3a.attempts.maximum</name>
<value>10</value>
<description>How many times we should retry commands on transient errors.</description>
@@ -731,6 +771,33 @@ for ldap providers in the same way as above does.
</property>
<property>
+ <name>fs.s3a.threads.max</name>
+ <value>256</value>
+ <description> Maximum number of concurrent active (part)uploads,
+ which each use a thread from the threadpool.</description>
+</property>
+
+<property>
+ <name>fs.s3a.threads.core</name>
+ <value>15</value>
+ <description>Number of core threads in the threadpool.</description>
+</property>
+
+<property>
+ <name>fs.s3a.threads.keepalivetime</name>
+ <value>60</value>
+ <description>Number of seconds a thread can be idle before being
+ terminated.</description>
+</property>
+
+<property>
+ <name>fs.s3a.max.total.tasks</name>
+ <value>1000</value>
+ <description>Number of (part)uploads allowed to the queue before
+ blocking additional uploads.</description>
+</property>
+
+<property>
<name>fs.s3a.multipart.size</name>
<value>104857600</value>
<description>How big (in bytes) to split upload or copy operations up into.</description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/78a7e8d3/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
index 375f82c..8f511dd 100644
--- a/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
+++ b/hadoop-tools/hadoop-aws/src/site/markdown/tools/hadoop-aws/index.md
@@ -32,7 +32,7 @@ The specifics of using these filesystems are documented below.
## Warning: Object Stores are not filesystems.
-Amazon S3 is an example of "an object store". In order to achieve scalalablity
+Amazon S3 is an example of "an object store". In order to achieve scalability
and especially high availability, S3 has —as many other cloud object stores have
done— relaxed some of the constraints which classic "POSIX" filesystems promise.
@@ -165,6 +165,46 @@ If you do any of these: change your credentials immediately!
</property>
<property>
+ <name>fs.s3a.endpoint</name>
+ <description>AWS S3 endpoint to connect to. An up-to-date list is
+ provided in the AWS Documentation: regions and endpoints. Without this
+ property, the standard region (s3.amazonaws.com) is assumed.
+ </description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.host</name>
+ <description>Hostname of the (optional) proxy server for S3 connections.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.port</name>
+ <description>Proxy server port. If this property is not set
+ but fs.s3a.proxy.host is, port 80 or 443 is assumed (consistent with
+ the value of fs.s3a.connection.ssl.enabled).</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.username</name>
+ <description>Username for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.password</name>
+ <description>Password for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.domain</name>
+ <description>Domain for authenticating with proxy server.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.proxy.workstation</name>
+ <description>Workstation for authenticating with proxy server.</description>
+ </property>
+
+ <property>
<name>fs.s3a.attempts.maximum</name>
<value>10</value>
<description>How many times we should retry commands on transient errors.</description>
@@ -184,6 +224,33 @@ If you do any of these: change your credentials immediately!
</property>
<property>
+ <name>fs.s3a.threads.max</name>
+ <value>256</value>
+ <description> Maximum number of concurrent active (part)uploads,
+ which each use a thread from the threadpool.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.threads.core</name>
+ <value>15</value>
+ <description>Number of core threads in the threadpool.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.threads.keepalivetime</name>
+ <value>60</value>
+ <description>Number of seconds a thread can be idle before being
+ terminated.</description>
+ </property>
+
+ <property>
+ <name>fs.s3a.max.total.tasks</name>
+ <value>1000</value>
+ <description>Number of (part)uploads allowed to the queue before
+ blocking additional uploads.</description>
+ </property>
+
+ <property>
<name>fs.s3a.multipart.size</name>
<value>104857600</value>
<description>How big (in bytes) to split upload or copy operations up into.</description>
@@ -231,6 +298,9 @@ If you do any of these: change your credentials immediately!
## Testing the S3 filesystem clients
+Due to eventual consistency, tests may fail without reason. Transient
+failures, which no longer occur upon rerunning the test, should thus be ignored.
+
To test the S3* filesystem clients, you need to provide two files
which pass in authentication details to the test runner
@@ -256,7 +326,10 @@ each filesystem for its testing.
2. `test.fs.s3.name` : the URL of the bucket for "S3" tests
The contents of each bucket will be destroyed during the test process:
-do not use the bucket for any purpose other than testing.
+do not use the bucket for any purpose other than testing. Furthermore, for
+s3a, all in-progress multi-part uploads to the bucket will be aborted at the
+start of a test (by forcing fs.s3a.multipart.purge=true) to clean up the
+temporary state of previously failed tests.
Example:
[46/50] [abbrv] hadoop git commit: HADOOP-11545.
ArrayIndexOutOfBoundsException is thrown with "hadoop credential list
-provider". Contributed by Brahma Reddy Battula.
Posted by zj...@apache.org.
HADOOP-11545. ArrayIndexOutOfBoundsException is thrown with "hadoop credential list -provider". Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/17146099
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/17146099
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/17146099
Branch: refs/heads/YARN-2928
Commit: 17146099197000d85b3aedc84a672111f2c7908a
Parents: 2ecea5a
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Feb 18 11:17:10 2015 -0800
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Feb 18 11:17:10 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../hadoop/security/alias/CredentialShell.java | 19 ++++++++++++++++---
.../hadoop/security/alias/TestCredShell.java | 15 +++++++++++++++
3 files changed, 34 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17146099/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index f248555..92bd48d 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -962,6 +962,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11599. Client#getTimeout should use IPC_CLIENT_PING_DEFAULT when
IPC_CLIENT_PING_KEY is not configured. (zhihai xu via ozawa)
+ HADOOP-11545. ArrayIndexOutOfBoundsException is thrown with "hadoop
+ credential list -provider". (Brahma Reddy Battula via aajisaka)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17146099/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
index f397403..e8a721f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/alias/CredentialShell.java
@@ -97,6 +97,10 @@ public class CredentialShell extends Configured implements Tool {
for (int i = 0; i < args.length; i++) { // parse command line
if (args[i].equals("create")) {
+ if (i == args.length - 1) {
+ printCredShellUsage();
+ return 1;
+ }
String alias = args[++i];
command = new CreateCommand(alias);
if (alias.equals("-help")) {
@@ -104,6 +108,10 @@ public class CredentialShell extends Configured implements Tool {
return 0;
}
} else if (args[i].equals("delete")) {
+ if (i == args.length - 1) {
+ printCredShellUsage();
+ return 1;
+ }
String alias = args[++i];
command = new DeleteCommand(alias);
if (alias.equals("-help")) {
@@ -113,6 +121,10 @@ public class CredentialShell extends Configured implements Tool {
} else if (args[i].equals("list")) {
command = new ListCommand();
} else if (args[i].equals("-provider")) {
+ if (i == args.length - 1) {
+ printCredShellUsage();
+ return 1;
+ }
userSuppliedProvider = true;
getConf().set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH,
args[++i]);
@@ -195,7 +207,7 @@ public class CredentialShell extends Configured implements Tool {
}
private class ListCommand extends Command {
- public static final String USAGE = "list [-provider] [-help]";
+ public static final String USAGE = "list [-provider provider-path]";
public static final String DESC =
"The list subcommand displays the aliases contained within \n" +
"a particular provider - as configured in core-site.xml or " +
@@ -237,7 +249,7 @@ public class CredentialShell extends Configured implements Tool {
private class DeleteCommand extends Command {
public static final String USAGE =
- "delete <alias> [-provider] [-f] [-help]";
+ "delete <alias> [-f] [-provider provider-path]";
public static final String DESC =
"The delete subcommand deletes the credential\n" +
"specified as the <alias> argument from within the provider\n" +
@@ -308,7 +320,8 @@ public class CredentialShell extends Configured implements Tool {
}
private class CreateCommand extends Command {
- public static final String USAGE = "create <alias> [-provider] [-help]";
+ public static final String USAGE =
+ "create <alias> [-provider provider-path]";
public static final String DESC =
"The create subcommand creates a new credential for the name specified\n" +
"as the <alias> argument within the provider indicated through\n" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/17146099/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
index 7ba4bc1..7551df6 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/alias/TestCredShell.java
@@ -212,4 +212,19 @@ public class TestCredShell {
0, shell.init(new String[] {cmd, "-help"}));
}
}
+
+ @Test
+ public void testEmptyArgForCommands() throws Exception {
+ CredentialShell shell = new CredentialShell();
+ String[] command = { "list", "-provider" };
+ assertEquals("Expected empty argument on " + command + " to return 1", 1,
+ shell.init(command));
+
+ for (String cmd : Arrays.asList("create", "delete")) {
+ shell.setConf(new Configuration());
+ assertEquals("Expected empty argument on " + cmd + " to return 1", 1,
+ shell.init(new String[] { cmd }));
+ }
+
+ }
}
[20/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/markdown/HistoryServerRest.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/markdown/HistoryServerRest.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/markdown/HistoryServerRest.md
new file mode 100644
index 0000000..8a78754
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/markdown/HistoryServerRest.md
@@ -0,0 +1,2361 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+MapReduce History Server REST API's.
+====================================
+
+* [MapReduce History Server REST API's.](#MapReduce_History_Server_REST_APIs.)
+ * [Overview](#Overview)
+ * [History Server Information API](#History_Server_Information_API)
+ * [URI](#URI)
+ * [HTTP Operations Supported](#HTTP_Operations_Supported)
+ * [Query Parameters Supported](#Query_Parameters_Supported)
+ * [Elements of the historyInfo object](#Elements_of_the_historyInfo_object)
+ * [Response Examples](#Response_Examples)
+ * [MapReduce API's](#MapReduce_APIs)
+ * [Jobs API](#Jobs_API)
+ * [Job API](#Job_API)
+ * [Elements of the acls object](#Elements_of_the_acls_object)
+ * [Job Attempts API](#Job_Attempts_API)
+ * [Job Counters API](#Job_Counters_API)
+ * [Job Conf API](#Job_Conf_API)
+ * [Tasks API](#Tasks_API)
+ * [Task API](#Task_API)
+ * [Task Counters API](#Task_Counters_API)
+ * [Task Attempts API](#Task_Attempts_API)
+ * [Task Attempt API](#Task_Attempt_API)
+ * [Task Attempt Counters API](#Task_Attempt_Counters_API)
+
+Overview
+--------
+
+The history server REST API's allow the user to get status on finished applications.
+
+History Server Information API
+------------------------------
+
+The history server information resource provides overall information about the history server.
+
+### URI
+
+Both of the following URI's give you the history server information, from an application id identified by the appid value.
+
+ * http://<history server http address:port>/ws/v1/history
+ * http://<history server http address:port>/ws/v1/history/info
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *historyInfo* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| startedOn | long | The time the history server was started (in ms since epoch) |
+| hadoopVersion | string | Version of hadoop common |
+| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum |
+| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/info
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "historyInfo" : {
+ "startedOn":1353512830963,
+ "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012",
+ "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7",
+ "hadoopVersion" : "0.23.1-SNAPSHOT"
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/info
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 330
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <historyInfo>
+ <startedOn>1353512830963</startedOn>
+ <hadoopVersion>0.23.1-SNAPSHOT</hadoopVersion>
+ <hadoopBuildVersion>0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7</hadoopBuildVersion>
+ <hadoopVersionBuiltOn>Wed Jan 11 21:18:36 UTC 2012</hadoopVersionBuiltOn>
+ </historyInfo>
+
+MapReduce API's
+---------------
+
+The following list of resources apply to MapReduce.
+
+### Jobs API
+
+The jobs resource provides a list of the MapReduce jobs that have finished. It does not currently return a full list of parameters
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all jobs that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity.
+
+ * user - user name
+ * state - the job state
+ * queue - queue name
+ * limit - total number of app objects to be returned
+ * startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch
+ * startedTimeEnd - jobs with start time ending with this time, specified in ms since epoch
+ * finishedTimeBegin - jobs with finish time beginning with this time, specified in ms since epoch
+ * finishedTimeEnd - jobs with finish time ending with this time, specified in ms since epoch
+
+#### Elements of the *jobs* object
+
+When you make a request for the list of jobs, the information will be returned as an array of job objects. See also
+[Job API](#Job_API)
+for syntax of the job object. Except this is a subset of a full job. Only startTime, finishTime, id, name, queue, user, state, mapsTotal, mapsCompleted, reducesTotal, and reducesCompleted are returned.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| job | array of job objects(json)/zero or more job objects(XML) | The collection of job objects |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobs" : {
+ "job" : [
+ {
+ "submitTime" : 1326381344449,
+ "state" : "SUCCEEDED",
+ "user" : "user1",
+ "reducesTotal" : 1,
+ "mapsCompleted" : 1,
+ "startTime" : 1326381344489,
+ "id" : "job_1326381300833_1_1",
+ "name" : "word count",
+ "reducesCompleted" : 1,
+ "mapsTotal" : 1,
+ "queue" : "default",
+ "finishTime" : 1326381356010
+ },
+ {
+ "submitTime" : 1326381446500
+ "state" : "SUCCEEDED",
+ "user" : "user1",
+ "reducesTotal" : 1,
+ "mapsCompleted" : 1,
+ "startTime" : 1326381446529,
+ "id" : "job_1326381300833_2_2",
+ "name" : "Sleep job",
+ "reducesCompleted" : 1,
+ "mapsTotal" : 1,
+ "queue" : "default",
+ "finishTime" : 1326381582106
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 1922
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobs>
+ <job>
+ <submitTime>1326381344449</submitTime>
+ <startTime>1326381344489</startTime>
+ <finishTime>1326381356010</finishTime>
+ <id>job_1326381300833_1_1</id>
+ <name>word count</name>
+ <queue>default</queue>
+ <user>user1</user>
+ <state>SUCCEEDED</state>
+ <mapsTotal>1</mapsTotal>
+ <mapsCompleted>1</mapsCompleted>
+ <reducesTotal>1</reducesTotal>
+ <reducesCompleted>1</reducesCompleted>
+ </job>
+ <job>
+ <submitTime>1326381446500</submitTime>
+ <startTime>1326381446529</startTime>
+ <finishTime>1326381582106</finishTime>
+ <id>job_1326381300833_2_2</id>
+ <name>Sleep job</name>
+ <queue>default</queue>
+ <user>user1</user>
+ <state>SUCCEEDED</state>
+ <mapsTotal>1</mapsTotal>
+ <mapsCompleted>1</mapsCompleted>
+ <reducesTotal>1</reducesTotal>
+ <reducesCompleted>1</reducesCompleted>
+ </job>
+ </jobs>
+
+### Job API
+
+A Job resource contains information about a particular job identified by jobid.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *job* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The job id |
+| name | string | The job name |
+| queue | string | The queue the job was submitted to |
+| user | string | The user name |
+| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL\_WAIT, KILLED, ERROR |
+| diagnostics | string | A diagnostic message |
+| submitTime | long | The time the job submitted (in ms since epoch) |
+| startTime | long | The time the job started (in ms since epoch) |
+| finishTime | long | The time the job finished (in ms since epoch) |
+| mapsTotal | int | The total number of maps |
+| mapsCompleted | int | The number of completed maps |
+| reducesTotal | int | The total number of reduces |
+| reducesCompleted | int | The number of completed reduces |
+| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master |
+| avgMapTime | long | The average time of a map task (in ms) |
+| avgReduceTime | long | The average time of the reduce (in ms) |
+| avgShuffleTime | long | The average time of the shuffle (in ms) |
+| avgMergeTime | long | The average time of the merge (in ms) |
+| failedReduceAttempts | int | The number of failed reduce attempts |
+| killedReduceAttempts | int | The number of killed reduce attempts |
+| successfulReduceAttempts | int | The number of successful reduce attempts |
+| failedMapAttempts | int | The number of failed map attempts |
+| killedMapAttempts | int | The number of killed map attempts |
+| successfulMapAttempts | int | The number of successful map attempts |
+| acls | array of acls(json)/zero or more acls objects(xml) | A collection of acls objects |
+
+### Elements of the *acls* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| value | string | The acl value |
+| name | string | The acl name |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Server: Jetty(6.1.26)
+ Content-Length: 720
+
+Response Body:
+
+ {
+ "job" : {
+ "submitTime": 1326381446500,
+ "avgReduceTime" : 124961,
+ "failedReduceAttempts" : 0,
+ "state" : "SUCCEEDED",
+ "successfulReduceAttempts" : 1,
+ "acls" : [
+ {
+ "value" : " ",
+ "name" : "mapreduce.job.acl-modify-job"
+ },
+ {
+ "value" : " ",
+ "name" : "mapreduce.job.acl-view-job"
+ }
+ ],
+ "user" : "user1",
+ "reducesTotal" : 1,
+ "mapsCompleted" : 1,
+ "startTime" : 1326381446529,
+ "id" : "job_1326381300833_2_2",
+ "avgMapTime" : 2638,
+ "successfulMapAttempts" : 1,
+ "name" : "Sleep job",
+ "avgShuffleTime" : 2540,
+ "reducesCompleted" : 1,
+ "diagnostics" : "",
+ "failedMapAttempts" : 0,
+ "avgMergeTime" : 2589,
+ "killedReduceAttempts" : 0,
+ "mapsTotal" : 1,
+ "queue" : "default",
+ "uberized" : false,
+ "killedMapAttempts" : 0,
+ "finishTime" : 1326381582106
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 983
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <job>
+ <submitTime>1326381446500</submitTime>
+ <startTime>1326381446529</startTime>
+ <finishTime>1326381582106</finishTime>
+ <id>job_1326381300833_2_2</id>
+ <name>Sleep job</name>
+ <queue>default</queue>
+ <user>user1</user>
+ <state>SUCCEEDED</state>
+ <mapsTotal>1</mapsTotal>
+ <mapsCompleted>1</mapsCompleted>
+ <reducesTotal>1</reducesTotal>
+ <reducesCompleted>1</reducesCompleted>
+ <uberized>false</uberized>
+ <diagnostics/>
+ <avgMapTime>2638</avgMapTime>
+ <avgReduceTime>124961</avgReduceTime>
+ <avgShuffleTime>2540</avgShuffleTime>
+ <avgMergeTime>2589</avgMergeTime>
+ <failedReduceAttempts>0</failedReduceAttempts>
+ <killedReduceAttempts>0</killedReduceAttempts>
+ <successfulReduceAttempts>1</successfulReduceAttempts>
+ <failedMapAttempts>0</failedMapAttempts>
+ <killedMapAttempts>0</killedMapAttempts>
+ <successfulMapAttempts>1</successfulMapAttempts>
+ <acls>
+ <name>mapreduce.job.acl-modify-job</name>
+ <value> </value>
+ </acls>
+ <acls>
+ <name>mapreduce.job.acl-view-job</name>
+ <value> </value>
+ </acls>
+ </job>
+
+### Job Attempts API
+
+With the job attempts API, you can obtain a collection of resources that represent a job attempt. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/jobattempts
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *jobAttempts* object
+
+When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects.
+
+jobAttempts:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects |
+
+#### Elements of the *jobAttempt* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The job attempt id |
+| nodeId | string | The node id of the node the attempt ran on |
+| nodeHttpAddress | string | The node http address of the node the attempt ran on |
+| logsLink | string | The http link to the job attempt logs |
+| containerId | string | The id of the container for the job attempt |
+| startTime | long | The start time of the attempt (in ms since epoch) |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattempts
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobAttempts" : {
+ "jobAttempt" : [
+ {
+ "nodeId" : "host.domain.com:8041",
+ "nodeHttpAddress" : "host.domain.com:8042",
+ "startTime" : 1326381444693,
+ "id" : 1,
+ "logsLink" : "http://host.domain.com:19888/jobhistory/logs/host.domain.com:8041/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1",
+ "containerId" : "container_1326381300833_0002_01_000001"
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattmpts
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 575
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobAttempts>
+ <jobAttempt>
+ <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
+ <nodeId>host.domain.com:8041</nodeId>
+ <id>1</id>
+ <startTime>1326381444693</startTime>
+ <containerId>container_1326381300833_0002_01_000001</containerId>
+ <logsLink>http://host.domain.com:19888/jobhistory/logs/host.domain.com:8041/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1</logsLink>
+ </jobAttempt>
+ </jobAttempts>
+
+### Job Counters API
+
+With the job counters API, you can object a collection of resources that represent al the counters for that job.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/counters
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *jobCounters* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The job id |
+| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
+
+#### Elements of the *counterGroup* objecs
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| counterGroupName | string | The name of the counter group |
+| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
+
+#### Elements of the *counter* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the counter |
+| reduceCounterValue | long | The counter value of reduce tasks |
+| mapCounterValue | long | The counter value of map tasks |
+| totalCounterValue | long | The counter value of all tasks |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobCounters" : {
+ "id" : "job_1326381300833_2_2",
+ "counterGroup" : [
+ {
+ "counterGroupName" : "Shuffle Errors",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "BAD_ID"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "CONNECTION"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "IO_ERROR"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "WRONG_LENGTH"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "WRONG_MAP"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "WRONG_REDUCE"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2483,
+ "name" : "FILE_BYTES_READ"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 108525,
+ "name" : "FILE_BYTES_WRITTEN"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FILE_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FILE_LARGE_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FILE_WRITE_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 48,
+ "name" : "HDFS_BYTES_READ"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "HDFS_BYTES_WRITTEN"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "HDFS_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "HDFS_LARGE_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "HDFS_WRITE_OPS"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "MAP_INPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1200,
+ "name" : "MAP_OUTPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 4800,
+ "name" : "MAP_OUTPUT_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2235,
+ "name" : "MAP_OUTPUT_MATERIALIZED_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 48,
+ "name" : "SPLIT_RAW_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "COMBINE_INPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "COMBINE_OUTPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1200,
+ "name" : "REDUCE_INPUT_GROUPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2235,
+ "name" : "REDUCE_SHUFFLE_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1200,
+ "name" : "REDUCE_INPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "REDUCE_OUTPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2400,
+ "name" : "SPILLED_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "SHUFFLED_MAPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FAILED_SHUFFLE"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "MERGED_MAP_OUTPUTS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 113,
+ "name" : "GC_TIME_MILLIS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1830,
+ "name" : "CPU_MILLISECONDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 478068736,
+ "name" : "PHYSICAL_MEMORY_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2159284224,
+ "name" : "VIRTUAL_MEMORY_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 378863616,
+ "name" : "COMMITTED_HEAP_BYTES"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "BYTES_READ"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "BYTES_WRITTEN"
+ }
+ ]
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 7030
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobCounters>
+ <id>job_1326381300833_2_2</id>
+ <counterGroup>
+ <counterGroupName>Shuffle Errors</counterGroupName>
+ <counter>
+ <name>BAD_ID</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>CONNECTION</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>IO_ERROR</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>WRONG_LENGTH</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>WRONG_MAP</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>WRONG_REDUCE</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
+ <counter>
+ <name>FILE_BYTES_READ</name>
+ <totalCounterValue>2483</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_BYTES_WRITTEN</name>
+ <totalCounterValue>108525</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_READ_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_LARGE_READ_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_WRITE_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_READ</name>
+ <totalCounterValue>48</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_WRITTEN</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_READ_OPS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_LARGE_READ_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_WRITE_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
+ <counter>
+ <name>MAP_INPUT_RECORDS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MAP_OUTPUT_RECORDS</name>
+ <totalCounterValue>1200</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MAP_OUTPUT_BYTES</name>
+ <totalCounterValue>4800</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MAP_OUTPUT_MATERIALIZED_BYTES</name>
+ <totalCounterValue>2235</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>SPLIT_RAW_BYTES</name>
+ <totalCounterValue>48</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>COMBINE_INPUT_RECORDS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>COMBINE_OUTPUT_RECORDS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_GROUPS</name>
+ <totalCounterValue>1200</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_SHUFFLE_BYTES</name>
+ <totalCounterValue>2235</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_RECORDS</name>
+ <totalCounterValue>1200</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_OUTPUT_RECORDS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>SPILLED_RECORDS</name>
+ <totalCounterValue>2400</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>SHUFFLED_MAPS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FAILED_SHUFFLE</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MERGED_MAP_OUTPUTS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>GC_TIME_MILLIS</name>
+ <totalCounterValue>113</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>CPU_MILLISECONDS</name>
+ <totalCounterValue>1830</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>PHYSICAL_MEMORY_BYTES</name>
+ <totalCounterValue>478068736</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>VIRTUAL_MEMORY_BYTES</name>
+ <totalCounterValue>2159284224</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>COMMITTED_HEAP_BYTES</name>
+ <totalCounterValue>378863616</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_READ</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_WRITTEN</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ </jobCounters>
+
+### Job Conf API
+
+A job configuration resource contains information about the job configuration for this job.
+
+#### URI
+
+Use the following URI to obtain th job configuration information, from a job identified by the jobid value.
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/conf
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *conf* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| path | string | The path to the job configuration file |
+| property | array of the configuration properties(JSON)/zero or more configuration properties(XML) | Collection of configuration property objects |
+
+#### Elements of the *property* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the configuration property |
+| value | string | The value of the configuration property |
+| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file.
+
+ {
+ "conf" : {
+ "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml",
+ "property" : [
+ {
+ "value" : "/home/hadoop/hdfs/data",
+ "name" : "dfs.datanode.data.dir"
+ "source" : ["hdfs-site.xml", "job.xml"]
+ },
+ {
+ "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
+ "name" : "hadoop.http.filter.initializers"
+ "source" : ["programmatically", "job.xml"]
+ },
+ {
+ "value" : "/home/hadoop/tmp",
+ "name" : "mapreduce.cluster.temp.dir"
+ "source" : ["mapred-site.xml"]
+ },
+ ...
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 552
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <conf>
+ <path>hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml</path>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/home/hadoop/hdfs/data</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
+ </property>
+ <property>
+ <name>hadoop.http.filter.initializers</name>
+ <value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
+ <source>programmatically</source>
+ <source>job.xml</source>
+ </property>
+ <property>
+ <name>mapreduce.cluster.temp.dir</name>
+ <value>/home/hadoop/tmp</value>
+ <source>mapred-site.xml</source>
+ </property>
+ ...
+ </conf>
+
+### Tasks API
+
+With the tasks API, you can obtain a collection of resources that represent a task within a job. When you run a GET operation on this resource, you obtain a collection of Task Objects.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ * type - type of task, valid values are m or r. m for map task or r for reduce task.
+
+#### Elements of the *tasks* object
+
+When you make a request for the list of tasks , the information will be returned as an array of task objects. See also
+[Task API](#Task_API)
+for syntax of the task object.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects. |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "tasks" : {
+ "task" : [
+ {
+ "progress" : 100,
+ "elapsedTime" : 6777,
+ "state" : "SUCCEEDED",
+ "startTime" : 1326381446541,
+ "id" : "task_1326381300833_2_2_m_0",
+ "type" : "MAP",
+ "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0",
+ "finishTime" : 1326381453318
+ },
+ {
+ "progress" : 100,
+ "elapsedTime" : 135559,
+ "state" : "SUCCEEDED",
+ "startTime" : 1326381446544,
+ "id" : "task_1326381300833_2_2_r_0",
+ "type" : "REDUCE",
+ "successfulAttempt" : "attempt_1326381300833_2_2_r_0_0",
+ "finishTime" : 1326381582103
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 653
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <tasks>
+ <task>
+ <startTime>1326381446541</startTime>
+ <finishTime>1326381453318</finishTime>
+ <elapsedTime>6777</elapsedTime>
+ <progress>100.0</progress>
+ <id>task_1326381300833_2_2_m_0</id>
+ <state>SUCCEEDED</state>
+ <type>MAP</type>
+ <successfulAttempt>attempt_1326381300833_2_2_m_0_0</successfulAttempt>
+ </task>
+ <task>
+ <startTime>1326381446544</startTime>
+ <finishTime>1326381582103</finishTime>
+ <elapsedTime>135559</elapsedTime>
+ <progress>100.0</progress>
+ <id>task_1326381300833_2_2_r_0</id>
+ <state>SUCCEEDED</state>
+ <type>REDUCE</type>
+ <successfulAttempt>attempt_1326381300833_2_2_r_0_0</successfulAttempt>
+ </task>
+ </tasks>
+
+### Task API
+
+A Task resource contains information about a particular task within a job.
+
+#### URI
+
+Use the following URI to obtain an Task Object, from a task identified by the taskid value.
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *task* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task id |
+| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL\_WAIT, KILLED |
+| type | string | The task type - MAP or REDUCE |
+| successfulAttempt | string | The id of the last successful attempt |
+| progress | float | The progress of the task as a percent |
+| startTime | long | The time in which the task started (in ms since epoch) or -1 if it was never started |
+| finishTime | long | The time in which the task finished (in ms since epoch) |
+| elapsedTime | long | The elapsed time since the application started (in ms) |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "task" : {
+ "progress" : 100,
+ "elapsedTime" : 6777,
+ "state" : "SUCCEEDED",
+ "startTime" : 1326381446541,
+ "id" : "task_1326381300833_2_2_m_0",
+ "type" : "MAP",
+ "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0",
+ "finishTime" : 1326381453318
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 299
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <task>
+ <startTime>1326381446541</startTime>
+ <finishTime>1326381453318</finishTime>
+ <elapsedTime>6777</elapsedTime>
+ <progress>100.0</progress>
+ <id>task_1326381300833_2_2_m_0</id>
+ <state>SUCCEEDED</state>
+ <type>MAP</type>
+ <successfulAttempt>attempt_1326381300833_2_2_m_0_0</successfulAttempt>
+ </task>
+
+### Task Counters API
+
+With the task counters API, you can object a collection of resources that represent all the counters for that task.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *jobTaskCounters* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task id |
+| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
+
+#### Elements of the *counterGroup* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| counterGroupName | string | The name of the counter group |
+| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
+
+#### Elements of the *counter* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the counter |
+| value | long | The value of the counter |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobTaskCounters" : {
+ "id" : "task_1326381300833_2_2_m_0",
+ "taskCounterGroup" : [
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
+ "counter" : [
+ {
+ "value" : 2363,
+ "name" : "FILE_BYTES_READ"
+ },
+ {
+ "value" : 54372,
+ "name" : "FILE_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_WRITE_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_READ"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_WRITE_OPS"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "COMBINE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "COMBINE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_GROUPS"
+ },
+ {
+ "value" : 2235,
+ "name" : "REDUCE_SHUFFLE_BYTES"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "REDUCE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "SPILLED_RECORDS"
+ },
+ {
+ "value" : 1,
+ "name" : "SHUFFLED_MAPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FAILED_SHUFFLE"
+ },
+ {
+ "value" : 1,
+ "name" : "MERGED_MAP_OUTPUTS"
+ },
+ {
+ "value" : 26,
+ "name" : "GC_TIME_MILLIS"
+ },
+ {
+ "value" : 860,
+ "name" : "CPU_MILLISECONDS"
+ },
+ {
+ "value" : 107839488,
+ "name" : "PHYSICAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 1123147776,
+ "name" : "VIRTUAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 57475072,
+ "name" : "COMMITTED_HEAP_BYTES"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "Shuffle Errors",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BAD_ID"
+ },
+ {
+ "value" : 0,
+ "name" : "CONNECTION"
+ },
+ {
+ "value" : 0,
+ "name" : "IO_ERROR"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_LENGTH"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_MAP"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_REDUCE"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BYTES_WRITTEN"
+ }
+ ]
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 2660
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobTaskCounters>
+ <id>task_1326381300833_2_2_m_0</id>
+ <taskCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
+ <counter>
+ <name>FILE_BYTES_READ</name>
+ <value>2363</value>
+ </counter>
+ <counter>
+ <name>FILE_BYTES_WRITTEN</name>
+ <value>54372</value>
+ </counter>
+ <counter>
+ <name>FILE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_READ</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ </taskCounterGroup>
+ <taskCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
+ <counter>
+ <name>COMBINE_INPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>COMBINE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_GROUPS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_SHUFFLE_BYTES</name>
+ <value>2235</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_RECORDS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SPILLED_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SHUFFLED_MAPS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>FAILED_SHUFFLE</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>MERGED_MAP_OUTPUTS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>GC_TIME_MILLIS</name>
+ <value>26</value>
+ </counter>
+ <counter>
+ <name>CPU_MILLISECONDS</name>
+ <value>860</value>
+ </counter>
+ <counter>
+ <name>PHYSICAL_MEMORY_BYTES</name>
+ <value>107839488</value>
+ </counter>
+ <counter>
+ <name>VIRTUAL_MEMORY_BYTES</name>
+ <value>1123147776</value>
+ </counter>
+ <counter>
+ <name>COMMITTED_HEAP_BYTES</name>
+ <value>57475072</value>
+ </counter>
+ </taskCounterGroup>
+ <taskCounterGroup>
+ <counterGroupName>Shuffle Errors</counterGroupName>
+ <counter>
+ <name>BAD_ID</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>CONNECTION</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>IO_ERROR</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_LENGTH</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_MAP</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_REDUCE</name>
+ <value>0</value>
+ </counter>
+ </taskCounterGroup>
+ <taskCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ </taskCounterGroup>
+ </jobTaskCounters>
+
+### Task Attempts API
+
+With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *taskAttempts* object
+
+When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects. See also
+[Task Attempt API](#Task_Attempt_API)
+for syntax of the task object.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "taskAttempts" : {
+ "taskAttempt" : [
+ {
+ "assignedContainerId" : "container_1326381300833_0002_01_000002",
+ "progress" : 100,
+ "elapsedTime" : 2638,
+ "state" : "SUCCEEDED",
+ "diagnostics" : "",
+ "rack" : "/98.139.92.0",
+ "nodeHttpAddress" : "host.domain.com:8042",
+ "startTime" : 1326381450680,
+ "id" : "attempt_1326381300833_2_2_m_0_0",
+ "type" : "MAP",
+ "finishTime" : 1326381453318
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 537
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <taskAttempts>
+ <taskAttempt>
+ <startTime>1326381450680</startTime>
+ <finishTime>1326381453318</finishTime>
+ <elapsedTime>2638</elapsedTime>
+ <progress>100.0</progress>
+ <id>attempt_1326381300833_2_2_m_0_0</id>
+ <rack>/98.139.92.0</rack>
+ <state>SUCCEEDED</state>
+ <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
+ <diagnostics/>
+ <type>MAP</type>
+ <assignedContainerId>container_1326381300833_0002_01_000002</assignedContainerId>
+ </taskAttempt>
+ </taskAttempts>
+
+### Task Attempt API
+
+A Task Attempt resource contains information about a particular task attempt within a job.
+
+#### URI
+
+Use the following URI to obtain an Task Attempt Object, from a task identified by the attemptid value.
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *taskAttempt* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task id |
+| rack | string | The rack |
+| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT\_PENDING, SUCCESS\_CONTAINER\_CLEANUP, SUCCEEDED, FAIL\_CONTAINER\_CLEANUP, FAIL\_TASK\_CLEANUP, FAILED, KILL\_CONTAINER\_CLEANUP, KILL\_TASK\_CLEANUP, KILLED |
+| type | string | The type of task |
+| assignedContainerId | string | The container id this attempt is assigned to |
+| nodeHttpAddress | string | The http address of the node this task attempt ran on |
+| diagnostics | string | A diagnostics message |
+| progress | float | The progress of the task attempt as a percent |
+| startTime | long | The time in which the task attempt started (in ms since epoch) |
+| finishTime | long | The time in which the task attempt finished (in ms since epoch) |
+| elapsedTime | long | The elapsed time since the task attempt started (in ms) |
+
+For reduce task attempts you also have the following fields:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch) |
+| mergeFinishTime | long | The time at which merge finished (in ms since epoch) |
+| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish) |
+| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish) |
+| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task) |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "taskAttempt" : {
+ "assignedContainerId" : "container_1326381300833_0002_01_000002",
+ "progress" : 100,
+ "elapsedTime" : 2638,
+ "state" : "SUCCEEDED",
+ "diagnostics" : "",
+ "rack" : "/98.139.92.0",
+ "nodeHttpAddress" : "host.domain.com:8042",
+ "startTime" : 1326381450680,
+ "id" : "attempt_1326381300833_2_2_m_0_0",
+ "type" : "MAP",
+ "finishTime" : 1326381453318
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 691
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <taskAttempt>
+ <startTime>1326381450680</startTime>
+ <finishTime>1326381453318</finishTime>
+ <elapsedTime>2638</elapsedTime>
+ <progress>100.0</progress>
+ <id>attempt_1326381300833_2_2_m_0_0</id>
+ <rack>/98.139.92.0</rack>
+ <state>SUCCEEDED</state>
+ <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
+ <diagnostics/>
+ <type>MAP</type>
+ <assignedContainerId>container_1326381300833_0002_01_000002</assignedContainerId>
+ </taskAttempt>
+
+### Task Attempt Counters API
+
+With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt.
+
+#### URI
+
+ * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters
+
+#### HTTP Operations Supported
+
+ * GET
+
+#### Query Parameters Supported
+
+ None
+
+#### Elements of the *jobTaskAttemptCounters* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task attempt id |
+| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects |
+
+#### Elements of the *taskAttemptCounterGroup* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| counterGroupName | string | The name of the counter group |
+| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
+
+#### Elements of the *counter* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the counter |
+| value | long | The value of the counter |
+
+#### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobTaskAttemptCounters" : {
+ "taskAttemptCounterGroup" : [
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
+ "counter" : [
+ {
+ "value" : 2363,
+ "name" : "FILE_BYTES_READ"
+ },
+ {
+ "value" : 54372,
+ "name" : "FILE_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_WRITE_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_READ"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_WRITE_OPS"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "COMBINE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "COMBINE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_GROUPS"
+ },
+ {
+ "value" : 2235,
+ "name" : "REDUCE_SHUFFLE_BYTES"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "REDUCE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "SPILLED_RECORDS"
+ },
+ {
+ "value" : 1,
+ "name" : "SHUFFLED_MAPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FAILED_SHUFFLE"
+ },
+ {
+ "value" : 1,
+ "name" : "MERGED_MAP_OUTPUTS"
+ },
+ {
+ "value" : 26,
+ "name" : "GC_TIME_MILLIS"
+ },
+ {
+ "value" : 860,
+ "name" : "CPU_MILLISECONDS"
+ },
+ {
+ "value" : 107839488,
+ "name" : "PHYSICAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 1123147776,
+ "name" : "VIRTUAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 57475072,
+ "name" : "COMMITTED_HEAP_BYTES"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "Shuffle Errors",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BAD_ID"
+ },
+ {
+ "value" : 0,
+ "name" : "CONNECTION"
+ },
+ {
+ "value" : 0,
+ "name" : "IO_ERROR"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_LENGTH"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_MAP"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_REDUCE"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BYTES_WRITTEN"
+ }
+ ]
+ }
+ ],
+ "id" : "attempt_1326381300833_2_2_m_0_0"
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 2735
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobTaskAttemptCounters>
+ <id>attempt_1326381300833_2_2_m_0_0</id>
+ <taskAttemptCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
+ <counter>
+ <name>FILE_BYTES_READ</name>
+ <value>2363</value>
+ </counter>
+ <counter>
+ <name>FILE_BYTES_WRITTEN</name>
+ <value>54372</value>
+ </counter>
+ <counter>
+ <name>FILE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_READ</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ <taskAttemptCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
+ <counter>
+ <name>COMBINE_INPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>COMBINE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_GROUPS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_SHUFFLE_BYTES</name>
+ <value>2235</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_RECORDS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SPILLED_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SHUFFLED_MAPS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>FAILED_SHUFFLE</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>MERGED_MAP_OUTPUTS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>GC_TIME_MILLIS</name>
+ <value>26</value>
+ </counter>
+ <counter>
+ <name>CPU_MILLISECONDS</name>
+ <value>860</value>
+ </counter>
+ <counter>
+ <name>PHYSICAL_MEMORY_BYTES</name>
+ <value>107839488</value>
+ </counter>
+ <counter>
+ <name>VIRTUAL_MEMORY_BYTES</name>
+ <value>1123147776</value>
+ </counter>
+ <counter>
+ <name>COMMITTED_HEAP_BYTES</name>
+ <value>57475072</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ <taskAttemptCounterGroup>
+ <counterGroupName>Shuffle Errors</counterGroupName>
+ <counter>
+ <name>BAD_ID</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>CONNECTION</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>IO_ERROR</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_LENGTH</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_MAP</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_REDUCE</name>
+ <value>0</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ <taskAttemptCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ </jobTaskAttemptCounters>
[12/50] [abbrv] hadoop git commit: HDFS-7604. Track and display
failed DataNode storage locations in NameNode. Contributed by Chris Nauroth.
Posted by zj...@apache.org.
HDFS-7604. Track and display failed DataNode storage locations in NameNode. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9729b244
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9729b244
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9729b244
Branch: refs/heads/YARN-2928
Commit: 9729b244de50322c2cc889c97c2ffb2b4675cf77
Parents: 814afa4
Author: cnauroth <cn...@apache.org>
Authored: Mon Feb 16 14:43:02 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Mon Feb 16 14:43:02 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../DatanodeProtocolClientSideTranslatorPB.java | 9 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 7 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 25 ++
.../blockmanagement/DatanodeDescriptor.java | 39 +-
.../server/blockmanagement/DatanodeManager.java | 7 +-
.../blockmanagement/HeartbeatManager.java | 8 +-
.../hdfs/server/datanode/BPServiceActor.java | 12 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 8 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 108 ++++-
.../datanode/fsdataset/impl/FsVolumeList.java | 41 +-
.../fsdataset/impl/VolumeFailureInfo.java | 82 ++++
.../server/datanode/metrics/FSDatasetMBean.java | 19 +
.../hdfs/server/namenode/FSNamesystem.java | 52 ++-
.../hdfs/server/namenode/NameNodeRpcServer.java | 6 +-
.../namenode/metrics/FSNamesystemMBean.java | 13 +
.../hdfs/server/protocol/DatanodeProtocol.java | 5 +-
.../server/protocol/VolumeFailureSummary.java | 72 ++++
.../src/main/proto/DatanodeProtocol.proto | 17 +-
.../src/main/webapps/hdfs/dfshealth.html | 33 ++
.../src/main/webapps/hdfs/dfshealth.js | 43 ++
.../blockmanagement/TestBlockManager.java | 3 +-
.../TestNameNodePrunesMissingStorages.java | 4 +-
.../TestOverReplicatedBlocks.java | 2 +-
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../TestReplicationPolicyConsiderLoad.java | 8 +-
.../TestReplicationPolicyWithNodeGroup.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 22 +
.../server/datanode/TestBPOfferService.java | 4 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 4 +-
.../TestDataNodeVolumeFailureReporting.java | 421 +++++++++++++++++--
.../server/datanode/TestFsDatasetCache.java | 3 +-
.../hdfs/server/datanode/TestStorageReport.java | 4 +-
.../extdataset/ExternalDatasetImpl.java | 21 +
.../fsdataset/impl/TestFsDatasetImpl.java | 6 +-
.../fsdataset/impl/TestFsVolumeList.java | 7 +-
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
39 files changed, 1023 insertions(+), 107 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 324d5ae..cc24dc4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -634,6 +634,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7430. Refactor the BlockScanner to use O(1) memory and use multiple
threads (cmccabe)
+ HDFS-7604. Track and display failed DataNode storage locations in NameNode.
+ (cnauroth)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
index 46023ec..192916f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolClientSideTranslatorPB.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.ipc.ProtobufHelper;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.ProtocolMetaInterface;
@@ -121,8 +122,8 @@ public class DatanodeProtocolClientSideTranslatorPB implements
@Override
public HeartbeatResponse sendHeartbeat(DatanodeRegistration registration,
StorageReport[] reports, long cacheCapacity, long cacheUsed,
- int xmitsInProgress, int xceiverCount, int failedVolumes)
- throws IOException {
+ int xmitsInProgress, int xceiverCount, int failedVolumes,
+ VolumeFailureSummary volumeFailureSummary) throws IOException {
HeartbeatRequestProto.Builder builder = HeartbeatRequestProto.newBuilder()
.setRegistration(PBHelper.convert(registration))
.setXmitsInProgress(xmitsInProgress).setXceiverCount(xceiverCount)
@@ -134,6 +135,10 @@ public class DatanodeProtocolClientSideTranslatorPB implements
if (cacheUsed != 0) {
builder.setCacheUsed(cacheUsed);
}
+ if (volumeFailureSummary != null) {
+ builder.setVolumeFailureSummary(PBHelper.convertVolumeFailureSummary(
+ volumeFailureSummary));
+ }
HeartbeatResponseProto resp;
try {
resp = rpcProxy.sendHeartbeat(NULL_CONTROLLER, builder.build());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
index d016735..1a89090 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/DatanodeProtocolServerSideTranslatorPB.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
@@ -104,10 +105,14 @@ public class DatanodeProtocolServerSideTranslatorPB implements
try {
final StorageReport[] report = PBHelper.convertStorageReports(
request.getReportsList());
+ VolumeFailureSummary volumeFailureSummary =
+ request.hasVolumeFailureSummary() ? PBHelper.convertVolumeFailureSummary(
+ request.getVolumeFailureSummary()) : null;
response = impl.sendHeartbeat(PBHelper.convert(request.getRegistration()),
report, request.getCacheCapacity(), request.getCacheUsed(),
request.getXmitsInProgress(),
- request.getXceiverCount(), request.getFailedVolumes());
+ request.getXceiverCount(), request.getFailedVolumes(),
+ volumeFailureSummary);
} catch (IOException e) {
throw new ServiceException(e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
index 8ecd767..76db47b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelper.java
@@ -122,6 +122,7 @@ import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.KeyUpdateCom
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.NNHAStatusHeartbeatProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.ReceivedDeletedBlockInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.RegisterCommandProto;
+import org.apache.hadoop.hdfs.protocol.proto.DatanodeProtocolProtos.VolumeFailureSummaryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockKeyProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
@@ -215,6 +216,7 @@ import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLog;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.ShmId;
import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitShm.SlotId;
import org.apache.hadoop.hdfs.util.ExactSizeInputStream;
@@ -1901,6 +1903,29 @@ public class PBHelper {
return protos;
}
+ public static VolumeFailureSummary convertVolumeFailureSummary(
+ VolumeFailureSummaryProto proto) {
+ List<String> failedStorageLocations = proto.getFailedStorageLocationsList();
+ return new VolumeFailureSummary(
+ failedStorageLocations.toArray(new String[failedStorageLocations.size()]),
+ proto.getLastVolumeFailureDate(), proto.getEstimatedCapacityLostTotal());
+ }
+
+ public static VolumeFailureSummaryProto convertVolumeFailureSummary(
+ VolumeFailureSummary volumeFailureSummary) {
+ VolumeFailureSummaryProto.Builder builder =
+ VolumeFailureSummaryProto.newBuilder();
+ for (String failedStorageLocation:
+ volumeFailureSummary.getFailedStorageLocations()) {
+ builder.addFailedStorageLocations(failedStorageLocation);
+ }
+ builder.setLastVolumeFailureDate(
+ volumeFailureSummary.getLastVolumeFailureDate());
+ builder.setEstimatedCapacityLostTotal(
+ volumeFailureSummary.getEstimatedCapacityLostTotal());
+ return builder.build();
+ }
+
public static JournalInfo convert(JournalInfoProto info) {
int lv = info.hasLayoutVersion() ? info.getLayoutVersion() : 0;
int nsID = info.hasNamespaceID() ? info.getNamespaceID() : 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 833ab56..b898013 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.server.namenode.CachedBlock;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.LightWeightHashSet;
import org.apache.hadoop.util.IntrusiveCollection;
@@ -214,6 +215,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
private long lastBlocksScheduledRollTime = 0;
private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min
private int volumeFailures = 0;
+ private VolumeFailureSummary volumeFailureSummary = null;
/**
* When set to true, the node is not in include list and is not allowed
@@ -233,7 +235,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
*/
public DatanodeDescriptor(DatanodeID nodeID) {
super(nodeID);
- updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0);
+ updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null);
}
/**
@@ -244,7 +246,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
public DatanodeDescriptor(DatanodeID nodeID,
String networkLocation) {
super(nodeID, networkLocation);
- updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0);
+ updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null);
}
@VisibleForTesting
@@ -345,9 +347,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
* Updates stats from datanode heartbeat.
*/
public void updateHeartbeat(StorageReport[] reports, long cacheCapacity,
- long cacheUsed, int xceiverCount, int volFailures) {
+ long cacheUsed, int xceiverCount, int volFailures,
+ VolumeFailureSummary volumeFailureSummary) {
updateHeartbeatState(reports, cacheCapacity, cacheUsed, xceiverCount,
- volFailures);
+ volFailures, volumeFailureSummary);
heartbeatedSinceRegistration = true;
}
@@ -355,7 +358,8 @@ public class DatanodeDescriptor extends DatanodeInfo {
* process datanode heartbeat or stats initialization.
*/
public void updateHeartbeatState(StorageReport[] reports, long cacheCapacity,
- long cacheUsed, int xceiverCount, int volFailures) {
+ long cacheUsed, int xceiverCount, int volFailures,
+ VolumeFailureSummary volumeFailureSummary) {
long totalCapacity = 0;
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
@@ -370,7 +374,10 @@ public class DatanodeDescriptor extends DatanodeInfo {
// during the current DN registration session.
// When volumeFailures == this.volumeFailures, it implies there is no
// state change. No need to check for failed storage. This is an
- // optimization.
+ // optimization. Recent versions of the DataNode report a
+ // VolumeFailureSummary containing the date/time of the last volume
+ // failure. If that's available, then we check that instead for greater
+ // accuracy.
// 2. After DN restarts, volFailures might not increase and it is possible
// we still have new failed storage. For example, admins reduce
// available storages in configuration. Another corner case
@@ -379,8 +386,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
// one element in storageReports and that is A. b) A failed. c) Before
// DN sends HB to NN to indicate A has failed, DN restarts. d) After DN
// restarts, storageReports has one element which is B.
- boolean checkFailedStorages = (volFailures > this.volumeFailures) ||
- !heartbeatedSinceRegistration;
+ final boolean checkFailedStorages;
+ if (volumeFailureSummary != null && this.volumeFailureSummary != null) {
+ checkFailedStorages = volumeFailureSummary.getLastVolumeFailureDate() >
+ this.volumeFailureSummary.getLastVolumeFailureDate();
+ } else {
+ checkFailedStorages = (volFailures > this.volumeFailures) ||
+ !heartbeatedSinceRegistration;
+ }
if (checkFailedStorages) {
LOG.info("Number of failed storage changes from "
@@ -394,6 +407,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
setXceiverCount(xceiverCount);
setLastUpdate(Time.now());
this.volumeFailures = volFailures;
+ this.volumeFailureSummary = volumeFailureSummary;
for (StorageReport report : reports) {
DatanodeStorageInfo storage = updateStorage(report.getStorage());
if (checkFailedStorages) {
@@ -729,6 +743,15 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
/**
+ * Returns info about volume failures.
+ *
+ * @return info about volume failures, possibly null
+ */
+ public VolumeFailureSummary getVolumeFailureSummary() {
+ return volumeFailureSummary;
+ }
+
+ /**
* @param nodeReg DatanodeID to update registration for.
*/
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index f5fe161..45c56a8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1387,8 +1387,8 @@ public class DatanodeManager {
public DatanodeCommand[] handleHeartbeat(DatanodeRegistration nodeReg,
StorageReport[] reports, final String blockPoolId,
long cacheCapacity, long cacheUsed, int xceiverCount,
- int maxTransfers, int failedVolumes
- ) throws IOException {
+ int maxTransfers, int failedVolumes,
+ VolumeFailureSummary volumeFailureSummary) throws IOException {
synchronized (heartbeatManager) {
synchronized (datanodeMap) {
DatanodeDescriptor nodeinfo = null;
@@ -1410,7 +1410,8 @@ public class DatanodeManager {
heartbeatManager.updateHeartbeat(nodeinfo, reports,
cacheCapacity, cacheUsed,
- xceiverCount, failedVolumes);
+ xceiverCount, failedVolumes,
+ volumeFailureSummary);
// If we are in safemode, do not send back any recovery / replication
// requests. Don't even drain the existing queue of work.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 9286609..d60a39b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
import org.apache.hadoop.hdfs.server.namenode.Namesystem;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.Time;
@@ -192,7 +193,7 @@ class HeartbeatManager implements DatanodeStatistics {
addDatanode(d);
//update its timestamp
- d.updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0);
+ d.updateHeartbeatState(StorageReport.EMPTY_ARRAY, 0L, 0L, 0, 0, null);
}
}
@@ -217,10 +218,11 @@ class HeartbeatManager implements DatanodeStatistics {
synchronized void updateHeartbeat(final DatanodeDescriptor node,
StorageReport[] reports, long cacheCapacity, long cacheUsed,
- int xceiverCount, int failedVolumes) {
+ int xceiverCount, int failedVolumes,
+ VolumeFailureSummary volumeFailureSummary) {
stats.subtract(node);
node.updateHeartbeat(reports, cacheCapacity, cacheUsed,
- xceiverCount, failedVolumes);
+ xceiverCount, failedVolumes, volumeFailureSummary);
stats.add(node);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index 69210db..ff1ad78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.util.Time;
@@ -579,14 +580,19 @@ class BPServiceActor implements Runnable {
LOG.debug("Sending heartbeat with " + reports.length +
" storage reports from service actor: " + this);
}
-
+
+ VolumeFailureSummary volumeFailureSummary = dn.getFSDataset()
+ .getVolumeFailureSummary();
+ int numFailedVolumes = volumeFailureSummary != null ?
+ volumeFailureSummary.getFailedStorageLocations().length : 0;
return bpNamenode.sendHeartbeat(bpRegistration,
reports,
dn.getFSDataset().getCacheCapacity(),
dn.getFSDataset().getCacheUsed(),
dn.getXmitsInProgress(),
dn.getXceiverCount(),
- dn.getFSDataset().getNumFailedVolumes());
+ numFailedVolumes,
+ volumeFailureSummary);
}
//This must be called only by BPOfferService
@@ -1019,4 +1025,4 @@ class BPServiceActor implements Runnable {
}
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
index 7d880d3..437b795 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/FsDatasetSpi.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
import org.apache.hadoop.util.ReflectionUtils;
@@ -129,6 +130,13 @@ public interface FsDatasetSpi<V extends FsVolumeSpi> extends FSDatasetMBean {
/** @return a volume information map (name => info). */
public Map<String, Object> getVolumeInfoMap();
+ /**
+ * Returns info about volume failures.
+ *
+ * @return info about volume failures, possibly null
+ */
+ VolumeFailureSummary getVolumeFailureSummary();
+
/** @return a list of finalized blocks for the given block pool. */
public List<FinalizedReplica> getFinalizedBlocks(String bpid);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 8c8d744..2b9feb5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -99,6 +99,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.nativeio.NativeIO;
@@ -112,6 +113,7 @@ import org.apache.hadoop.util.Time;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.collect.Sets;
/**************************************************
* FSDataset manages a set of data blocks. Each block
@@ -264,9 +266,11 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
String[] dataDirs = conf.getTrimmedStrings(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY);
Collection<StorageLocation> dataLocations = DataNode.getStorageLocations(conf);
+ List<VolumeFailureInfo> volumeFailureInfos = getInitialVolumeFailureInfos(
+ dataLocations, storage);
int volsConfigured = (dataDirs == null) ? 0 : dataDirs.length;
- int volsFailed = volsConfigured - storage.getNumStorageDirs();
+ int volsFailed = volumeFailureInfos.size();
this.validVolsRequired = volsConfigured - volFailuresTolerated;
if (volFailuresTolerated < 0 || volFailuresTolerated >= volsConfigured) {
@@ -291,7 +295,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
DFSConfigKeys.DFS_DATANODE_FSDATASET_VOLUME_CHOOSING_POLICY_KEY,
RoundRobinVolumeChoosingPolicy.class,
VolumeChoosingPolicy.class), conf);
- volumes = new FsVolumeList(volsFailed, datanode.getBlockScanner(),
+ volumes = new FsVolumeList(volumeFailureInfos, datanode.getBlockScanner(),
blockChooserImpl);
asyncDiskService = new FsDatasetAsyncDiskService(datanode);
asyncLazyPersistService = new RamDiskAsyncLazyPersistService(datanode);
@@ -313,6 +317,36 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
DFSConfigKeys.DFS_DATANODE_BLOCK_PINNING_ENABLED_DEFAULT);
}
+ /**
+ * Gets initial volume failure information for all volumes that failed
+ * immediately at startup. The method works by determining the set difference
+ * between all configured storage locations and the actual storage locations in
+ * use after attempting to put all of them into service.
+ *
+ * @return each storage location that has failed
+ */
+ private static List<VolumeFailureInfo> getInitialVolumeFailureInfos(
+ Collection<StorageLocation> dataLocations, DataStorage storage) {
+ Set<String> failedLocationSet = Sets.newHashSetWithExpectedSize(
+ dataLocations.size());
+ for (StorageLocation sl: dataLocations) {
+ failedLocationSet.add(sl.getFile().getAbsolutePath());
+ }
+ for (Iterator<Storage.StorageDirectory> it = storage.dirIterator();
+ it.hasNext(); ) {
+ Storage.StorageDirectory sd = it.next();
+ failedLocationSet.remove(sd.getRoot().getAbsolutePath());
+ }
+ List<VolumeFailureInfo> volumeFailureInfos = Lists.newArrayListWithCapacity(
+ failedLocationSet.size());
+ long failureDate = Time.now();
+ for (String failedStorageLocation: failedLocationSet) {
+ volumeFailureInfos.add(new VolumeFailureInfo(failedStorageLocation,
+ failureDate));
+ }
+ return volumeFailureInfos;
+ }
+
private void addVolume(Collection<StorageLocation> dataLocations,
Storage.StorageDirectory sd) throws IOException {
final File dir = sd.getCurrentDir();
@@ -348,8 +382,14 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
final File dir = location.getFile();
// Prepare volume in DataStorage
- DataStorage.VolumeBuilder builder =
- dataStorage.prepareVolume(datanode, location.getFile(), nsInfos);
+ final DataStorage.VolumeBuilder builder;
+ try {
+ builder = dataStorage.prepareVolume(datanode, location.getFile(), nsInfos);
+ } catch (IOException e) {
+ volumes.addVolumeFailureInfo(new VolumeFailureInfo(
+ location.getFile().getAbsolutePath(), Time.now()));
+ throw e;
+ }
final Storage.StorageDirectory sd = builder.getStorageDirectory();
@@ -498,9 +538,65 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
/**
* Return the number of failed volumes in the FSDataset.
*/
- @Override
+ @Override // FSDatasetMBean
public int getNumFailedVolumes() {
- return volumes.numberOfFailedVolumes();
+ return volumes.getVolumeFailureInfos().length;
+ }
+
+ @Override // FSDatasetMBean
+ public String[] getFailedStorageLocations() {
+ VolumeFailureInfo[] infos = volumes.getVolumeFailureInfos();
+ List<String> failedStorageLocations = Lists.newArrayListWithCapacity(
+ infos.length);
+ for (VolumeFailureInfo info: infos) {
+ failedStorageLocations.add(info.getFailedStorageLocation());
+ }
+ return failedStorageLocations.toArray(
+ new String[failedStorageLocations.size()]);
+ }
+
+ @Override // FSDatasetMBean
+ public long getLastVolumeFailureDate() {
+ long lastVolumeFailureDate = 0;
+ for (VolumeFailureInfo info: volumes.getVolumeFailureInfos()) {
+ long failureDate = info.getFailureDate();
+ if (failureDate > lastVolumeFailureDate) {
+ lastVolumeFailureDate = failureDate;
+ }
+ }
+ return lastVolumeFailureDate;
+ }
+
+ @Override // FSDatasetMBean
+ public long getEstimatedCapacityLostTotal() {
+ long estimatedCapacityLostTotal = 0;
+ for (VolumeFailureInfo info: volumes.getVolumeFailureInfos()) {
+ estimatedCapacityLostTotal += info.getEstimatedCapacityLost();
+ }
+ return estimatedCapacityLostTotal;
+ }
+
+ @Override // FsDatasetSpi
+ public VolumeFailureSummary getVolumeFailureSummary() {
+ VolumeFailureInfo[] infos = volumes.getVolumeFailureInfos();
+ if (infos.length == 0) {
+ return null;
+ }
+ List<String> failedStorageLocations = Lists.newArrayListWithCapacity(
+ infos.length);
+ long lastVolumeFailureDate = 0;
+ long estimatedCapacityLostTotal = 0;
+ for (VolumeFailureInfo info: infos) {
+ failedStorageLocations.add(info.getFailedStorageLocation());
+ long failureDate = info.getFailureDate();
+ if (failureDate > lastVolumeFailureDate) {
+ lastVolumeFailureDate = failureDate;
+ }
+ estimatedCapacityLostTotal += info.getEstimatedCapacityLost();
+ }
+ return new VolumeFailureSummary(
+ failedStorageLocations.toArray(new String[failedStorageLocations.size()]),
+ lastVolumeFailureDate, estimatedCapacityLostTotal);
}
@Override // FSDatasetMBean
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
index ae2f5b4..4573172 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeList.java
@@ -22,9 +22,12 @@ import java.io.IOException;
import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collection;
import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicReference;
import com.google.common.collect.Lists;
@@ -40,21 +43,23 @@ import org.apache.hadoop.util.Time;
class FsVolumeList {
private final AtomicReference<FsVolumeImpl[]> volumes =
new AtomicReference<>(new FsVolumeImpl[0]);
+ // Tracks volume failures, sorted by volume path.
+ private final Map<String, VolumeFailureInfo> volumeFailureInfos =
+ Collections.synchronizedMap(new TreeMap<String, VolumeFailureInfo>());
private Object checkDirsMutex = new Object();
private final VolumeChoosingPolicy<FsVolumeImpl> blockChooser;
private final BlockScanner blockScanner;
- private volatile int numFailedVolumes;
- FsVolumeList(int failedVols, BlockScanner blockScanner,
+ FsVolumeList(List<VolumeFailureInfo> initialVolumeFailureInfos,
+ BlockScanner blockScanner,
VolumeChoosingPolicy<FsVolumeImpl> blockChooser) {
this.blockChooser = blockChooser;
this.blockScanner = blockScanner;
- this.numFailedVolumes = failedVols;
- }
-
- int numberOfFailedVolumes() {
- return numFailedVolumes;
+ for (VolumeFailureInfo volumeFailureInfo: initialVolumeFailureInfos) {
+ volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(),
+ volumeFailureInfo);
+ }
}
/**
@@ -238,7 +243,7 @@ class FsVolumeList {
}
removedVols.add(fsv);
removeVolume(fsv);
- numFailedVolumes++;
+ addVolumeFailureInfo(fsv);
} catch (ClosedChannelException e) {
FsDatasetImpl.LOG.debug("Caught exception when obtaining " +
"reference count on closed volume", e);
@@ -347,6 +352,26 @@ class FsVolumeList {
removeVolume(fsVolume);
}
}
+ removeVolumeFailureInfo(volume);
+ }
+
+ VolumeFailureInfo[] getVolumeFailureInfos() {
+ Collection<VolumeFailureInfo> infos = volumeFailureInfos.values();
+ return infos.toArray(new VolumeFailureInfo[infos.size()]);
+ }
+
+ void addVolumeFailureInfo(VolumeFailureInfo volumeFailureInfo) {
+ volumeFailureInfos.put(volumeFailureInfo.getFailedStorageLocation(),
+ volumeFailureInfo);
+ }
+
+ private void addVolumeFailureInfo(FsVolumeImpl vol) {
+ addVolumeFailureInfo(new VolumeFailureInfo(vol.getBasePath(), Time.now(),
+ vol.getCapacity()));
+ }
+
+ private void removeVolumeFailureInfo(File vol) {
+ volumeFailureInfos.remove(vol.getAbsolutePath());
}
void addBlockPool(final String bpid, final Configuration conf) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java
new file mode 100644
index 0000000..c3ce2a4
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/VolumeFailureInfo.java
@@ -0,0 +1,82 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
+
+/**
+ * Tracks information about failure of a data volume.
+ */
+final class VolumeFailureInfo {
+ private final String failedStorageLocation;
+ private final long failureDate;
+ private final long estimatedCapacityLost;
+
+ /**
+ * Creates a new VolumeFailureInfo, when the capacity lost from this volume
+ * failure is unknown. Typically, this means the volume failed immediately at
+ * startup, so there was never a chance to query its capacity.
+ *
+ * @param failedStorageLocation storage location that has failed
+ * @param failureDate date/time of failure in milliseconds since epoch
+ */
+ public VolumeFailureInfo(String failedStorageLocation, long failureDate) {
+ this(failedStorageLocation, failureDate, 0);
+ }
+
+ /**
+ * Creates a new VolumeFailureInfo.
+ *
+ * @param failedStorageLocation storage location that has failed
+ * @param failureDate date/time of failure in milliseconds since epoch
+ * @param estimatedCapacityLost estimate of capacity lost in bytes
+ */
+ public VolumeFailureInfo(String failedStorageLocation, long failureDate,
+ long estimatedCapacityLost) {
+ this.failedStorageLocation = failedStorageLocation;
+ this.failureDate = failureDate;
+ this.estimatedCapacityLost = estimatedCapacityLost;
+ }
+
+ /**
+ * Returns the storage location that has failed.
+ *
+ * @return storage location that has failed
+ */
+ public String getFailedStorageLocation() {
+ return this.failedStorageLocation;
+ }
+
+ /**
+ * Returns date/time of failure
+ *
+ * @return date/time of failure in milliseconds since epoch
+ */
+ public long getFailureDate() {
+ return this.failureDate;
+ }
+
+ /**
+ * Returns estimate of capacity lost. This is said to be an estimate, because
+ * in some cases it's impossible to know the capacity of the volume, such as if
+ * we never had a chance to query its capacity before the failure occurred.
+ *
+ * @return estimate of capacity lost in bytes
+ */
+ public long getEstimatedCapacityLost() {
+ return this.estimatedCapacityLost;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
index 8388c0b..5f22540 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/metrics/FSDatasetMBean.java
@@ -79,6 +79,25 @@ public interface FSDatasetMBean {
public int getNumFailedVolumes();
/**
+ * Returns each storage location that has failed, sorted.
+ * @return each storage location that has failed, sorted
+ */
+ String[] getFailedStorageLocations();
+
+ /**
+ * Returns the date/time of the last volume failure in milliseconds since
+ * epoch.
+ * @return date/time of last volume failure in milliseconds since epoch
+ */
+ long getLastVolumeFailureDate();
+
+ /**
+ * Returns an estimate of total capacity lost due to volume failures in bytes.
+ * @return estimate of total capacity lost in bytes
+ */
+ long getEstimatedCapacityLostTotal();
+
+ /**
* Returns the amount of cache used by the datanode (in bytes).
*/
public long getCacheUsed();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 5fd1efb..84ab179 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -250,6 +250,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
@@ -4412,8 +4413,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
*/
HeartbeatResponse handleHeartbeat(DatanodeRegistration nodeReg,
StorageReport[] reports, long cacheCapacity, long cacheUsed,
- int xceiverCount, int xmitsInProgress, int failedVolumes)
- throws IOException {
+ int xceiverCount, int xmitsInProgress, int failedVolumes,
+ VolumeFailureSummary volumeFailureSummary) throws IOException {
readLock();
try {
//get datanode commands
@@ -4421,7 +4422,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
- xmitsInProgress;
DatanodeCommand[] cmds = blockManager.getDatanodeManager().handleHeartbeat(
nodeReg, reports, blockPoolId, cacheCapacity, cacheUsed,
- xceiverCount, maxTransfer, failedVolumes);
+ xceiverCount, maxTransfer, failedVolumes, volumeFailureSummary);
//create ha status
final NNHAStatusHeartbeat haState = new NNHAStatusHeartbeat(
@@ -5943,6 +5944,32 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
@Override // FSNamesystemMBean
+ public int getVolumeFailuresTotal() {
+ List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
+ getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true);
+ int volumeFailuresTotal = 0;
+ for (DatanodeDescriptor node: live) {
+ volumeFailuresTotal += node.getVolumeFailures();
+ }
+ return volumeFailuresTotal;
+ }
+
+ @Override // FSNamesystemMBean
+ public long getEstimatedCapacityLostTotal() {
+ List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
+ getBlockManager().getDatanodeManager().fetchDatanodes(live, null, true);
+ long estimatedCapacityLostTotal = 0;
+ for (DatanodeDescriptor node: live) {
+ VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary();
+ if (volumeFailureSummary != null) {
+ estimatedCapacityLostTotal +=
+ volumeFailureSummary.getEstimatedCapacityLostTotal();
+ }
+ }
+ return estimatedCapacityLostTotal;
+ }
+
+ @Override // FSNamesystemMBean
public int getNumDecommissioningDataNodes() {
return getBlockManager().getDatanodeManager().getDecommissioningNodes()
.size();
@@ -6785,7 +6812,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final List<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
blockManager.getDatanodeManager().fetchDatanodes(live, null, true);
for (DatanodeDescriptor node : live) {
- Map<String, Object> innerinfo = ImmutableMap.<String, Object>builder()
+ ImmutableMap.Builder<String, Object> innerinfo =
+ ImmutableMap.<String,Object>builder();
+ innerinfo
.put("infoAddr", node.getInfoAddr())
.put("infoSecureAddr", node.getInfoSecureAddr())
.put("xferaddr", node.getXferAddr())
@@ -6801,9 +6830,18 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
.put("blockScheduled", node.getBlocksScheduled())
.put("blockPoolUsed", node.getBlockPoolUsed())
.put("blockPoolUsedPercent", node.getBlockPoolUsedPercent())
- .put("volfails", node.getVolumeFailures())
- .build();
- info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo);
+ .put("volfails", node.getVolumeFailures());
+ VolumeFailureSummary volumeFailureSummary = node.getVolumeFailureSummary();
+ if (volumeFailureSummary != null) {
+ innerinfo
+ .put("failedStorageLocations",
+ volumeFailureSummary.getFailedStorageLocations())
+ .put("lastVolumeFailureDate",
+ volumeFailureSummary.getLastVolumeFailureDate())
+ .put("estimatedCapacityLostTotal",
+ volumeFailureSummary.getEstimatedCapacityLostTotal());
+ }
+ info.put(node.getHostName() + ":" + node.getXferPort(), innerinfo.build());
}
return JSON.toString(info);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 54f5f85..da34bd2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -136,6 +136,7 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
@@ -1281,12 +1282,13 @@ class NameNodeRpcServer implements NamenodeProtocols {
public HeartbeatResponse sendHeartbeat(DatanodeRegistration nodeReg,
StorageReport[] report, long dnCacheCapacity, long dnCacheUsed,
int xmitsInProgress, int xceiverCount,
- int failedVolumes) throws IOException {
+ int failedVolumes, VolumeFailureSummary volumeFailureSummary)
+ throws IOException {
checkNNStartup();
verifyRequest(nodeReg);
return namesystem.handleHeartbeat(nodeReg, report,
dnCacheCapacity, dnCacheUsed, xceiverCount, xmitsInProgress,
- failedVolumes);
+ failedVolumes, volumeFailureSummary);
}
@Override // DatanodeProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
index 86f4bd6..b31b7b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/metrics/FSNamesystemMBean.java
@@ -132,6 +132,19 @@ public interface FSNamesystemMBean {
public int getNumDecomDeadDataNodes();
/**
+ * Number of failed data volumes across all live data nodes.
+ * @return number of failed data volumes across all live data nodes
+ */
+ int getVolumeFailuresTotal();
+
+ /**
+ * Returns an estimate of total capacity lost due to volume failures in bytes
+ * across all live data nodes.
+ * @return estimate of total capacity lost in bytes
+ */
+ long getEstimatedCapacityLostTotal();
+
+ /**
* Number of data nodes that are in the decommissioning state
*/
public int getNumDecommissioningDataNodes();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
index c3b0f68..047de56 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeProtocol.java
@@ -102,6 +102,7 @@ public interface DatanodeProtocol {
* @param xmitsInProgress number of transfers from this datanode to others
* @param xceiverCount number of active transceiver threads
* @param failedVolumes number of failed volumes
+ * @param volumeFailureSummary info about volume failures
* @throws IOException on error
*/
@Idempotent
@@ -111,7 +112,9 @@ public interface DatanodeProtocol {
long dnCacheUsed,
int xmitsInProgress,
int xceiverCount,
- int failedVolumes) throws IOException;
+ int failedVolumes,
+ VolumeFailureSummary volumeFailureSummary)
+ throws IOException;
/**
* blockReport() tells the NameNode about all the locally-stored blocks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java
new file mode 100644
index 0000000..1722dd0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/VolumeFailureSummary.java
@@ -0,0 +1,72 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.protocol;
+
+/**
+ * Summarizes information about data volume failures on a DataNode.
+ */
+public class VolumeFailureSummary {
+ private final String[] failedStorageLocations;
+ private final long lastVolumeFailureDate;
+ private final long estimatedCapacityLostTotal;
+
+ /**
+ * Creates a new VolumeFailureSummary.
+ *
+ * @param failedStorageLocations storage locations that have failed
+ * @param lastVolumeFailureDate date/time of last volume failure in
+ * milliseconds since epoch
+ * @param estimatedCapacityLostTotal estimate of capacity lost in bytes
+ */
+ public VolumeFailureSummary(String[] failedStorageLocations,
+ long lastVolumeFailureDate, long estimatedCapacityLostTotal) {
+ this.failedStorageLocations = failedStorageLocations;
+ this.lastVolumeFailureDate = lastVolumeFailureDate;
+ this.estimatedCapacityLostTotal = estimatedCapacityLostTotal;
+ }
+
+ /**
+ * Returns each storage location that has failed, sorted.
+ *
+ * @return each storage location that has failed, sorted
+ */
+ public String[] getFailedStorageLocations() {
+ return this.failedStorageLocations;
+ }
+
+ /**
+ * Returns the date/time of the last volume failure in milliseconds since
+ * epoch.
+ *
+ * @return date/time of last volume failure in milliseconds since epoch
+ */
+ public long getLastVolumeFailureDate() {
+ return this.lastVolumeFailureDate;
+ }
+
+ /**
+ * Returns estimate of capacity lost. This is said to be an estimate, because
+ * in some cases it's impossible to know the capacity of the volume, such as if
+ * we never had a chance to query its capacity before the failure occurred.
+ *
+ * @return estimate of capacity lost in bytes
+ */
+ public long getEstimatedCapacityLostTotal() {
+ return this.estimatedCapacityLostTotal;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
index 187761a..348f346 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/DatanodeProtocol.proto
@@ -161,6 +161,17 @@ message RegisterDatanodeResponseProto {
}
/**
+ * failedStorageLocations - storage locations that have failed
+ * lastVolumeFailureDate - date/time of last volume failure
+ * estimatedCapacityLost - estimate of total capacity lost due to volume failures
+ */
+message VolumeFailureSummaryProto {
+ repeated string failedStorageLocations = 1;
+ required uint64 lastVolumeFailureDate = 2;
+ required uint64 estimatedCapacityLostTotal = 3;
+}
+
+/**
* registration - datanode registration information
* capacity - total storage capacity available at the datanode
* dfsUsed - storage used by HDFS
@@ -168,9 +179,12 @@ message RegisterDatanodeResponseProto {
* blockPoolUsed - storage used by the block pool
* xmitsInProgress - number of transfers from this datanode to others
* xceiverCount - number of active transceiver threads
- * failedVolumes - number of failed volumes
+ * failedVolumes - number of failed volumes. This is redundant with the
+ * information included in volumeFailureSummary, but the field is retained
+ * for backwards compatibility.
* cacheCapacity - total cache capacity available at the datanode
* cacheUsed - amount of cache used
+ * volumeFailureSummary - info about volume failures
*/
message HeartbeatRequestProto {
required DatanodeRegistrationProto registration = 1; // Datanode info
@@ -180,6 +194,7 @@ message HeartbeatRequestProto {
optional uint32 failedVolumes = 5 [ default = 0 ];
optional uint64 cacheCapacity = 6 [ default = 0 ];
optional uint64 cacheUsed = 7 [default = 0 ];
+ optional VolumeFailureSummaryProto volumeFailureSummary = 8;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
index 9c83f3a..391ca79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.html
@@ -34,6 +34,7 @@
<ul class="nav navbar-nav" id="ui-tabs">
<li><a href="#tab-overview">Overview</a></li>
<li><a href="#tab-datanode">Datanodes</a></li>
+ <li><a href="#tab-datanode-volume-failures">Datanode Volume Failures</a></li>
<li><a href="#tab-snapshot">Snapshot</a></li>
<li><a href="#tab-startup-progress">Startup Progress</a></li>
<li class="dropdown">
@@ -59,6 +60,7 @@
<div class="tab-content">
<div class="tab-pane" id="tab-overview"></div>
<div class="tab-pane" id="tab-datanode"></div>
+ <div class="tab-pane" id="tab-datanode-volume-failures"></div>
<div class="tab-pane" id="tab-snapshot"></div>
<div class="tab-pane" id="tab-startup-progress"></div>
</div>
@@ -165,6 +167,7 @@
<tr><th><a href="#tab-datanode">Live Nodes</a></th><td>{NumLiveDataNodes} (Decommissioned: {NumDecomLiveDataNodes})</td></tr>
<tr><th><a href="#tab-datanode">Dead Nodes</a></th><td>{NumDeadDataNodes} (Decommissioned: {NumDecomDeadDataNodes})</td></tr>
<tr><th><a href="#tab-datanode">Decommissioning Nodes</a></th><td>{NumDecommissioningDataNodes}</td></tr>
+ <tr><th><a href="#tab-datanode-volume-failures">Total Datanode Volume Failures</a></th><td>{VolumeFailuresTotal} ({EstimatedCapacityLostTotal|fmt_bytes})</td></tr>
<tr><th title="Excludes missing blocks.">Number of Under-Replicated Blocks</th><td>{UnderReplicatedBlocks}</td></tr>
<tr><th>Number of Blocks Pending Deletion</th><td>{PendingDeletionBlocks}</td></tr>
<tr><th>Block Deletion Start Time</th><td>{BlockDeletionStartTime|date_tostring}</td></tr>
@@ -324,6 +327,36 @@
</small>
</script>
+<script type="text/x-dust-template" id="tmpl-datanode-volume-failures">
+<div class="page-header"><h1>Datanode Volume Failures</h1></div>
+<small>
+{?LiveNodes}
+<table class="table">
+ <thead>
+ <tr>
+ <th>Node</th>
+ <th>Last Failure Date</th>
+ <th>Failed Volumes</th>
+ <th>Estimated Capacity Lost</th>
+ <th>Failed Storage Locations</th>
+ </tr>
+ </thead>
+ {#LiveNodes}
+ <tr>
+ <td>{name} ({xferaddr})</td>
+ <td>{#helper_date_tostring value="{lastVolumeFailureDate}"/}</td>
+ <td>{volfails}</td>
+ <td>{estimatedCapacityLostTotal|fmt_bytes}</td>
+ <td>{#failedStorageLocations}{.}{@sep}<br />{/sep}{/failedStorageLocations}</td>
+ </tr>
+ {/LiveNodes}
+</table>
+{:else}
+There are no reported volume failures.
+{/LiveNodes}
+</small>
+</script>
+
<script type="text/x-dust-template" id="tmpl-startup-progress">
<div class="page-header"><h1>Startup Progress</h1></div>
<p>Elapsed Time: {elapsedTime|fmt_time}, Percent Complete: {percentComplete|fmt_percentage}</p>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
index f6dc627..a045e42 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/dfshealth.js
@@ -21,6 +21,7 @@
dust.loadSource(dust.compile($('#tmpl-dfshealth').html(), 'dfshealth'));
dust.loadSource(dust.compile($('#tmpl-startup-progress').html(), 'startup-progress'));
dust.loadSource(dust.compile($('#tmpl-datanode').html(), 'datanode-info'));
+ dust.loadSource(dust.compile($('#tmpl-datanode-volume-failures').html(), 'datanode-volume-failures'));
dust.loadSource(dust.compile($('#tmpl-snapshot').html(), 'snapshot-info'));
function load_overview() {
@@ -193,6 +194,45 @@
})).error(ajax_error_handler);
}
+ function load_datanode_volume_failures() {
+
+ var HELPERS = {
+ 'helper_date_tostring' : function (chunk, ctx, bodies, params) {
+ var value = dust.helpers.tap(params.value, chunk, ctx);
+ return chunk.write('' + new Date(Number(value)).toLocaleString());
+ }
+ };
+
+ function workaround(r) {
+ function node_map_to_array(nodes) {
+ var res = [];
+ for (var n in nodes) {
+ var p = nodes[n];
+ // Filter the display to only datanodes with volume failures.
+ if (p.volfails > 0) {
+ p.name = n;
+ res.push(p);
+ }
+ }
+ return res;
+ }
+
+ r.LiveNodes = node_map_to_array(JSON.parse(r.LiveNodes));
+ return r;
+ }
+
+ $.get(
+ '/jmx?qry=Hadoop:service=NameNode,name=NameNodeInfo',
+ guard_with_startup_progress(function (resp) {
+ var data = workaround(resp.beans[0]);
+ var base = dust.makeBase(HELPERS);
+ dust.render('datanode-volume-failures', base.push(data), function(err, out) {
+ $('#tab-datanode-volume-failures').html(out);
+ $('#ui-tabs a[href="#tab-datanode-volume-failures"]').tab('show');
+ });
+ })).error(ajax_error_handler);
+ }
+
function load_snapshot_info() {
$.get(
'/jmx?qry=Hadoop:service=NameNode,name=SnapshotInfo',
@@ -210,6 +250,9 @@
case "#tab-datanode":
load_datanode_info();
break;
+ case "#tab-datanode-volume-failures":
+ load_datanode_volume_failures();
+ break;
case "#tab-snapshot":
load_snapshot_info();
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index c7dfcf9..698f9f4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -114,7 +114,8 @@ public class TestBlockManager {
2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L,
2 * HdfsConstants.MIN_BLOCKS_FOR_WRITE*BLOCK_SIZE, 0L);
dn.updateHeartbeat(
- BlockManagerTestUtil.getStorageReportsForDatanode(dn), 0L, 0L, 0, 0);
+ BlockManagerTestUtil.getStorageReportsForDatanode(dn), 0L, 0L, 0, 0,
+ null);
bm.getDatanodeManager().checkIfClusterIsNowMultiRack(dn);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
index 88300de..b67ae7a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestNameNodePrunesMissingStorages.java
@@ -18,7 +18,6 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
-import org.apache.commons.lang.ArrayUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.math3.stat.inference.TestUtils;
@@ -87,7 +86,8 @@ public class TestNameNodePrunesMissingStorages {
// Stop the DataNode and send fake heartbeat with missing storage.
cluster.stopDataNode(0);
- cluster.getNameNodeRpc().sendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0, 0);
+ cluster.getNameNodeRpc().sendHeartbeat(dnReg, prunedReports, 0L, 0L, 0, 0,
+ 0, null);
// Check that the missing storage was pruned.
assertThat(dnDescriptor.getStorageInfos().length, is(expectedStoragesAfterTest));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
index 2942d0f..6bbb0c3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestOverReplicatedBlocks.java
@@ -107,7 +107,7 @@ public class TestOverReplicatedBlocks {
datanode.getStorageInfos()[0].setUtilizationForTesting(100L, 100L, 0, 100L);
datanode.updateHeartbeat(
BlockManagerTestUtil.getStorageReportsForDatanode(datanode),
- 0L, 0L, 0, 0);
+ 0L, 0L, 0, 0, null);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
index 1c008ac..3e6b85f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicy.java
@@ -99,7 +99,7 @@ public class TestReplicationPolicy {
capacity, dfsUsed, remaining, blockPoolUsed);
dn.updateHeartbeat(
BlockManagerTestUtil.getStorageReportsForDatanode(dn),
- dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures);
+ dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures, null);
}
@BeforeClass
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
index a1f3e38..d9066e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyConsiderLoad.java
@@ -97,7 +97,7 @@ public class TestReplicationPolicyConsiderLoad {
2*HdfsConstants.MIN_BLOCKS_FOR_WRITE*blockSize, 0L);
dataNodes[i].updateHeartbeat(
BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[i]),
- 0L, 0L, 0, 0);
+ 0L, 0L, 0, 0, null);
}
}
@@ -115,17 +115,17 @@ public class TestReplicationPolicyConsiderLoad {
BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[3]),
blockPoolId, dataNodes[3].getCacheCapacity(),
dataNodes[3].getCacheRemaining(),
- 2, 0, 0);
+ 2, 0, 0, null);
dnManager.handleHeartbeat(dnrList.get(4),
BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[4]),
blockPoolId, dataNodes[4].getCacheCapacity(),
dataNodes[4].getCacheRemaining(),
- 4, 0, 0);
+ 4, 0, 0, null);
dnManager.handleHeartbeat(dnrList.get(5),
BlockManagerTestUtil.getStorageReportsForDatanode(dataNodes[5]),
blockPoolId, dataNodes[5].getCacheCapacity(),
dataNodes[5].getCacheRemaining(),
- 4, 0, 0);
+ 4, 0, 0, null);
// value in the above heartbeats
final int load = 2 + 4 + 4;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
index 526c490..a1ea73c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestReplicationPolicyWithNodeGroup.java
@@ -185,7 +185,7 @@ public class TestReplicationPolicyWithNodeGroup {
capacity, dfsUsed, remaining, blockPoolUsed);
dn.updateHeartbeat(
BlockManagerTestUtil.getStorageReportsForDatanode(dn),
- dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures);
+ dnCacheCapacity, dnCacheUsed, xceiverCount, volFailures, null);
}
private static void setupDataNodeCapacity() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index 36b62e5..991aa9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -35,6 +35,7 @@ import javax.management.NotCompliantMBeanException;
import javax.management.ObjectName;
import javax.management.StandardMBean;
+import org.apache.commons.lang.ArrayUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.StorageType;
@@ -57,6 +58,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.util.DataChecksum;
@@ -628,6 +630,26 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
}
@Override // FSDatasetMBean
+ public String[] getFailedStorageLocations() {
+ return null;
+ }
+
+ @Override // FSDatasetMBean
+ public long getLastVolumeFailureDate() {
+ return 0;
+ }
+
+ @Override // FSDatasetMBean
+ public long getEstimatedCapacityLostTotal() {
+ return 0;
+ }
+
+ @Override // FsDatasetSpi
+ public VolumeFailureSummary getVolumeFailureSummary() {
+ return new VolumeFailureSummary(ArrayUtils.EMPTY_STRING_ARRAY, 0, 0);
+ }
+
+ @Override // FSDatasetMBean
public long getCacheUsed() {
return 0l;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
index e21ce38..bc49793 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBPOfferService.java
@@ -53,6 +53,7 @@ import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.test.PathUtils;
import org.apache.hadoop.util.Time;
@@ -137,7 +138,8 @@ public class TestBPOfferService {
Mockito.anyLong(),
Mockito.anyInt(),
Mockito.anyInt(),
- Mockito.anyInt());
+ Mockito.anyInt(),
+ Mockito.any(VolumeFailureSummary.class));
mockHaStatuses[nnIdx] = new NNHAStatusHeartbeat(HAServiceState.STANDBY, 0);
return mock;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
index c7a4084..7490723 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestBlockRecovery.java
@@ -77,6 +77,7 @@ import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.DataChecksum;
@@ -160,7 +161,8 @@ public class TestBlockRecovery {
Mockito.anyLong(),
Mockito.anyInt(),
Mockito.anyInt(),
- Mockito.anyInt()))
+ Mockito.anyInt(),
+ Mockito.any(VolumeFailureSummary.class)))
.thenReturn(new HeartbeatResponse(
new DatanodeCommand[0],
new NNHAStatusHeartbeat(HAServiceState.ACTIVE, 1),
[23/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
new file mode 100644
index 0000000..ccc9590
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduceTutorial.md
@@ -0,0 +1,1156 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+MapReduce Tutorial
+==================
+
+* [MapReduce Tutorial](#MapReduce_Tutorial)
+ * [Purpose](#Purpose)
+ * [Prerequisites](#Prerequisites)
+ * [Overview](#Overview)
+ * [Inputs and Outputs](#Inputs_and_Outputs)
+ * [Example: WordCount v1.0](#Example:_WordCount_v1.0)
+ * [Source Code](#Source_Code)
+ * [Usage](#Usage)
+ * [Walk-through](#Walk-through)
+ * [MapReduce - User Interfaces](#MapReduce_-_User_Interfaces)
+ * [Payload](#Payload)
+ * [Mapper](#Mapper)
+ * [Reducer](#Reducer)
+ * [Partitioner](#Partitioner)
+ * [Counter](#Counter)
+ * [Job Configuration](#Job_Configuration)
+ * [Task Execution & Environment](#Task_Execution__Environment)
+ * [Memory Management](#Memory_Management)
+ * [Map Parameters](#Map_Parameters)
+ * [Shuffle/Reduce Parameters](#ShuffleReduce_Parameters)
+ * [Configured Parameters](#Configured_Parameters)
+ * [Task Logs](#Task_Logs)
+ * [Distributing Libraries](#Distributing_Libraries)
+ * [Job Submission and Monitoring](#Job_Submission_and_Monitoring)
+ * [Job Control](#Job_Control)
+ * [Job Input](#Job_Input)
+ * [InputSplit](#InputSplit)
+ * [RecordReader](#RecordReader)
+ * [Job Output](#Job_Output)
+ * [OutputCommitter](#OutputCommitter)
+ * [Task Side-Effect Files](#Task_Side-Effect_Files)
+ * [RecordWriter](#RecordWriter)
+ * [Other Useful Features](#Other_Useful_Features)
+ * [Submitting Jobs to Queues](#Submitting_Jobs_to_Queues)
+ * [Counters](#Counters)
+ * [DistributedCache](#DistributedCache)
+ * [Profiling](#Profiling)
+ * [Debugging](#Debugging)
+ * [Data Compression](#Data_Compression)
+ * [Skipping Bad Records](#Skipping_Bad_Records)
+ * [Example: WordCount v2.0](#Example:_WordCount_v2.0)
+ * [Source Code](#Source_Code)
+ * [Sample Runs](#Sample_Runs)
+ * [Highlights](#Highlights)
+
+Purpose
+-------
+
+This document comprehensively describes all user-facing facets of the Hadoop MapReduce framework and serves as a tutorial.
+
+Prerequisites
+-------------
+
+Ensure that Hadoop is installed, configured and is running. More details:
+
+* [Single Node Setup](../../hadoop-project-dist/hadoop-common/SingleCluster.html)
+ for first-time users.
+
+* [Cluster Setup](../../hadoop-project-dist/hadoop-common/ClusterSetup.html)
+ for large, distributed clusters.
+
+Overview
+--------
+
+Hadoop MapReduce is a software framework for easily writing applications which process vast amounts of data (multi-terabyte data-sets) in-parallel on large clusters (thousands of nodes) of commodity hardware in a reliable, fault-tolerant manner.
+
+A MapReduce *job* usually splits the input data-set into independent chunks which are processed by the *map tasks* in a completely parallel manner. The framework sorts the outputs of the maps, which are then input to the *reduce tasks*. Typically both the input and the output of the job are stored in a file-system. The framework takes care of scheduling tasks, monitoring them and re-executes the failed tasks.
+
+Typically the compute nodes and the storage nodes are the same, that is, the MapReduce framework and the Hadoop Distributed File System (see [HDFS Architecture Guide](../../hadoop-project-dist/hadoop-hdfs/HdfsDesign.html)) are running on the same set of nodes. This configuration allows the framework to effectively schedule tasks on the nodes where data is already present, resulting in very high aggregate bandwidth across the cluster.
+
+The MapReduce framework consists of a single master `ResourceManager`, one slave `NodeManager` per cluster-node, and `MRAppMaster` per application (see [YARN Architecture Guide](../../hadoop-yarn/hadoop-yarn-site/YARN.html)).
+
+Minimally, applications specify the input/output locations and supply *map* and *reduce* functions via implementations of appropriate interfaces and/or abstract-classes. These, and other job parameters, comprise the *job configuration*.
+
+The Hadoop *job client* then submits the job (jar/executable etc.) and configuration to the `ResourceManager` which then assumes the responsibility of distributing the software/configuration to the slaves, scheduling tasks and monitoring them, providing status and diagnostic information to the job-client.
+
+Although the Hadoop framework is implemented in Java™, MapReduce applications need not be written in Java.
+
+* [Hadoop Streaming](../../api/org/apache/hadoop/streaming/package-summary.html)
+ is a utility which allows users to create and run jobs
+ with any executables (e.g. shell utilities) as the mapper and/or the
+ reducer.
+
+* [Hadoop Pipes](../../api/org/apache/hadoop/mapred/pipes/package-summary.html)
+ is a [SWIG](http://www.swig.org/)-compatible C++ API to
+ implement MapReduce applications (non JNI™ based).
+
+Inputs and Outputs
+------------------
+
+The MapReduce framework operates exclusively on `<key, value>` pairs, that is, the framework views the input to the job as a set of `<key, value>` pairs and produces a set of `<key, value>` pairs as the output of the job, conceivably of different types.
+
+The `key` and `value` classes have to be serializable by the framework and hence need to implement the [Writable](../../api/org/apache/hadoop/io/Writable.html) interface. Additionally, the key classes have to implement the [WritableComparable](../../api/org/apache/hadoop/io/WritableComparable.html) interface to facilitate sorting by the framework.
+
+Input and Output types of a MapReduce job:
+
+(input) `<k1, v1> ->` **map** `-> <k2, v2> ->` **combine** `-> <k2, v2> ->` **reduce** `-> <k3, v3>` (output)
+
+Example: WordCount v1.0
+-----------------------
+
+Before we jump into the details, lets walk through an example MapReduce application to get a flavour for how they work.
+
+`WordCount` is a simple application that counts the number of occurrences of each word in a given input set.
+
+This works with a local-standalone, pseudo-distributed or fully-distributed Hadoop installation ([Single Node Setup](../../hadoop-project-dist/hadoop-common/SingleCluster.html)).
+
+### Source Code
+
+```java
+import java.io.IOException;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+
+public class WordCount {
+
+ public static class TokenizerMapper
+ extends Mapper<Object, Text, Text, IntWritable>{
+
+ private final static IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+
+ public void map(Object key, Text value, Context context
+ ) throws IOException, InterruptedException {
+ StringTokenizer itr = new StringTokenizer(value.toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ }
+ }
+ }
+
+ public static class IntSumReducer
+ extends Reducer<Text,IntWritable,Text,IntWritable> {
+ private IntWritable result = new IntWritable();
+
+ public void reduce(Text key, Iterable<IntWritable> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ int sum = 0;
+ for (IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(key, result);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = new Configuration();
+ Job job = Job.getInstance(conf, "word count");
+ job.setJarByClass(WordCount.class);
+ job.setMapperClass(TokenizerMapper.class);
+ job.setCombinerClass(IntSumReducer.class);
+ job.setReducerClass(IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+ FileInputFormat.addInputPath(job, new Path(args[0]));
+ FileOutputFormat.setOutputPath(job, new Path(args[1]));
+ System.exit(job.waitForCompletion(true) ? 0 : 1);
+ }
+}
+```
+
+### Usage
+
+Assuming environment variables are set as follows:
+
+```bash
+export JAVA_HOME=/usr/java/default
+export PATH=${JAVA_HOME}/bin:${PATH}
+export HADOOP_CLASSPATH=${JAVA_HOME}/lib/tools.jar
+```
+
+Compile `WordCount.java` and create a jar:
+
+ $ bin/hadoop com.sun.tools.javac.Main WordCount.java
+ $ jar cf wc.jar WordCount*.class
+
+Assuming that:
+
+* `/user/joe/wordcount/input` - input directory in HDFS
+* `/user/joe/wordcount/output` - output directory in HDFS
+
+Sample text-files as input:
+
+ $ bin/hadoop fs -ls /user/joe/wordcount/input/ /user/joe/wordcount/input/file01 /user/joe/wordcount/input/file02
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/input/file01
+ Hello World Bye World
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/input/file02
+ Hello Hadoop Goodbye Hadoop
+
+Run the application:
+
+ $ bin/hadoop jar wc.jar WordCount /user/joe/wordcount/input /user/joe/wordcount/output
+
+Output:
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/output/part-r-00000`
+ Bye 1
+ Goodbye 1
+ Hadoop 2
+ Hello 2
+ World 2`
+
+Applications can specify a comma separated list of paths which would be present in the current working directory of the task using the option `-files`. The `-libjars` option allows applications to add jars to the classpaths of the maps and reduces. The option `-archives` allows them to pass comma separated list of archives as arguments. These archives are unarchived and a link with name of the archive is created in the current working directory of tasks. More details about the command line options are available at [Commands Guide](../../hadoop-project-dist/hadoop-common/CommandsManual.html).
+
+Running `wordcount` example with `-libjars`, `-files` and `-archives`:
+
+ bin/hadoop jar hadoop-mapreduce-examples-<ver>.jar wordcount -files cachefile.txt -libjars mylib.jar -archives myarchive.zip input output
+
+Here, myarchive.zip will be placed and unzipped into a directory by the name "myarchive.zip".
+
+Users can specify a different symbolic name for files and archives passed through `-files` and `-archives` option, using \#.
+
+For example,
+
+ bin/hadoop jar hadoop-mapreduce-examples-<ver>.jar wordcount -files dir1/dict.txt#dict1,dir2/dict.txt#dict2 -archives mytar.tgz#tgzdir input output
+
+Here, the files dir1/dict.txt and dir2/dict.txt can be accessed by tasks using the symbolic names dict1 and dict2 respectively. The archive mytar.tgz will be placed and unarchived into a directory by the name "tgzdir".
+
+### Walk-through
+
+The `WordCount` application is quite straight-forward.
+
+```java
+public void map(Object key, Text value, Context context
+ ) throws IOException, InterruptedException {
+ StringTokenizer itr = new StringTokenizer(value.toString());
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ }
+}
+```
+
+The `Mapper` implementation, via the `map` method, processes one line at a time, as provided by the specified `TextInputFormat`. It then splits the line into tokens separated by whitespaces, via the `StringTokenizer`, and emits a key-value pair of `< <word>, 1>`.
+
+For the given sample input the first map emits:
+
+ < Hello, 1>
+ < World, 1>
+ < Bye, 1>
+ < World, 1>
+
+The second map emits:
+
+ < Hello, 1>
+ < Hadoop, 1>
+ < Goodbye, 1>
+ < Hadoop, 1>
+
+We'll learn more about the number of maps spawned for a given job, and how to control them in a fine-grained manner, a bit later in the tutorial.
+
+ job.setCombinerClass(IntSumReducer.class);
+
+`WordCount` also specifies a `combiner`. Hence, the output of each map is passed through the local combiner (which is same as the `Reducer` as per the job configuration) for local aggregation, after being sorted on the *key*s.
+
+The output of the first map:
+
+ < Bye, 1>
+ < Hello, 1>
+ < World, 2>`
+
+The output of the second map:
+
+ < Goodbye, 1>
+ < Hadoop, 2>
+ < Hello, 1>`
+
+```java
+public void reduce(Text key, Iterable<IntWritable> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ int sum = 0;
+ for (IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(key, result);
+}
+```
+
+The `Reducer` implementation, via the `reduce` method just sums up the values, which are the occurence counts for each key (i.e. words in this example).
+
+Thus the output of the job is:
+
+ < Bye, 1>
+ < Goodbye, 1>
+ < Hadoop, 2>
+ < Hello, 2>
+ < World, 2>`
+
+The `main` method specifies various facets of the job, such as the input/output paths (passed via the command line), key/value types, input/output formats etc., in the `Job`. It then calls the `job.waitForCompletion` to submit the job and monitor its progress.
+
+We'll learn more about `Job`, `InputFormat`, `OutputFormat` and other interfaces and classes a bit later in the tutorial.
+
+MapReduce - User Interfaces
+---------------------------
+
+This section provides a reasonable amount of detail on every user-facing aspect of the MapReduce framework. This should help users implement, configure and tune their jobs in a fine-grained manner. However, please note that the javadoc for each class/interface remains the most comprehensive documentation available; this is only meant to be a tutorial.
+
+Let us first take the `Mapper` and `Reducer` interfaces. Applications typically implement them to provide the `map` and `reduce` methods.
+
+We will then discuss other core interfaces including `Job`, `Partitioner`, `InputFormat`, `OutputFormat`, and others.
+
+Finally, we will wrap up by discussing some useful features of the framework such as the `DistributedCache`, `IsolationRunner` etc.
+
+### Payload
+
+Applications typically implement the `Mapper` and `Reducer` interfaces to provide the `map` and `reduce` methods. These form the core of the job.
+
+#### Mapper
+
+[Mapper](../../api/org/apache/hadoop/mapreduce/Mapper.html) maps input key/value pairs to a set of intermediate key/value pairs.
+
+Maps are the individual tasks that transform input records into intermediate records. The transformed intermediate records do not need to be of the same type as the input records. A given input pair may map to zero or many output pairs.
+
+The Hadoop MapReduce framework spawns one map task for each `InputSplit` generated by the `InputFormat` for the job.
+
+Overall, `Mapper` implementations are passed the `Job` for the job via the [Job.setMapperClass(Class)](../../api/org/apache/hadoop/mapreduce/Job.html) method. The framework then calls [map(WritableComparable, Writable, Context)](../../api/org/apache/hadoop/mapreduce/Mapper.html) for each key/value pair in the `InputSplit` for that task. Applications can then override the `cleanup(Context)` method to perform any required cleanup.
+
+Output pairs do not need to be of the same types as input pairs. A given input pair may map to zero or many output pairs. Output pairs are collected with calls to context.write(WritableComparable, Writable).
+
+Applications can use the `Counter` to report its statistics.
+
+All intermediate values associated with a given output key are subsequently grouped by the framework, and passed to the `Reducer`(s) to determine the final output. Users can control the grouping by specifying a `Comparator` via [Job.setGroupingComparatorClass(Class)](../../api/org/apache/hadoop/mapreduce/Job.html).
+
+The `Mapper` outputs are sorted and then partitioned per `Reducer`. The total number of partitions is the same as the number of reduce tasks for the job. Users can control which keys (and hence records) go to which `Reducer` by implementing a custom `Partitioner`.
+
+Users can optionally specify a `combiner`, via [Job.setCombinerClass(Class)](../../api/org/apache/hadoop/mapreduce/Job.html), to perform local aggregation of the intermediate outputs, which helps to cut down the amount of data transferred from the `Mapper` to the `Reducer`.
+
+The intermediate, sorted outputs are always stored in a simple (key-len, key, value-len, value) format. Applications can control if, and how, the intermediate outputs are to be compressed and the [CompressionCodec](../../api/org/apache/hadoop/io/compress/CompressionCodec.html) to be used via the `Configuration`.
+
+##### How Many Maps?
+
+The number of maps is usually driven by the total size of the inputs, that is, the total number of blocks of the input files.
+
+The right level of parallelism for maps seems to be around 10-100 maps per-node, although it has been set up to 300 maps for very cpu-light map tasks. Task setup takes a while, so it is best if the maps take at least a minute to execute.
+
+Thus, if you expect 10TB of input data and have a blocksize of `128MB`, you'll end up with 82,000 maps, unless Configuration.set(`MRJobConfig.NUM_MAPS`, int) (which only provides a hint to the framework) is used to set it even higher.
+
+#### Reducer
+
+[Reducer](../../api/org/apache/hadoop/mapreduce/Reducer.html) reduces a set of intermediate values which share a key to a smaller set of values.
+
+The number of reduces for the job is set by the user via [Job.setNumReduceTasks(int)](../../api/org/apache/hadoop/mapreduce/Job.html).
+
+Overall, `Reducer` implementations are passed the `Job` for the job via the [Job.setReducerClass(Class)](../../api/org/apache/hadoop/mapreduce/Job.html) method and can override it to initialize themselves. The framework then calls [reduce(WritableComparable, Iterable\<Writable\>, Context)](../../api/org/apache/hadoop/mapreduce/Reducer.html) method for each `<key, (list of values)>` pair in the grouped inputs. Applications can then override the `cleanup(Context)` method to perform any required cleanup.
+
+`Reducer` has 3 primary phases: shuffle, sort and reduce.
+
+##### Shuffle
+
+Input to the `Reducer` is the sorted output of the mappers. In this phase the framework fetches the relevant partition of the output of all the mappers, via HTTP.
+
+##### Sort
+
+The framework groups `Reducer` inputs by keys (since different mappers may have output the same key) in this stage.
+
+The shuffle and sort phases occur simultaneously; while map-outputs are being fetched they are merged.
+
+##### Secondary Sort
+
+If equivalence rules for grouping the intermediate keys are required to be different from those for grouping keys before reduction, then one may specify a `Comparator` via [Job.setSortComparatorClass(Class)](../../api/org/apache/hadoop/mapreduce/Job.html). Since [Job.setGroupingComparatorClass(Class)](../../api/org/apache/hadoop/mapreduce/Job.html) can be used to control how intermediate keys are grouped, these can be used in conjunction to simulate *secondary sort on values*.
+
+##### Reduce
+
+In this phase the reduce(WritableComparable, Iterable\<Writable\>, Context) method is called for each `<key, (list of values)>` pair in the grouped inputs.
+
+The output of the reduce task is typically written to the [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html) via Context.write(WritableComparable, Writable).
+
+Applications can use the `Counter` to report its statistics.
+
+The output of the `Reducer` is *not sorted*.
+
+##### How Many Reduces?
+
+The right number of reduces seems to be `0.95` or `1.75` multiplied by (\<*no. of nodes*\> \* \<*no. of maximum containers per node*\>).
+
+With `0.95` all of the reduces can launch immediately and start transferring map outputs as the maps finish. With `1.75` the faster nodes will finish their first round of reduces and launch a second wave of reduces doing a much better job of load balancing.
+
+Increasing the number of reduces increases the framework overhead, but increases load balancing and lowers the cost of failures.
+
+The scaling factors above are slightly less than whole numbers to reserve a few reduce slots in the framework for speculative-tasks and failed tasks.
+
+##### Reducer NONE
+
+It is legal to set the number of reduce-tasks to *zero* if no reduction is desired.
+
+In this case the outputs of the map-tasks go directly to the `FileSystem`, into the output path set by [FileOutputFormat.setOutputPath(Job, Path)](../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html). The framework does not sort the map-outputs before writing them out to the `FileSystem`.
+
+#### Partitioner
+
+[Partitioner](../../api/org/apache/hadoop/mapreduce/Partitioner.html) partitions the key space.
+
+Partitioner controls the partitioning of the keys of the intermediate map-outputs. The key (or a subset of the key) is used to derive the partition, typically by a *hash function*. The total number of partitions is the same as the number of reduce tasks for the job. Hence this controls which of the `m` reduce tasks the intermediate key (and hence the record) is sent to for reduction.
+
+[HashPartitioner](../../api/org/apache/hadoop/mapreduce/lib/partition/HashPartitioner.html) is the default `Partitioner`.
+
+#### Counter
+
+[Counter](../../api/org/apache/hadoop/mapreduce/Counter.html) is a facility for MapReduce applications to report its statistics.
+
+`Mapper` and `Reducer` implementations can use the `Counter` to report statistics.
+
+Hadoop MapReduce comes bundled with a [library](../../api/org/apache/hadoop/mapreduce/package-summary.html) of generally useful mappers, reducers, and partitioners.
+
+### Job Configuration
+
+[Job](../../api/org/apache/hadoop/mapreduce/Job.html) represents a MapReduce job configuration.
+
+`Job` is the primary interface for a user to describe a MapReduce job to the Hadoop framework for execution. The framework tries to faithfully execute the job as described by `Job`, however:
+
+* Some configuration parameters may have been marked as final by
+ administrators
+ (see [Final Parameters](../../api/org/apache/hadoop/conf/Configuration.html#FinalParams))
+ and hence cannot be altered.
+
+* While some job parameters are straight-forward to set (e.g.
+ [Job.setNumReduceTasks(int)](../../api/org/apache/hadoop/mapreduce/Job.html))
+ , other parameters interact subtly with the
+ rest of the framework and/or job configuration and are more complex to set
+ (e.g. [Configuration.set(`JobContext.NUM_MAPS`, int)](../../api/org/apache/hadoop/conf/Configuration.html)).
+
+`Job` is typically used to specify the `Mapper`, combiner (if any), `Partitioner`, `Reducer`, `InputFormat`, `OutputFormat` implementations.
+[FileInputFormat](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html)
+indicates the set of input files
+([FileInputFormat.setInputPaths(Job, Path...)](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html)/
+[FileInputFormat.addInputPath(Job, Path)](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html)) and
+([FileInputFormat.setInputPaths(Job, String...)](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html)/
+[FileInputFormat.addInputPaths(Job, String))](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html) and where the output files should be written
+([FileOutputFormat.setOutputPath(Path)](../../api/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat.html)).
+
+Optionally, `Job` is used to specify other advanced facets of the job such as the `Comparator` to be used, files to be put in the `DistributedCache`, whether intermediate and/or job outputs are to be compressed (and how), whether job tasks can be executed in a *speculative* manner
+([setMapSpeculativeExecution(boolean)](../../api/org/apache/hadoop/mapreduce/Job.html))/
+[setReduceSpeculativeExecution(boolean)](../../api/org/apache/hadoop/mapreduce/Job.html)),
+maximum number of attempts per task
+([setMaxMapAttempts(int)](../../api/org/apache/hadoop/mapreduce/Job.html)/
+[setMaxReduceAttempts(int)](../../api/org/apache/hadoop/mapreduce/Job.html)) etc.
+
+Of course, users can use
+[Configuration.set(String, String)](../../api/org/apache/hadoop/conf/Configuration.html)/
+[Configuration.get(String)](../../api/org/apache/hadoop/conf/Configuration.html)
+to set/get arbitrary parameters needed by
+applications. However, use the `DistributedCache` for large amounts of
+(read-only) data.
+
+### Task Execution & Environment
+
+The `MRAppMaster` executes the `Mapper`/`Reducer` *task* as a child process in a separate jvm.
+
+The child-task inherits the environment of the parent `MRAppMaster`. The user can specify additional options to the child-jvm via the `mapreduce.{map|reduce}.java.opts` and configuration parameter in the `Job` such as non-standard paths for the run-time linker to search shared libraries via `-Djava.library.path=<>` etc. If the `mapreduce.{map|reduce}.java.opts` parameters contains the symbol *@taskid@* it is interpolated with value of `taskid` of the MapReduce task.
+
+Here is an example with multiple arguments and substitutions, showing jvm GC logging, and start of a passwordless JVM JMX agent so that it can connect with jconsole and the likes to watch child memory, threads and get thread dumps. It also sets the maximum heap-size of the map and reduce child jvm to 512MB & 1024MB respectively. It also adds an additional path to the `java.library.path` of the child-jvm.
+
+```xml
+<property>
+ <name>mapreduce.map.java.opts</name>
+ <value>
+ -Xmx512M -Djava.library.path=/home/mycompany/lib -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false
+ </value>
+</property>
+
+<property>
+ <name>mapreduce.reduce.java.opts</name>
+ <value>
+ -Xmx1024M -Djava.library.path=/home/mycompany/lib -verbose:gc -Xloggc:/tmp/@taskid@.gc
+ -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false
+ </value>
+</property>
+```
+
+#### Memory Management
+
+Users/admins can also specify the maximum virtual memory of the launched child-task, and any sub-process it launches recursively, using `mapreduce.{map|reduce}.memory.mb`. Note that the value set here is a per process limit. The value for `mapreduce.{map|reduce}.memory.mb` should be specified in mega bytes (MB). And also the value must be greater than or equal to the -Xmx passed to JavaVM, else the VM might not start.
+
+Note: `mapreduce.{map|reduce}.java.opts` are used only for configuring the launched child tasks from MRAppMaster. Configuring the memory options for daemons is documented in [Configuring the Environment of the Hadoop Daemons](../../hadoop-project-dist/hadoop-common/ClusterSetup.html#Configuring_Environment_of_Hadoop_Daemons).
+
+The memory available to some parts of the framework is also configurable. In map and reduce tasks, performance may be influenced by adjusting parameters influencing the concurrency of operations and the frequency with which data will hit disk. Monitoring the filesystem counters for a job- particularly relative to byte counts from the map and into the reduce- is invaluable to the tuning of these parameters.
+
+#### Map Parameters
+
+A record emitted from a map will be serialized into a buffer and metadata will be stored into accounting buffers. As described in the following options, when either the serialization buffer or the metadata exceed a threshold, the contents of the buffers will be sorted and written to disk in the background while the map continues to output records. If either buffer fills completely while the spill is in progress, the map thread will block. When the map is finished, any remaining records are written to disk and all on-disk segments are merged into a single file. Minimizing the number of spills to disk can decrease map time, but a larger buffer also decreases the memory available to the mapper.
+
+| Name | Type | Description |
+|:---- |:---- |:---- |
+| mapreduce.task.io.sort.mb | int | The cumulative size of the serialization and accounting buffers storing records emitted from the map, in megabytes. |
+| mapreduce.map.sort.spill.percent | float | The soft limit in the serialization buffer. Once reached, a thread will begin to spill the contents to disk in the background. |
+
+Other notes
+
+* If either spill threshold is exceeded while a spill is in progress,
+ collection will continue until the spill is finished. For example, if
+ `mapreduce.map.sort.spill.percent` is set to 0.33, and the remainder
+ of the buffer is filled while the spill runs, the next spill will include
+ all the collected records, or 0.66 of the buffer, and will not generate
+ additional spills. In other words, the thresholds are defining triggers,
+ not blocking.
+
+* A record larger than the serialization buffer will first trigger a spill,
+ then be spilled to a separate file. It is undefined whether or not this
+ record will first pass through the combiner.
+
+#### Shuffle/Reduce Parameters
+
+As described previously, each reduce fetches the output assigned to it by the Partitioner via HTTP into memory and periodically merges these outputs to disk. If intermediate compression of map outputs is turned on, each output is decompressed into memory. The following options affect the frequency of these merges to disk prior to the reduce and the memory allocated to map output during the reduce.
+
+| Name | Type | Description |
+|:---- |:---- |:---- |
+| mapreduce.task.io.soft.factor | int | Specifies the number of segments on disk to be merged at the same time. It limits the number of open files and compression codecs during merge. If the number of files exceeds this limit, the merge will proceed in several passes. Though this limit also applies to the map, most jobs should be configured so that hitting this limit is unlikely there. |
+| mapreduce.reduce.merge.inmem.thresholds | int | The number of sorted map outputs fetched into memory before being merged to disk. Like the spill thresholds in the preceding note, this is not defining a unit of partition, but a trigger. In practice, this is usually set very high (1000) or disabled (0), since merging in-memory segments is often less expensive than merging from disk (see notes following this table). This threshold influences only the frequency of in-memory merges during the shuffle. |
+| mapreduce.reduce.shuffle.merge.percent | float | The memory threshold for fetched map outputs before an in-memory merge is started, expressed as a percentage of memory allocated to storing map outputs in memory. Since map outputs that can't fit in memory can be stalled, setting this high may decrease parallelism between the fetch and merge. Conversely, values as high as 1.0 have been effective for reduces whose input can fit entirely in memory. This parameter influences only the frequency of in-memory merges during the shuffle. |
+| mapreduce.reduce.shuffle.input.buffer.percent | float | The percentage of memory- relative to the maximum heapsize as typically specified in `mapreduce.reduce.java.opts`- that can be allocated to storing map outputs during the shuffle. Though some memory should be set aside for the framework, in general it is advantageous to set this high enough to store large and numerous map outputs. |
+| mapreduce.reduce.input.buffer.percent | float | The percentage of memory relative to the maximum heapsize in which map outputs may be retained during the reduce. When the reduce begins, map outputs will be merged to disk until those that remain are under the resource limit this defines. By default, all map outputs are merged to disk before the reduce begins to maximize the memory available to the reduce. For less memory-intensive reduces, this should be increased to avoid trips to disk. |
+
+Other notes
+
+* If a map output is larger than 25 percent of the memory allocated to
+ copying map outputs, it will be written directly to disk without first
+ staging through memory.
+
+* When running with a combiner, the reasoning about high merge thresholds
+ and large buffers may not hold. For merges started before all map outputs
+ have been fetched, the combiner is run while spilling to disk. In some
+ cases, one can obtain better reduce times by spending resources combining
+ map outputs- making disk spills small and parallelizing spilling and
+ fetching- rather than aggressively increasing buffer sizes.
+
+* When merging in-memory map outputs to disk to begin the reduce, if an
+ intermediate merge is necessary because there are segments to spill and at
+ least `mapreduce.task.io.sort.factor` segments already on disk, the
+ in-memory map outputs will be part of the intermediate merge.
+
+#### Configured Parameters
+
+The following properties are localized in the job configuration for each task's execution:
+
+| Name | Type | Description |
+|:---- |:---- |:---- |
+| mapreduce.job.id | String | The job id |
+| mapreduce.job.jar | String | job.jar location in job directory |
+| mapreduce.job.local.dir | String | The job specific shared scratch space |
+| mapreduce.task.id | String | The task id |
+| mapreduce.task.attempt.id | String | The task attempt id |
+| mapreduce.task.is.map | boolean | Is this a map task |
+| mapreduce.task.partition | int | The id of the task within the job |
+| mapreduce.map.input.file | String | The filename that the map is reading from |
+| mapreduce.map.input.start | long | The offset of the start of the map input split |
+| mapreduce.map.input.length | long | The number of bytes in the map input split |
+| mapreduce.task.output.dir | String | The task's temporary output directory |
+
+**Note:** During the execution of a streaming job, the names of the "mapreduce" parameters are transformed. The dots ( . ) become underscores ( \_ ). For example, mapreduce.job.id becomes mapreduce\_job\_id and mapreduce.job.jar becomes mapreduce\_job\_jar. To get the values in a streaming job's mapper/reducer use the parameter names with the underscores.
+
+#### Task Logs
+
+The standard output (stdout) and error (stderr) streams and the syslog of the task are read by the NodeManager and logged to `${HADOOP_LOG_DIR}/userlogs`.
+
+#### Distributing Libraries
+
+The [DistributedCache](#DistributedCache) can also be used to distribute both jars and native
+libraries for use in the map and/or reduce tasks. The child-jvm always has
+its *current working directory* added to the `java.library.path` and
+`LD_LIBRARY_PATH`. And hence the cached libraries can be loaded via
+[System.loadLibrary](http://docs.oracle.com/javase/7/docs/api/java/lang/System.html) or
+[System.load](http://docs.oracle.com/javase/7/docs/api/java/lang/System.html).
+More details on how to load shared libraries through distributed cache are documented at
+[Native Libraries](../../hadoop-project-dist/hadoop-common/NativeLibraries.html#Native_Shared_Libraries).
+
+### Job Submission and Monitoring
+
+[Job](../../api/org/apache/hadoop/mapreduce/Job.html) is the primary interface by which user-job interacts with the `ResourceManager`.
+
+`Job` provides facilities to submit jobs, track their progress, access component-tasks' reports and logs, get the MapReduce cluster's status information and so on.
+
+The job submission process involves:
+
+1. Checking the input and output specifications of the job.
+
+2. Computing the `InputSplit` values for the job.
+
+3. Setting up the requisite accounting information for the
+ `DistributedCache` of the job, if necessary.
+
+4. Copying the job's jar and configuration to the MapReduce system
+ directory on the `FileSystem`.
+
+5. Submitting the job to the `ResourceManager` and optionally
+ monitoring it's status.
+
+Job history files are also logged to user specified directory `mapreduce.jobhistory.intermediate-done-dir` and `mapreduce.jobhistory.done-dir`, which defaults to job output directory.
+
+User can view the history logs summary in specified directory using the following command `$ mapred job -history output.jhist` This command will print job details, failed and killed tip details. More details about the job such as successful tasks and task attempts made for each task can be viewed using the following command `$ mapred job -history all output.jhist`
+
+Normally the user uses `Job` to create the application, describe various facets of the job, submit the job, and monitor its progress.
+
+#### Job Control
+
+Users may need to chain MapReduce jobs to accomplish complex tasks which cannot be done via a single MapReduce job. This is fairly easy since the output of the job typically goes to distributed file-system, and the output, in turn, can be used as the input for the next job.
+
+However, this also means that the onus on ensuring jobs are complete (success/failure) lies squarely on the clients. In such cases, the various job-control options are:
+
+* [Job.submit()](../../api/org/apache/hadoop/mapreduce/Job.html) :
+ Submit the job to the cluster and return immediately.
+
+* [Job.waitForCompletion(boolean)](../../api/org/apache/hadoop/mapreduce/Job.html) :
+ Submit the job to the cluster and wait for it to finish.
+
+### Job Input
+
+[InputFormat](../../api/org/apache/hadoop/mapreduce/InputFormat.html) describes the input-specification for a MapReduce job.
+
+The MapReduce framework relies on the `InputFormat` of the job to:
+
+1. Validate the input-specification of the job.
+
+2. Split-up the input file(s) into logical `InputSplit` instances,
+ each of which is then assigned to an individual `Mapper`.
+
+3. Provide the `RecordReader` implementation used to glean input
+ records from the logical `InputSplit` for processing by the
+ `Mapper`.
+
+The default behavior of file-based `InputFormat` implementations,
+typically sub-classes of
+[FileInputFormat](../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html),
+is to split the input into *logical* `InputSplit`
+instances based on the total size, in bytes, of the input files. However, the
+`FileSystem` blocksize of the input files is treated as an upper bound
+for input splits. A lower bound on the split size can be set via
+`mapreduce.input.fileinputformat.split.minsize`.
+
+Clearly, logical splits based on input-size is insufficient for many applications since record boundaries must be respected. In such cases, the application should implement a `RecordReader`, who is responsible for respecting record-boundaries and presents a record-oriented view of the logical `InputSplit` to the individual task.
+
+[TextInputFormat](../../api/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.html)
+is the default `InputFormat`.
+
+If `TextInputFormat` is the `InputFormat` for a given job, the framework detects input-files with the *.gz* extensions and automatically decompresses them using the appropriate `CompressionCodec`. However, it must be noted that compressed files with the above extensions cannot be *split* and each compressed file is processed in its entirety by a single mapper.
+
+#### InputSplit
+
+[InputSplit](../../api/org/apache/hadoop/mapreduce/InputSplit.html)
+represents the data to be processed by an individual `Mapper`.
+
+Typically `InputSplit` presents a byte-oriented view of the input, and it is the responsibility of `RecordReader` to process and present a record-oriented view.
+
+[FileSplit](../../api/org/apache/hadoop/mapreduce/lib/input/FileSplit.html)
+is the default `InputSplit`. It sets `mapreduce.map.input.file` to
+the path of the input file for the logical split.
+
+#### RecordReader
+
+[RecordReader](../../api/org/apache/hadoop/mapreduce/RecordReader.html) reads `<key, value>` pairs from an `InputSplit`.
+
+Typically the `RecordReader` converts the byte-oriented view of the input, provided by the `InputSplit`, and presents a record-oriented to the `Mapper` implementations for processing. `RecordReader` thus assumes the responsibility of processing record boundaries and presents the tasks with keys and values.
+
+### Job Output
+
+[OutputFormat](../../api/org/apache/hadoop/mapreduce/OutputFormat.html)
+describes the output-specification for a MapReduce job.
+
+The MapReduce framework relies on the `OutputFormat` of the job to:
+
+1. Validate the output-specification of the job; for example, check that
+ the output directory doesn't already exist.
+
+2. Provide the `RecordWriter` implementation used to write the output
+ files of the job. Output files are stored in a `FileSystem`.
+
+`TextOutputFormat` is the default `OutputFormat`.
+
+#### OutputCommitter
+
+[OutputCommitter](../../api/org/apache/hadoop/mapreduce/OutputCommitter.html)
+describes the commit of task output for a MapReduce job.
+
+The MapReduce framework relies on the `OutputCommitter` of the job to:
+
+1. Setup the job during initialization. For example, create the temporary
+ output directory for the job during the initialization of the job. Job
+ setup is done by a separate task when the job is in PREP state and
+ after initializing tasks. Once the setup task completes, the job will
+ be moved to RUNNING state.
+
+2. Cleanup the job after the job completion. For example, remove the
+ temporary output directory after the job completion. Job cleanup is
+ done by a separate task at the end of the job. Job is declared
+ SUCCEDED/FAILED/KILLED after the cleanup task completes.
+
+3. Setup the task temporary output. Task setup is done as part of the
+ same task, during task initialization.
+
+4. Check whether a task needs a commit. This is to avoid the commit
+ procedure if a task does not need commit.
+
+5. Commit of the task output. Once task is done, the task will commit
+ it's output if required.
+
+6. Discard the task commit. If the task has been failed/killed, the
+ output will be cleaned-up. If task could not cleanup (in exception
+ block), a separate task will be launched with same attempt-id to do
+ the cleanup.
+
+`FileOutputCommitter` is the default `OutputCommitter`. Job setup/cleanup tasks occupy map or reduce containers, whichever is available on the NodeManager. And JobCleanup task, TaskCleanup tasks and JobSetup task have the highest priority, and in that order.
+
+#### Task Side-Effect Files
+
+In some applications, component tasks need to create and/or write to side-files, which differ from the actual job-output files.
+
+In such cases there could be issues with two instances of the same `Mapper` or `Reducer` running simultaneously (for example, speculative tasks) trying to open and/or write to the same file (path) on the `FileSystem`. Hence the application-writer will have to pick unique names per task-attempt (using the attemptid, say `attempt_200709221812_0001_m_000000_0`), not just per task.
+
+To avoid these issues the MapReduce framework, when the `OutputCommitter` is `FileOutputCommitter`, maintains a special `${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}` sub-directory accessible via `${mapreduce.task.output.dir}` for each task-attempt on the `FileSystem` where the output of the task-attempt is stored. On successful completion of the task-attempt, the files in the `${mapreduce.output.fileoutputformat.outputdir}/_temporary/_${taskid}` (only) are *promoted* to `${mapreduce.output.fileoutputformat.outputdir}`. Of course, the framework discards the sub-directory of unsuccessful task-attempts. This process is completely transparent to the application.
+
+The application-writer can take advantage of this feature by creating any side-files required in `${mapreduce.task.output.dir}` during execution of a task via
+[FileOutputFormat.getWorkOutputPath(Conext)](../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html),
+and the framework will promote them similarly for succesful task-attempts, thus eliminating the need to pick unique paths per task-attempt.
+
+Note: The value of `${mapreduce.task.output.dir}` during execution of a particular task-attempt is actually `${mapreduce.output.fileoutputformat.outputdir}/_temporary/_{$taskid}`, and this value is set by the MapReduce framework. So, just create any side-files in the path returned by
+[FileOutputFormat.getWorkOutputPath(Conext)](../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html)
+from MapReduce task to take advantage of this feature.
+
+The entire discussion holds true for maps of jobs with reducer=NONE (i.e. 0 reduces) since output of the map, in that case, goes directly to HDFS.
+
+#### RecordWriter
+
+[RecordWriter](../../api/org/apache/hadoop/mapreduce/RecordWriter.html)
+writes the output `<key, value>` pairs to an output file.
+
+RecordWriter implementations write the job outputs to the `FileSystem`.
+
+### Other Useful Features
+
+#### Submitting Jobs to Queues
+
+Users submit jobs to Queues. Queues, as collection of jobs, allow the system to provide specific functionality. For example, queues use ACLs to control which users who can submit jobs to them. Queues are expected to be primarily used by Hadoop Schedulers.
+
+Hadoop comes configured with a single mandatory queue, called 'default'. Queue names are defined in the `mapreduce.job.queuename`\> property of the Hadoop site configuration. Some job schedulers, such as the
+[Capacity Scheduler](../../hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html),
+support multiple queues.
+
+A job defines the queue it needs to be submitted to through the `mapreduce.job.queuename` property, or through the Configuration.set(`MRJobConfig.QUEUE_NAME`, String) API. Setting the queue name is optional. If a job is submitted without an associated queue name, it is submitted to the 'default' queue.
+
+#### Counters
+
+`Counters` represent global counters, defined either by the MapReduce framework or applications. Each `Counter` can be of any `Enum` type. Counters of a particular `Enum` are bunched into groups of type `Counters.Group`.
+
+Applications can define arbitrary `Counters` (of type `Enum`) and update them via
+[Counters.incrCounter(Enum, long)](../../api/org/apache/hadoop/mapred/Counters.html)
+or Counters.incrCounter(String, String, long) in the `map` and/or `reduce` methods. These counters are then globally aggregated by the framework.
+
+#### DistributedCache
+
+`DistributedCache` distributes application-specific, large, read-only files efficiently.
+
+`DistributedCache` is a facility provided by the MapReduce framework to cache files (text, archives, jars and so on) needed by applications.
+
+Applications specify the files to be cached via urls (hdfs://) in the `Job`. The `DistributedCache` assumes that the files specified via hdfs:// urls are already present on the `FileSystem`.
+
+The framework will copy the necessary files to the slave node before any tasks for the job are executed on that node. Its efficiency stems from the fact that the files are only copied once per job and the ability to cache archives which are un-archived on the slaves.
+
+`DistributedCache` tracks the modification timestamps of the cached files. Clearly the cache files should not be modified by the application or externally while the job is executing.
+
+`DistributedCache` can be used to distribute simple, read-only data/text files and more complex types such as archives and jars. Archives (zip, tar, tgz and tar.gz files) are *un-archived* at the slave nodes. Files have *execution permissions* set.
+
+The files/archives can be distributed by setting the property `mapreduce.job.cache.{files |archives}`. If more than one file/archive has to be distributed, they can be added as comma separated paths. The properties can also be set by APIs
+[Job.addCacheFile(URI)](../../api/org/apache/hadoop/mapreduce/Job.html)/
+[Job.addCacheArchive(URI)](../../api/org/apache/hadoop/mapreduce/Job.html)
+and
+[Job.setCacheFiles(URI[])](../../api/org/apache/hadoop/mapreduce/Job.html)/
+[Job.setCacheArchives(URI[])](../../api/org/apache/hadoop/mapreduce/Job.html)
+where URI is of the form `hdfs://host:port/absolute-path\#link-name`. In Streaming, the files can be distributed through command line option `-cacheFile/-cacheArchive`.
+
+The `DistributedCache` can also be used as a rudimentary software distribution mechanism for use in the map and/or reduce tasks. It can be used to distribute both jars and native libraries. The
+[Job.addArchiveToClassPath(Path)](../../api/org/apache/hadoop/mapreduce/Job.html) or
+[Job.addFileToClassPath(Path)](../../api/org/apache/hadoop/mapreduce/Job.html)
+api can be used to cache files/jars and also add them to the *classpath* of child-jvm. The same can be done by setting the configuration properties `mapreduce.job.classpath.{files |archives}`. Similarly the cached files that are symlinked into the working directory of the task can be used to distribute native libraries and load them.
+
+##### Private and Public DistributedCache Files
+
+DistributedCache files can be private or public, that determines how they can be shared on the slave nodes.
+
+* "Private" DistributedCache files are cached in a localdirectory private to
+ the user whose jobs need these files. These files are shared by all tasks
+ and jobs of the specific user only and cannot be accessed by jobs of
+ other users on the slaves. A DistributedCache file becomes private by
+ virtue of its permissions on the file system where the files are
+ uploaded, typically HDFS. If the file has no world readable access, or if
+ the directory path leading to the file has no world executable access for
+ lookup, then the file becomes private.
+
+* "Public" DistributedCache files are cached in a global directory and the
+ file access is setup such that they are publicly visible to all users.
+ These files can be shared by tasks and jobs of all users on the slaves. A
+ DistributedCache file becomes public by virtue of its permissions on the
+ file system where the files are uploaded, typically HDFS. If the file has
+ world readable access, AND if the directory path leading to the file has
+ world executable access for lookup, then the file becomes public. In other
+ words, if the user intends to make a file publicly available to all users,
+ the file permissions must be set to be world readable, and the directory
+ permissions on the path leading to the file must be world executable.
+
+#### Profiling
+
+Profiling is a utility to get a representative (2 or 3) sample of built-in java profiler for a sample of maps and reduces.
+
+User can specify whether the system should collect profiler information for some of the tasks in the job by setting the configuration property `mapreduce.task.profile`. The value can be set using the api Configuration.set(`MRJobConfig.TASK_PROFILE`, boolean). If the value is set `true`, the task profiling is enabled. The profiler information is stored in the user log directory. By default, profiling is not enabled for the job.
+
+Once user configures that profiling is needed, she/he can use the configuration property `mapreduce.task.profile.{maps|reduces}` to set the ranges of MapReduce tasks to profile. The value can be set using the api Configuration.set(`MRJobConfig.NUM_{MAP|REDUCE}_PROFILES`, String). By default, the specified range is `0-2`.
+
+User can also specify the profiler configuration arguments by setting the configuration property `mapreduce.task.profile.params`. The value can be specified using the api Configuration.set(`MRJobConfig.TASK_PROFILE_PARAMS`, String). If the string contains a `%s`, it will be replaced with the name of the profiling output file when the task runs. These parameters are passed to the task child JVM on the command line. The default value for the profiling parameters is `-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s`.
+
+#### Debugging
+
+The MapReduce framework provides a facility to run user-provided scripts for debugging. When a MapReduce task fails, a user can run a debug script, to process task logs for example. The script is given access to the task's stdout and stderr outputs, syslog and jobconf. The output from the debug script's stdout and stderr is displayed on the console diagnostics and also as part of the job UI.
+
+In the following sections we discuss how to submit a debug script with a job. The script file needs to be distributed and submitted to the framework.
+
+##### How to distribute the script file:
+
+The user needs to use [DistributedCache](#DistributedCache) to *distribute* and *symlink* thescript file.
+
+##### How to submit the script:
+
+A quick way to submit the debug script is to set values for the properties `mapreduce.map.debug.script` and `mapreduce.reduce.debug.script`, for debugging map and reduce tasks respectively. These properties can also be set by using APIs
+[Configuration.set(`MRJobConfig.MAP_DEBUG_SCRIPT`, String)](../../api/org/apache/hadoop/conf/Configuration.html) and
+[Configuration.set(`MRJobConfig.REDUCE_DEBUG_SCRIPT`, String)](../../api/org/apache/hadoop/conf/Configuration.html).
+In streaming mode, a debug script can be submitted with the command-line options `-mapdebug` and `-reducedebug`, for debugging map and reduce tasks respectively.
+
+The arguments to the script are the task's stdout, stderr, syslog and jobconf files. The debug command, run on the node where the MapReduce task failed, is:<br/>
+`$script $stdout $stderr $syslog $jobconf`
+
+Pipes programs have the c++ program name as a fifth argument for the command. Thus for the pipes programs the command is<br/>
+`$script $stdout $stderr $syslog $jobconf $program`
+
+##### Default Behavior:
+
+For pipes, a default script is run to process core dumps under gdb, prints stack trace and gives info about running threads.
+
+#### Data Compression
+
+Hadoop MapReduce provides facilities for the application-writer to specify compression for both intermediate map-outputs and the job-outputs i.e. output of the reduces. It also comes bundled with
+[CompressionCodec](../../api/org/apache/hadoop/io/compress/CompressionCodec.html)
+implementation for the
+[zlib](http://www.zlib.net) compression algorithm. The
+[gzip](http://www.gzip.org),
+[bzip2](http://www.bzip.org),
+[snappy](http://code.google.com/p/snappy/), and
+[lz4](http://code.google.com/p/lz4/) file format are also supported.
+
+Hadoop also provides native implementations of the above compression codecs for reasons of both performance (zlib) and non-availability of Java libraries. More details on their usage and availability are available
+[here](../../hadoop-project-dist/hadoop-common/NativeLibraries.html).
+
+##### Intermediate Outputs
+
+Applications can control compression of intermediate map-outputs via the Configuration.set(`MRJobConfig.MAP_OUTPUT_COMPRESS`, boolean) api and the `CompressionCodec` to be used via the Configuration.set(`MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC`, Class) api.
+
+##### Job Outputs
+
+Applications can control compression of job-outputs via the
+[FileOutputFormat.setCompressOutput(Job, boolean)](../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html)
+api and the `CompressionCodec` to be used can be specified via the FileOutputFormat.setOutputCompressorClass(Job, Class) api.
+
+If the job outputs are to be stored in the
+[SequenceFileOutputFormat](../../api/org/apache/hadoop/mapreduce/lib/output/SequenceFileOutputFormat.html),
+the required `SequenceFile.CompressionType` (i.e. `RECORD` / `BLOCK` - defaults to `RECORD`) can be specified via the SequenceFileOutputFormat.setOutputCompressionType(Job, SequenceFile.CompressionType) api.
+
+#### Skipping Bad Records
+
+Hadoop provides an option where a certain set of bad input records can be skipped when processing map inputs. Applications can control this feature through the
+[SkipBadRecords](../../api/org/apache/hadoop/mapred/SkipBadRecords.html) class.
+
+This feature can be used when map tasks crash deterministically on certain input. This usually happens due to bugs in the map function. Usually, the user would have to fix these bugs. This is, however, not possible sometimes. The bug may be in third party libraries, for example, for which the source code is not available. In such cases, the task never completes successfully even after multiple attempts, and the job fails. With this feature, only a small portion of data surrounding the bad records is lost, which may be acceptable for some applications (those performing statistical analysis on very large data, for example).
+
+By default this feature is disabled. For enabling it, refer to
+[SkipBadRecords.setMapperMaxSkipRecords(Configuration, long)](../../api/org/apache/hadoop/mapred/SkipBadRecords.html) and
+[SkipBadRecords.setReducerMaxSkipGroups(Configuration, long)](../../api/org/apache/hadoop/mapred/SkipBadRecords.html).
+
+With this feature enabled, the framework gets into 'skipping mode' after a certain number of map failures. For more details, see
+[SkipBadRecords.setAttemptsToStartSkipping(Configuration, int)](../../api/org/apache/hadoop/mapred/SkipBadRecords.html).
+In 'skipping mode', map tasks maintain the range of records being processed. To do this, the framework relies on the processed record counter. See
+[SkipBadRecords.COUNTER\_MAP\_PROCESSED\_RECORDS](../../api/org/apache/hadoop/mapred/SkipBadRecords.html) and
+[SkipBadRecords.COUNTER\_REDUCE\_PROCESSED\_GROUPS](../../api/org/apache/hadoop/mapred/SkipBadRecords.html).
+This counter enables the framework to know how many records have been processed successfully, and hence, what record range caused a task to crash. On further attempts, this range of records is skipped.
+
+The number of records skipped depends on how frequently the processed record counter is incremented by the application. It is recommended that this counter be incremented after every record is processed. This may not be possible in some applications that typically batch their processing. In such cases, the framework may skip additional records surrounding the bad record. Users can control the number of skipped records through
+[SkipBadRecords.setMapperMaxSkipRecords(Configuration, long)](../../api/org/apache/hadoop/mapred/SkipBadRecords.html) and
+[SkipBadRecords.setReducerMaxSkipGroups(Configuration, long)](../../api/org/apache/hadoop/mapred/SkipBadRecords.html).
+The framework tries to narrow the range of skipped records using a binary search-like approach. The skipped range is divided into two halves and only one half gets executed. On subsequent failures, the framework figures out which half contains bad records. A task will be re-executed till the acceptable skipped value is met or all task attempts are exhausted. To increase the number of task attempts, use
+[Job.setMaxMapAttempts(int)](../../api/org/apache/hadoop/mapreduce/Job.html) and
+[Job.setMaxReduceAttempts(int)](../../api/org/apache/hadoop/mapreduce/Job.html)
+
+Skipped records are written to HDFS in the sequence file format, for later analysis. The location can be changed through
+[SkipBadRecords.setSkipOutputPath(JobConf, Path)](../../api/org/apache/hadoop/mapred/SkipBadRecords.html).
+
+### Example: WordCount v2.0
+
+Here is a more complete `WordCount` which uses many of the features provided by the MapReduce framework we discussed so far.
+
+This needs the HDFS to be up and running, especially for the `DistributedCache`-related features. Hence it only works with a
+[pseudo-distributed](../../hadoop-project-dist/hadoop-common/SingleCluster.html) or
+[fully-distributed](../../hadoop-project-dist/hadoop-common/ClusterSetup.html) Hadoop installation.
+
+#### Source Code
+
+```java
+import java.io.BufferedReader;
+import java.io.FileReader;
+import java.io.IOException;
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.StringTokenizer;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.IntWritable;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.Mapper;
+import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.Counter;
+import org.apache.hadoop.util.GenericOptionsParser;
+import org.apache.hadoop.util.StringUtils;
+
+public class WordCount2 {
+
+ public static class TokenizerMapper
+ extends Mapper<Object, Text, Text, IntWritable>{
+
+ static enum CountersEnum { INPUT_WORDS }
+
+ private final static IntWritable one = new IntWritable(1);
+ private Text word = new Text();
+
+ private boolean caseSensitive;
+ private Set<String> patternsToSkip = new HashSet<String>();
+
+ private Configuration conf;
+ private BufferedReader fis;
+
+ @Override
+ public void setup(Context context) throws IOException,
+ InterruptedException {
+ conf = context.getConfiguration();
+ caseSensitive = conf.getBoolean("wordcount.case.sensitive", true);
+ if (conf.getBoolean("wordcount.skip.patterns", true)) {
+ URI[] patternsURIs = Job.getInstance(conf).getCacheFiles();
+ for (URI patternsURI : patternsURIs) {
+ Path patternsPath = new Path(patternsURI.getPath());
+ String patternsFileName = patternsPath.getName().toString();
+ parseSkipFile(patternsFileName);
+ }
+ }
+ }
+
+ private void parseSkipFile(String fileName) {
+ try {
+ fis = new BufferedReader(new FileReader(fileName));
+ String pattern = null;
+ while ((pattern = fis.readLine()) != null) {
+ patternsToSkip.add(pattern);
+ }
+ } catch (IOException ioe) {
+ System.err.println("Caught exception while parsing the cached file '"
+ + StringUtils.stringifyException(ioe));
+ }
+ }
+
+ @Override
+ public void map(Object key, Text value, Context context
+ ) throws IOException, InterruptedException {
+ String line = (caseSensitive) ?
+ value.toString() : value.toString().toLowerCase();
+ for (String pattern : patternsToSkip) {
+ line = line.replaceAll(pattern, "");
+ }
+ StringTokenizer itr = new StringTokenizer(line);
+ while (itr.hasMoreTokens()) {
+ word.set(itr.nextToken());
+ context.write(word, one);
+ Counter counter = context.getCounter(CountersEnum.class.getName(),
+ CountersEnum.INPUT_WORDS.toString());
+ counter.increment(1);
+ }
+ }
+ }
+
+ public static class IntSumReducer
+ extends Reducer<Text,IntWritable,Text,IntWritable> {
+ private IntWritable result = new IntWritable();
+
+ public void reduce(Text key, Iterable<IntWritable> values,
+ Context context
+ ) throws IOException, InterruptedException {
+ int sum = 0;
+ for (IntWritable val : values) {
+ sum += val.get();
+ }
+ result.set(sum);
+ context.write(key, result);
+ }
+ }
+
+ public static void main(String[] args) throws Exception {
+ Configuration conf = new Configuration();
+ GenericOptionsParser optionParser = new GenericOptionsParser(conf, args);
+ String[] remainingArgs = optionParser.getRemainingArgs();
+ if (!(remainingArgs.length != 2 | | remainingArgs.length != 4)) {
+ System.err.println("Usage: wordcount <in> <out> [-skip skipPatternFile]");
+ System.exit(2);
+ }
+ Job job = Job.getInstance(conf, "word count");
+ job.setJarByClass(WordCount2.class);
+ job.setMapperClass(TokenizerMapper.class);
+ job.setCombinerClass(IntSumReducer.class);
+ job.setReducerClass(IntSumReducer.class);
+ job.setOutputKeyClass(Text.class);
+ job.setOutputValueClass(IntWritable.class);
+
+ List<String> otherArgs = new ArrayList<String>();
+ for (int i=0; i < remainingArgs.length; ++i) {
+ if ("-skip".equals(remainingArgs[i])) {
+ job.addCacheFile(new Path(remainingArgs[++i]).toUri());
+ job.getConfiguration().setBoolean("wordcount.skip.patterns", true);
+ } else {
+ otherArgs.add(remainingArgs[i]);
+ }
+ }
+ FileInputFormat.addInputPath(job, new Path(otherArgs.get(0)));
+ FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1)));
+
+ System.exit(job.waitForCompletion(true) ? 0 : 1);
+ }
+}
+```
+
+#### Sample Runs
+
+Sample text-files as input:
+
+ $ bin/hadoop fs -ls /user/joe/wordcount/input/
+ /user/joe/wordcount/input/file01
+ /user/joe/wordcount/input/file02
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/input/file01
+ Hello World, Bye World!
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/input/file02
+ Hello Hadoop, Goodbye to hadoop.
+
+Run the application:
+
+ $ bin/hadoop jar wc.jar WordCount2 /user/joe/wordcount/input /user/joe/wordcount/output
+
+Output:
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/output/part-r-00000
+ Bye 1
+ Goodbye 1
+ Hadoop, 1
+ Hello 2
+ World! 1
+ World, 1
+ hadoop. 1
+ to 1
+
+Notice that the inputs differ from the first version we looked at, and how they affect the outputs.
+
+Now, lets plug-in a pattern-file which lists the word-patterns to be ignored, via the `DistributedCache`.
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/patterns.txt
+ \.
+ \,
+ \!
+ to
+
+Run it again, this time with more options:
+
+ $ bin/hadoop jar wc.jar WordCount2 -Dwordcount.case.sensitive=true /user/joe/wordcount/input /user/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt
+
+As expected, the output:
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/output/part-r-00000
+ Bye 1
+ Goodbye 1
+ Hadoop 1
+ Hello 2
+ World 2
+ hadoop 1
+
+Run it once more, this time switch-off case-sensitivity:
+
+ $ bin/hadoop jar wc.jar WordCount2 -Dwordcount.case.sensitive=false /user/joe/wordcount/input /user/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt
+
+Sure enough, the output:
+
+ $ bin/hadoop fs -cat /user/joe/wordcount/output/part-r-00000
+ bye 1
+ goodbye 1
+ hadoop 2
+ hello 2
+ horld 2
+
+#### Highlights
+
+The second version of `WordCount` improves upon the previous one by using some features offered by the MapReduce framework:
+
+* Demonstrates how applications can access configuration parameters in the
+ `setup` method of the `Mapper` (and `Reducer`)
+ implementations.
+
+* Demonstrates how the `DistributedCache` can be used to distribute
+ read-only data needed by the jobs. Here it allows the user to specify
+ word-patterns to skip while counting.
+
+* Demonstrates the utility of the `GenericOptionsParser` to handle
+ generic Hadoop command-line options.
+
+* Demonstrates how applications can use `Counters` and how they can set
+ application-specific status information passed to the `map` (and
+ `reduce`) method.
+
+*Java and JNI are trademarks or registered trademarks of Oracle America, Inc. in the United States and other countries.*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduce_Compatibility_Hadoop1_Hadoop2.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduce_Compatibility_Hadoop1_Hadoop2.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduce_Compatibility_Hadoop1_Hadoop2.md
new file mode 100644
index 0000000..11caab7
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapReduce_Compatibility_Hadoop1_Hadoop2.md
@@ -0,0 +1,69 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Apache Hadoop MapReduce - Migrating from Apache Hadoop 1.x to Apache Hadoop 2.x
+===============================================================================
+
+Introduction
+------------
+
+This document provides information for users to migrate their Apache Hadoop MapReduce applications from Apache Hadoop 1.x to Apache Hadoop 2.x.
+
+In Apache Hadoop 2.x we have spun off resource management capabilities into Apache Hadoop YARN, a general purpose, distributed application management framework while Apache Hadoop MapReduce (aka MRv2) remains as a pure distributed computation framework.
+
+In general, the previous MapReduce runtime (aka MRv1) has been reused and no major surgery has been conducted on it. Therefore, MRv2 is able to ensure satisfactory compatibility with MRv1 applications. However, due to some improvements and code refactorings, a few APIs have been rendered backward-incompatible.
+
+The remainder of this page will discuss the scope and the level of backward compatibility that we support in Apache Hadoop MapReduce 2.x (MRv2).
+
+Binary Compatibility
+--------------------
+
+First, we ensure binary compatibility to the applications that use old **mapred** APIs. This means that applications which were built against MRv1 **mapred** APIs can run directly on YARN without recompilation, merely by pointing them to an Apache Hadoop 2.x cluster via configuration.
+
+Source Compatibility
+--------------------
+
+We cannot ensure complete binary compatibility with the applications that use **mapreduce** APIs, as these APIs have evolved a lot since MRv1. However, we ensure source compatibility for **mapreduce** APIs that break binary compatibility. In other words, users should recompile their applications that use **mapreduce** APIs against MRv2 jars. One notable binary incompatibility break is Counter and CounterGroup.
+
+Not Supported
+-------------
+
+MRAdmin has been removed in MRv2 because because `mradmin` commands no longer exist. They have been replaced by the commands in `rmadmin`. We neither support binary compatibility nor source compatibility for the applications that use this class directly.
+
+Tradeoffs between MRv1 Users and Early MRv2 Adopters
+----------------------------------------------------
+
+Unfortunately, maintaining binary compatibility for MRv1 applications may lead to binary incompatibility issues for early MRv2 adopters, in particular Hadoop 0.23 users. For **mapred** APIs, we have chosen to be compatible with MRv1 applications, which have a larger user base. For **mapreduce** APIs, if they don't significantly break Hadoop 0.23 applications, we still change them to be compatible with MRv1 applications. Below is the list of MapReduce APIs which are incompatible with Hadoop 0.23.
+
+| **Problematic Function** | **Incompatibility Issue** |
+|:---- |:---- |
+| `org.apache.hadoop.util.ProgramDriver#drive` | Return type changes from `void` to `int` |
+| `org.apache.hadoop.mapred.jobcontrol.Job#getMapredJobID` | Return type changes from `String` to `JobID` |
+| `org.apache.hadoop.mapred.TaskReport#getTaskId` | Return type changes from `String` to `TaskID` |
+| `org.apache.hadoop.mapred.ClusterStatus#UNINITIALIZED_MEMORY_VALUE` | Data type changes from `long` to `int` |
+| `org.apache.hadoop.mapreduce.filecache.DistributedCache#getArchiveTimestamps` | Return type changes from `long[]` to `String[]` |
+| `org.apache.hadoop.mapreduce.filecache.DistributedCache#getFileTimestamps` | Return type changes from `long[]` to `String[]` |
+| `org.apache.hadoop.mapreduce.Job#failTask` | Return type changes from `void` to `boolean` |
+| `org.apache.hadoop.mapreduce.Job#killTask` | Return type changes from `void` to `boolean` |
+| `org.apache.hadoop.mapreduce.Job#getTaskCompletionEvents` | Return type changes from `o.a.h.mapred.TaskCompletionEvent[]` to `o.a.h.mapreduce.TaskCompletionEvent[]` |
+
+Malicious
+---------
+
+For the users who are going to try `hadoop-examples-1.x.x.jar` on YARN, please note that `hadoop -jar hadoop-examples-1.x.x.jar` will still use `hadoop-mapreduce-examples-2.x.x.jar`, which is installed together with other MRv2 jars. By default Hadoop framework jars appear before the users' jars in the classpath, such that the classes from the 2.x.x jar will still be picked. Users should remove `hadoop-mapreduce-examples-2.x.x.jar` from the classpath of all the nodes in a cluster. Otherwise, users need to set `HADOOP_USER_CLASSPATH_FIRST=true` and `HADOOP_CLASSPATH=...:hadoop-examples-1.x.x.jar` to run their target examples jar, and add the following configuration in `mapred-site.xml` to make the processes in YARN containers pick this jar as well.
+
+ <property>
+ <name>mapreduce.job.user.classpath.first</name>
+ <value>true</value>
+ </property>
[14/50] [abbrv] hadoop git commit: HADOOP-11000. HAServiceProtocol's
health state is incorrectly transitioned to SERVICE_NOT_RESPONDING
(Contributed by Ming Ma)
Posted by zj...@apache.org.
HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned to SERVICE_NOT_RESPONDING (Contributed by Ming Ma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cf4b7f50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cf4b7f50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cf4b7f50
Branch: refs/heads/YARN-2928
Commit: cf4b7f506dd338ecf2ed4c643b6a6a334e070fca
Parents: 500e6a0
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Feb 17 14:55:56 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Feb 17 14:55:56 2015 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../org/apache/hadoop/ha/HealthMonitor.java | 35 ++++++----
.../org/apache/hadoop/ha/DummyHAService.java | 73 +++++++++++++++++---
.../org/apache/hadoop/ha/TestHealthMonitor.java | 4 +-
4 files changed, 94 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 522ec47..51305bb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -934,6 +934,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11467. KerberosAuthenticator can connect to a non-secure cluster.
(yzhangal via rkanter)
+ HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned
+ to SERVICE_NOT_RESPONDING (Ming Ma via vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
index 0d14444..8c87629 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HealthMonitor.java
@@ -30,6 +30,7 @@ import static org.apache.hadoop.fs.CommonConfigurationKeys.*;
import org.apache.hadoop.ha.HAServiceProtocol;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
import org.apache.hadoop.ha.HealthCheckFailedException;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.util.Daemon;
@@ -201,18 +202,20 @@ public class HealthMonitor {
status = proxy.getServiceStatus();
proxy.monitorHealth();
healthy = true;
- } catch (HealthCheckFailedException e) {
- LOG.warn("Service health check failed for " + targetToMonitor
- + ": " + e.getMessage());
- enterState(State.SERVICE_UNHEALTHY);
} catch (Throwable t) {
- LOG.warn("Transport-level exception trying to monitor health of " +
- targetToMonitor + ": " + t.getLocalizedMessage());
- RPC.stopProxy(proxy);
- proxy = null;
- enterState(State.SERVICE_NOT_RESPONDING);
- Thread.sleep(sleepAfterDisconnectMillis);
- return;
+ if (isHealthCheckFailedException(t)) {
+ LOG.warn("Service health check failed for " + targetToMonitor
+ + ": " + t.getMessage());
+ enterState(State.SERVICE_UNHEALTHY);
+ } else {
+ LOG.warn("Transport-level exception trying to monitor health of " +
+ targetToMonitor + ": " + t.getCause() + " " + t.getLocalizedMessage());
+ RPC.stopProxy(proxy);
+ proxy = null;
+ enterState(State.SERVICE_NOT_RESPONDING);
+ Thread.sleep(sleepAfterDisconnectMillis);
+ return;
+ }
}
if (status != null) {
@@ -225,7 +228,15 @@ public class HealthMonitor {
Thread.sleep(checkIntervalMillis);
}
}
-
+
+ private boolean isHealthCheckFailedException(Throwable t) {
+ return ((t instanceof HealthCheckFailedException) ||
+ (t instanceof RemoteException &&
+ ((RemoteException)t).unwrapRemoteException(
+ HealthCheckFailedException.class) instanceof
+ HealthCheckFailedException));
+ }
+
private synchronized void setLastServiceStatus(HAServiceStatus status) {
this.lastServiceState = status;
for (ServiceStateCallback cb : serviceStateCallbacks) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
index e9189e2..aef6c4d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/DummyHAService.java
@@ -22,15 +22,25 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
+import com.google.protobuf.BlockingService;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ha.HAServiceProtocol.HAServiceState;
+import org.apache.hadoop.ha.protocolPB.HAServiceProtocolPB;
+import org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ha.proto.HAServiceProtocolProtos.HAServiceProtocolService;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AccessControlException;
import org.mockito.Mockito;
import com.google.common.collect.Lists;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.HA_HM_RPC_TIMEOUT_DEFAULT;
+
/**
* Test-only implementation of {@link HAServiceTarget}, which returns
* a mock implementation.
@@ -50,22 +60,33 @@ class DummyHAService extends HAServiceTarget {
DummySharedResource sharedResource;
public int fenceCount = 0;
public int activeTransitionCount = 0;
+ boolean testWithProtoBufRPC = false;
static ArrayList<DummyHAService> instances = Lists.newArrayList();
int index;
DummyHAService(HAServiceState state, InetSocketAddress address) {
+ this(state, address, false);
+ }
+
+ DummyHAService(HAServiceState state, InetSocketAddress address,
+ boolean testWithProtoBufRPC) {
this.state = state;
- this.proxy = makeMock();
+ this.testWithProtoBufRPC = testWithProtoBufRPC;
+ if (testWithProtoBufRPC) {
+ this.address = startAndGetRPCServerAddress(address);
+ } else {
+ this.address = address;
+ }
+ Configuration conf = new Configuration();
+ this.proxy = makeMock(conf, HA_HM_RPC_TIMEOUT_DEFAULT);
try {
- Configuration conf = new Configuration();
- conf.set(DUMMY_FENCE_KEY, DummyFencer.class.getName());
+ conf.set(DUMMY_FENCE_KEY, DummyFencer.class.getName());
this.fencer = Mockito.spy(
NodeFencer.create(conf, DUMMY_FENCE_KEY));
} catch (BadFencingConfigurationException e) {
throw new RuntimeException(e);
}
- this.address = address;
synchronized (instances) {
instances.add(this);
this.index = instances.size();
@@ -75,9 +96,42 @@ class DummyHAService extends HAServiceTarget {
public void setSharedResource(DummySharedResource rsrc) {
this.sharedResource = rsrc;
}
-
- private HAServiceProtocol makeMock() {
- return Mockito.spy(new MockHAProtocolImpl());
+
+ private InetSocketAddress startAndGetRPCServerAddress(InetSocketAddress serverAddress) {
+ Configuration conf = new Configuration();
+
+ try {
+ RPC.setProtocolEngine(conf,
+ HAServiceProtocolPB.class, ProtobufRpcEngine.class);
+ HAServiceProtocolServerSideTranslatorPB haServiceProtocolXlator =
+ new HAServiceProtocolServerSideTranslatorPB(new MockHAProtocolImpl());
+ BlockingService haPbService = HAServiceProtocolService
+ .newReflectiveBlockingService(haServiceProtocolXlator);
+
+ Server server = new RPC.Builder(conf)
+ .setProtocol(HAServiceProtocolPB.class)
+ .setInstance(haPbService)
+ .setBindAddress(serverAddress.getHostName())
+ .setPort(serverAddress.getPort()).build();
+ server.start();
+ return NetUtils.getConnectAddress(server);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+
+ private HAServiceProtocol makeMock(Configuration conf, int timeoutMs) {
+ HAServiceProtocol service;
+ if (!testWithProtoBufRPC) {
+ service = new MockHAProtocolImpl();
+ } else {
+ try {
+ service = super.getProxy(conf, timeoutMs);
+ } catch (IOException e) {
+ return null;
+ }
+ }
+ return Mockito.spy(service);
}
@Override
@@ -93,6 +147,9 @@ class DummyHAService extends HAServiceTarget {
@Override
public HAServiceProtocol getProxy(Configuration conf, int timeout)
throws IOException {
+ if (testWithProtoBufRPC) {
+ proxy = makeMock(conf, timeout);
+ }
return proxy;
}
@@ -168,7 +225,7 @@ class DummyHAService extends HAServiceTarget {
public HAServiceStatus getServiceStatus() throws IOException {
checkUnreachable();
HAServiceStatus ret = new HAServiceStatus(state);
- if (state == HAServiceState.STANDBY) {
+ if (state == HAServiceState.STANDBY || state == HAServiceState.ACTIVE) {
ret.setReadyToBecomeActive();
}
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cf4b7f50/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
index db534de..b58793f 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ha/TestHealthMonitor.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ha;
import static org.junit.Assert.*;
import java.io.IOException;
+import java.net.InetSocketAddress;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
@@ -54,7 +55,8 @@ public class TestHealthMonitor {
conf.setInt(CommonConfigurationKeys.HA_HM_CONNECT_RETRY_INTERVAL_KEY, 50);
conf.setInt(CommonConfigurationKeys.HA_HM_SLEEP_AFTER_DISCONNECT_KEY, 50);
- svc = new DummyHAService(HAServiceState.ACTIVE, null);
+ svc = new DummyHAService(HAServiceState.ACTIVE,
+ new InetSocketAddress("0.0.0.0", 0), true);
hm = new HealthMonitor(conf, svc) {
@Override
protected HAServiceProtocol createProxy() throws IOException {
[19/50] [abbrv] hadoop git commit: HDFS-7803. Wrong command mentioned
in HDFSHighAvailabilityWithQJM documentation (Arshad Mohammad via aw)
Posted by zj...@apache.org.
HDFS-7803. Wrong command mentioned in HDFSHighAvailabilityWithQJM documentation (Arshad Mohammad via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34b78d51
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34b78d51
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34b78d51
Branch: refs/heads/YARN-2928
Commit: 34b78d51b5b3dba1988b46c47af1739a4ed7b339
Parents: 826267f
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 17 06:46:37 2015 -1000
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 17 06:46:37 2015 -1000
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b78d51/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index f28e41e..5e54731 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -301,6 +301,9 @@ Trunk (Unreleased)
HDFS-7791. dfs count -v should be added to quota documentation (Akira
AJISAKA via aw)
+ HDFS-7803. Wrong command mentioned in HDFSHighAvailabilityWithQJM
+ documentation (Arshad Mohammad via aw)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/34b78d51/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index 0c84f75..a285fde 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -371,7 +371,7 @@ Once the JournalNodes have been started, one must initially synchronize the two
sufficient edits transactions to be able to start both NameNodes.
* If you are converting a non-HA NameNode to be HA, you should run the
- command "*hdfs -initializeSharedEdits*", which will initialize the
+ command "*hdfs namenode -initializeSharedEdits*", which will initialize the
JournalNodes with the edits data from the local NameNode edits directories.
At this point you may start both of your HA NameNodes as you normally would start a NameNode.
[15/50] [abbrv] hadoop git commit: HADOOP-11600. Fix up source codes
to be compiled with Guava 17.0. (ozawa)
Posted by zj...@apache.org.
HADOOP-11600. Fix up source codes to be compiled with Guava 17.0. (ozawa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2f0f756b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2f0f756b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2f0f756b
Branch: refs/heads/YARN-2928
Commit: 2f0f756b26ea83e142a5b9379fa75862c2fc6ad5
Parents: cf4b7f5
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Feb 17 21:56:20 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Feb 17 21:56:20 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../java/org/apache/hadoop/fs/shell/XAttrCommands.java | 10 ++++++++--
.../hdfs/tools/offlineImageViewer/PBImageTextWriter.java | 2 +-
3 files changed, 11 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f0f756b/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 51305bb..c3aafe8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -605,6 +605,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11589. NetUtils.createSocketAddr should trim the input URI.
(Rakesh R via ozawa)
+ HADOOP-11600. Fix up source codes to be compiled with Guava 17.0. (ozawa)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f0f756b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
index 44e970b..4efda87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/XAttrCommands.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.shell;
import java.io.IOException;
import java.util.Iterator;
import java.util.LinkedList;
+import java.util.Locale;
import java.util.Map;
import java.util.Map.Entry;
@@ -77,9 +78,14 @@ class XAttrCommands extends FsCommand {
name = StringUtils.popOptionWithArgument("-n", args);
String en = StringUtils.popOptionWithArgument("-e", args);
if (en != null) {
- encoding = enValueOfFunc.apply(en.toUpperCase());
+ try {
+ encoding = enValueOfFunc.apply(en.toUpperCase(Locale.ENGLISH));
+ } catch (IllegalArgumentException e) {
+ throw new IllegalArgumentException(
+ "Invalid/unsupported encoding option specified: " + en);
+ }
Preconditions.checkArgument(encoding != null,
- "Invalid/unsupported encoding option specified: " + en);
+ "Invalid/unsupported encoding option specified: " + en);
}
boolean r = StringUtils.popOption("-R", args);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2f0f756b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
index 0da263d..d228920 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
@@ -19,7 +19,6 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
-import com.google.common.io.LimitInputStream;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.PermissionStatus;
@@ -33,6 +32,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.INode;
import org.apache.hadoop.hdfs.server.namenode.INodeId;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.util.LimitInputStream;
import org.apache.hadoop.util.Time;
import org.fusesource.leveldbjni.JniDBFactory;
import org.iq80.leveldb.DB;
[25/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredAppMasterRest.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredAppMasterRest.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredAppMasterRest.apt.vm
deleted file mode 100644
index c33f647..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredAppMasterRest.apt.vm
+++ /dev/null
@@ -1,2709 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- MapReduce Application Master REST API's.
- ---
- ---
- ${maven.build.timestamp}
-
-MapReduce Application Master REST API's.
-
-%{toc|section=1|fromDepth=0|toDepth=2}
-
-* Overview
-
- The MapReduce Application Master REST API's allow the user to get status on the running MapReduce application master. Currently this is the equivalent to a running MapReduce job. The information includes the jobs the app master is running and all the job particulars like tasks, counters, configuration, attempts, etc. The application master should be accessed via the proxy. This proxy is configurable to run either on the resource manager or on a separate host. The proxy URL usually looks like: http://<proxy http address:port>/proxy/{appid}.
-
-* Mapreduce Application Master Info API
-
- The MapReduce application master information resource provides overall information about that mapreduce application master. This includes application id, time it was started, user, name, etc.
-
-** URI
-
- Both of the following URI's give you the MapReduce application master information, from an application id identified by the appid value.
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/info
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <info> object
-
- When you make a request for the mapreduce application master information, the information will be returned as an info object.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| appId | long | The application id |
-*---------------+--------------+-------------------------------+
-| startedOn | long | The time the application started (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| name | string | The name of the application |
-*---------------+--------------+-------------------------------+
-| user | string | The user name of the user who started the application |
-*---------------+--------------+-------------------------------+
-| elapsedTime | long | The time since the application was started (in ms)|
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0003/ws/v1/mapreduce/info
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "info" : {
- "appId" : "application_1326232085508_0003",
- "startedOn" : 1326238244047,
- "user" : "user1",
- "name" : "Sleep job",
- "elapsedTime" : 32374
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
------
- Accept: application/xml
- GET http://<proxy http address:port>/proxy/application_1326232085508_0003/ws/v1/mapreduce/info
------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 223
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<info>
- <appId>application_1326232085508_0003</appId>
- <name>Sleep job</name>
- <user>user1</user>
- <startedOn>1326238244047</startedOn>
- <elapsedTime>32407</elapsedTime>
-</info>
-+---+
-
-* Jobs API
-
- The jobs resource provides a list of the jobs running on this application master. See also {{Job API}} for syntax of the job object.
-
-** URI
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <jobs> object
-
- When you make a request for the list of jobs, the information will be returned as a collection of job objects. See also {{Job API}} for syntax of the job object.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| job | array of job objects(JSON)/Zero or more job objects(XML) | The collection of job objects |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobs" : {
- "job" : [
- {
- "runningReduceAttempts" : 1,
- "reduceProgress" : 100,
- "failedReduceAttempts" : 0,
- "newMapAttempts" : 0,
- "mapsRunning" : 0,
- "state" : "RUNNING",
- "successfulReduceAttempts" : 0,
- "reducesRunning" : 1,
- "acls" : [
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-modify-job"
- },
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-view-job"
- }
- ],
- "reducesPending" : 0,
- "user" : "user1",
- "reducesTotal" : 1,
- "mapsCompleted" : 1,
- "startTime" : 1326238769379,
- "id" : "job_1326232085508_4_4",
- "successfulMapAttempts" : 1,
- "runningMapAttempts" : 0,
- "newReduceAttempts" : 0,
- "name" : "Sleep job",
- "mapsPending" : 0,
- "elapsedTime" : 59377,
- "reducesCompleted" : 0,
- "mapProgress" : 100,
- "diagnostics" : "",
- "failedMapAttempts" : 0,
- "killedReduceAttempts" : 0,
- "mapsTotal" : 1,
- "uberized" : false,
- "killedMapAttempts" : 0,
- "finishTime" : 0
- }
- ]
- }
- }
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 1214
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobs>
- <job>
- <startTime>1326238769379</startTime>
- <finishTime>0</finishTime>
- <elapsedTime>59416</elapsedTime>
- <id>job_1326232085508_4_4</id>
- <name>Sleep job</name>
- <user>user1</user>
- <state>RUNNING</state>
- <mapsTotal>1</mapsTotal>
- <mapsCompleted>1</mapsCompleted>
- <reducesTotal>1</reducesTotal>
- <reducesCompleted>0</reducesCompleted>
- <mapProgress>100.0</mapProgress>
- <reduceProgress>100.0</reduceProgress>
- <mapsPending>0</mapsPending>
- <mapsRunning>0</mapsRunning>
- <reducesPending>0</reducesPending>
- <reducesRunning>1</reducesRunning>
- <uberized>false</uberized>
- <diagnostics/>
- <newReduceAttempts>0</newReduceAttempts>
- <runningReduceAttempts>1</runningReduceAttempts>
- <failedReduceAttempts>0</failedReduceAttempts>
- <killedReduceAttempts>0</killedReduceAttempts>
- <successfulReduceAttempts>0</successfulReduceAttempts>
- <newMapAttempts>0</newMapAttempts>
- <runningMapAttempts>0</runningMapAttempts>
- <failedMapAttempts>0</failedMapAttempts>
- <killedMapAttempts>0</killedMapAttempts>
- <successfulMapAttempts>1</successfulMapAttempts>
- <acls>
- <name>mapreduce.job.acl-modify-job</name>
- <value> </value>
- </acls>
- <acls>
- <name>mapreduce.job.acl-view-job</name>
- <value> </value>
- </acls>
- </job>
-</jobs>
-+---+
-
-* {Job API}
-
- A job resource contains information about a particular job that was started by this application master. Certain fields are only accessible if user has permissions - depends on acl settings.
-
-** URI
-
- Use the following URI to obtain a job object, for a job identified by the jobid value.
-
-------
- * http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid}
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <job> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The job id|
-*---------------+--------------+-------------------------------+
-| name | string | The job name |
-*---------------+--------------+-------------------------------+
-| user | string | The user name |
-*---------------+--------------+-------------------------------+
-| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR|
-*---------------+--------------+-------------------------------+
-| startTime | long | The time the job started (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| finishTime | long | The time the job finished (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| elapsedTime | long | The elapsed time since job started (in ms)|
-*---------------+--------------+-------------------------------+
-| mapsTotal | int | The total number of maps |
-*---------------+--------------+-------------------------------+
-| mapsCompleted | int | The number of completed maps |
-*---------------+--------------+-------------------------------+
-| reducesTotal | int | The total number of reduces |
-*---------------+--------------+-------------------------------+
-| reducesCompleted | int | The number of completed reduces|
-*---------------+--------------+-------------------------------+
-| diagnostics | string | A diagnostic message |
-*---------------+--------------+-------------------------------+
-| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master|
-*---------------+--------------+-------------------------------+
-| mapsPending | int | The number of maps still to be run|
-*---------------+--------------+-------------------------------+
-| mapsRunning | int | The number of running maps |
-*---------------+--------------+-------------------------------+
-| reducesPending | int | The number of reduces still to be run |
-*---------------+--------------+-------------------------------+
-| reducesRunning | int | The number of running reduces|
-*---------------+--------------+-------------------------------+
-| newReduceAttempts | int | The number of new reduce attempts |
-*---------------+--------------+-------------------------------+
-| runningReduceAttempts | int | The number of running reduce attempts |
-*---------------+--------------+-------------------------------+
-| failedReduceAttempts | int | The number of failed reduce attempts |
-*---------------+--------------+-------------------------------+
-| killedReduceAttempts | int | The number of killed reduce attempts |
-*---------------+--------------+-------------------------------+
-| successfulReduceAttempts | int | The number of successful reduce attempts |
-*---------------+--------------+-------------------------------+
-| newMapAttempts | int | The number of new map attempts |
-*---------------+--------------+-------------------------------+
-| runningMapAttempts | int | The number of running map attempts |
-*---------------+--------------+-------------------------------+
-| failedMapAttempts | int | The number of failed map attempts |
-*---------------+--------------+-------------------------------+
-| killedMapAttempts | int | The number of killed map attempts |
-*---------------+--------------+-------------------------------+
-| successfulMapAttempts | int | The number of successful map attempts |
-*---------------+--------------+-------------------------------+
-| acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <acls> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| value | string | The acl value|
-*---------------+--------------+-------------------------------+
-| name | string | The acl name |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Server: Jetty(6.1.26)
- Content-Length: 720
-+---+
-
- Response Body:
-
-+---+
-{
- "job" : {
- "runningReduceAttempts" : 1,
- "reduceProgress" : 100,
- "failedReduceAttempts" : 0,
- "newMapAttempts" : 0,
- "mapsRunning" : 0,
- "state" : "RUNNING",
- "successfulReduceAttempts" : 0,
- "reducesRunning" : 1,
- "acls" : [
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-modify-job"
- },
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-view-job"
- }
- ],
- "reducesPending" : 0,
- "user" : "user1",
- "reducesTotal" : 1,
- "mapsCompleted" : 1,
- "startTime" : 1326238769379,
- "id" : "job_1326232085508_4_4",
- "successfulMapAttempts" : 1,
- "runningMapAttempts" : 0,
- "newReduceAttempts" : 0,
- "name" : "Sleep job",
- "mapsPending" : 0,
- "elapsedTime" : 59437,
- "reducesCompleted" : 0,
- "mapProgress" : 100,
- "diagnostics" : "",
- "failedMapAttempts" : 0,
- "killedReduceAttempts" : 0,
- "mapsTotal" : 1,
- "uberized" : false,
- "killedMapAttempts" : 0,
- "finishTime" : 0
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 1201
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<job>
- <startTime>1326238769379</startTime>
- <finishTime>0</finishTime>
- <elapsedTime>59474</elapsedTime>
- <id>job_1326232085508_4_4</id>
- <name>Sleep job</name>
- <user>user1</user>
- <state>RUNNING</state>
- <mapsTotal>1</mapsTotal>
- <mapsCompleted>1</mapsCompleted>
- <reducesTotal>1</reducesTotal>
- <reducesCompleted>0</reducesCompleted>
- <mapProgress>100.0</mapProgress>
- <reduceProgress>100.0</reduceProgress>
- <mapsPending>0</mapsPending>
- <mapsRunning>0</mapsRunning>
- <reducesPending>0</reducesPending>
- <reducesRunning>1</reducesRunning>
- <uberized>false</uberized>
- <diagnostics/>
- <newReduceAttempts>0</newReduceAttempts>
- <runningReduceAttempts>1</runningReduceAttempts>
- <failedReduceAttempts>0</failedReduceAttempts>
- <killedReduceAttempts>0</killedReduceAttempts>
- <successfulReduceAttempts>0</successfulReduceAttempts>
- <newMapAttempts>0</newMapAttempts>
- <runningMapAttempts>0</runningMapAttempts>
- <failedMapAttempts>0</failedMapAttempts>
- <killedMapAttempts>0</killedMapAttempts>
- <successfulMapAttempts>1</successfulMapAttempts>
- <acls>
- <name>mapreduce.job.acl-modify-job</name>
- <value> </value>
- </acls>
- <acls>
- <name>mapreduce.job.acl-view-job</name> <value> </value>
- </acls>
-</job>
-+---+
-
-* Job Attempts API
-
- With the job attempts API, you can obtain a collection of resources that represent the job attempts. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects.
-
-** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/jobs/{jobid}/jobattempts
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <jobAttempts> object
-
- When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects |
-*---------------+--------------+--------------------------------+
-
-** Elements of the <jobAttempt> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The job attempt id |
-*---------------+--------------+--------------------------------+
-| nodeId | string | The node id of the node the attempt ran on|
-*---------------+--------------+--------------------------------+
-| nodeHttpAddress | string | The node http address of the node the attempt ran on|
-*---------------+--------------+--------------------------------+
-| logsLink | string | The http link to the job attempt logs |
-*---------------+--------------+--------------------------------+
-| containerId | string | The id of the container for the job attempt |
-*---------------+--------------+--------------------------------+
-| startTime | long | The start time of the attempt (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobAttempts" : {
- "jobAttempt" : [
- {
- "nodeId" : "host.domain.com:8041",
- "nodeHttpAddress" : "host.domain.com:8042",
- "startTime" : 1326238773493,
- "id" : 1,
- "logsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326232085508_0004_01_000001",
- "containerId" : "container_1326232085508_0004_01_000001"
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 498
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobAttempts>
- <jobAttempt>
- <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
- <nodeId>host.domain.com:8041</nodeId>
- <id>1</id>
- <startTime>1326238773493</startTime>
- <containerId>container_1326232085508_0004_01_000001</containerId>
- <logsLink>http://host.domain.com:8042/node/containerlogs/container_1326232085508_0004_01_000001</logsLink>
- </jobAttempt>
-</jobAttempts>
-+---+
-
-* Job Counters API
-
- With the job counters API, you can object a collection of resources that represent all the counters for that job.
-
-** URI
-
-------
- * http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid}/counters
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <jobCounters> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The job id |
-*---------------+--------------+-------------------------------+
-| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <counterGroup> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| counterGroupName | string | The name of the counter group |
-*---------------+--------------+-------------------------------+
-| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <counter> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the counter |
-*---------------+--------------+-------------------------------+
-| reduceCounterValue | long | The counter value of reduce tasks |
-*---------------+--------------+-------------------------------+
-| mapCounterValue | long | The counter value of map tasks |
-*---------------+--------------+-------------------------------+
-| totalCounterValue | long | The counter value of all tasks |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobCounters" : {
- "id" : "job_1326232085508_4_4",
- "counterGroup" : [
- {
- "counterGroupName" : "Shuffle Errors",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "BAD_ID"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "CONNECTION"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "IO_ERROR"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "WRONG_LENGTH"
- }, {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "WRONG_MAP"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "WRONG_REDUCE"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2483,
- "name" : "FILE_BYTES_READ"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 108763,
- "name" : "FILE_BYTES_WRITTEN"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FILE_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FILE_LARGE_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FILE_WRITE_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 48,
- "name" : "HDFS_BYTES_READ"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "HDFS_BYTES_WRITTEN"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "HDFS_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "HDFS_LARGE_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "HDFS_WRITE_OPS"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "MAP_INPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1200,
- "name" : "MAP_OUTPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 4800,
- "name" : "MAP_OUTPUT_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2235,
- "name" : "MAP_OUTPUT_MATERIALIZED_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 48,
- "name" : "SPLIT_RAW_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "COMBINE_INPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "COMBINE_OUTPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 460,
- "name" : "REDUCE_INPUT_GROUPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2235,
- "name" : "REDUCE_SHUFFLE_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 460,
- "name" : "REDUCE_INPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "REDUCE_OUTPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1200,
- "name" : "SPILLED_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "SHUFFLED_MAPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FAILED_SHUFFLE"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "MERGED_MAP_OUTPUTS"
- }, {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 58,
- "name" : "GC_TIME_MILLIS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1580,
- "name" : "CPU_MILLISECONDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 462643200,
- "name" : "PHYSICAL_MEMORY_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2149728256,
- "name" : "VIRTUAL_MEMORY_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 357957632,
- "name" : "COMMITTED_HEAP_BYTES"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "BYTES_READ"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "BYTES_WRITTEN"
- }
- ]
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 7027
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobCounters>
- <id>job_1326232085508_4_4</id>
- <counterGroup>
- <counterGroupName>Shuffle Errors</counterGroupName>
- <counter>
- <name>BAD_ID</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>CONNECTION</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>IO_ERROR</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>WRONG_LENGTH</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>WRONG_MAP</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>WRONG_REDUCE</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
- <counter>
- <name>FILE_BYTES_READ</name>
- <totalCounterValue>2483</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_BYTES_WRITTEN</name>
- <totalCounterValue>108763</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_READ_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_LARGE_READ_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_WRITE_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_BYTES_READ</name>
- <totalCounterValue>48</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_BYTES_WRITTEN</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_READ_OPS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_LARGE_READ_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_WRITE_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
- <counter>
- <name>MAP_INPUT_RECORDS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MAP_OUTPUT_RECORDS</name>
- <totalCounterValue>1200</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MAP_OUTPUT_BYTES</name>
- <totalCounterValue>4800</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MAP_OUTPUT_MATERIALIZED_BYTES</name>
- <totalCounterValue>2235</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>SPLIT_RAW_BYTES</name>
- <totalCounterValue>48</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>COMBINE_INPUT_RECORDS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>COMBINE_OUTPUT_RECORDS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_INPUT_GROUPS</name>
- <totalCounterValue>460</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_SHUFFLE_BYTES</name>
- <totalCounterValue>2235</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_INPUT_RECORDS</name>
- <totalCounterValue>460</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_OUTPUT_RECORDS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>SPILLED_RECORDS</name>
- <totalCounterValue>1200</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>SHUFFLED_MAPS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FAILED_SHUFFLE</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MERGED_MAP_OUTPUTS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>GC_TIME_MILLIS</name>
- <totalCounterValue>58</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>CPU_MILLISECONDS</name>
- <totalCounterValue>1580</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>PHYSICAL_MEMORY_BYTES</name>
- <totalCounterValue>462643200</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>VIRTUAL_MEMORY_BYTES</name>
- <totalCounterValue>2149728256</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>COMMITTED_HEAP_BYTES</name>
- <totalCounterValue>357957632</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_READ</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter> </counterGroup> <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
- <counter> <name>BYTES_WRITTEN</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
-</jobCounters>
-+---+
-
-* Job Conf API
-
- A job configuration resource contains information about the job configuration for this job.
-
-** URI
-
- Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value.
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/conf
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <conf> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| path | string | The path to the job configuration file|
-*---------------+--------------+-------------------------------+
-| property | array of the configuration properties(JSON)/zero or more property objects(XML) | Collection of property objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <property> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the configuration property |
-*---------------+--------------+-------------------------------+
-| value | string | The value of the configuration property |
-*---------------+--------------+-------------------------------+
-| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
- This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file.
-
-+---+
-{
- "conf" : {
- "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml",
- "property" : [
- {
- "value" : "/home/hadoop/hdfs/data",
- "name" : "dfs.datanode.data.dir",
- "source" : ["hdfs-site.xml", "job.xml"]
- },
- {
- "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
- "name" : "hadoop.http.filter.initializers"
- "source" : ["programmatically", "job.xml"]
- },
- {
- "value" : "/home/hadoop/tmp",
- "name" : "mapreduce.cluster.temp.dir"
- "source" : ["mapred-site.xml"]
- },
- ...
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 552
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<conf>
- <path>hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml</path>
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/home/hadoop/hdfs/data</value>
- <source>hdfs-site.xml</source>
- <source>job.xml</source>
- </property>
- <property>
- <name>hadoop.http.filter.initializers</name>
- <value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
- <source>programmatically</source>
- <source>job.xml</source>
- </property>
- <property>
- <name>mapreduce.cluster.temp.dir</name>
- <value>/home/hadoop/tmp</value>
- <source>mapred-site.xml</source>
- </property>
- ...
-</conf>
-+---+
-
-* Tasks API
-
- With the tasks API, you can obtain a collection of resources that represent all the tasks for a job. When you run a GET operation on this resource, you obtain a collection of Task Objects.
-
-** URI
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- * type - type of task, valid values are m or r. m for map task or r for reduce task.
-------
-
-** Elements of the <tasks> object
-
- When you make a request for the list of tasks , the information will be returned as an array of task objects.
- See also {{Task API}} for syntax of the task object.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects |
-*---------------+--------------+--------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "tasks" : {
- "task" : [
- {
- "progress" : 100,
- "elapsedTime" : 2768,
- "state" : "SUCCEEDED",
- "startTime" : 1326238773493,
- "id" : "task_1326232085508_4_4_m_0",
- "type" : "MAP",
- "successfulAttempt" : "attempt_1326232085508_4_4_m_0_0",
- "finishTime" : 1326238776261
- },
- {
- "progress" : 100,
- "elapsedTime" : 0,
- "state" : "RUNNING",
- "startTime" : 1326238777460,
- "id" : "task_1326232085508_4_4_r_0",
- "type" : "REDUCE",
- "successfulAttempt" : "",
- "finishTime" : 0
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 603
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<tasks>
- <task>
- <startTime>1326238773493</startTime>
- <finishTime>1326238776261</finishTime>
- <elapsedTime>2768</elapsedTime>
- <progress>100.0</progress>
- <id>task_1326232085508_4_4_m_0</id>
- <state>SUCCEEDED</state>
- <type>MAP</type>
- <successfulAttempt>attempt_1326232085508_4_4_m_0_0</successfulAttempt>
- </task>
- <task>
- <startTime>1326238777460</startTime>
- <finishTime>0</finishTime>
- <elapsedTime>0</elapsedTime>
- <progress>100.0</progress>
- <id>task_1326232085508_4_4_r_0</id>
- <state>RUNNING</state>
- <type>REDUCE</type>
- <successfulAttempt/>
- </task>
-</tasks>
-+---+
-
-* {Task API}
-
- A Task resource contains information about a particular task within a job.
-
-** URI
-
- Use the following URI to obtain an Task Object, from a task identified by the {taskid} value.
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <task> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task id |
-*---------------+--------------+--------------------------------+
-| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED |
-*---------------+--------------+--------------------------------+
-| type | string | The task type - MAP or REDUCE|
-*---------------+--------------+--------------------------------+
-| successfulAttempt | string | The the id of the last successful attempt |
-*---------------+--------------+--------------------------------+
-| progress | float | The progress of the task as a percent|
-*---------------+--------------+--------------------------------+
-| startTime | long | The time in which the task started (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| finishTime | long | The time in which the task finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| elapsedTime | long | The elapsed time since the application started (in ms)|
-*---------------+--------------+--------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "task" : {
- "progress" : 100,
- "elapsedTime" : 0,
- "state" : "RUNNING",
- "startTime" : 1326238777460,
- "id" : "task_1326232085508_4_4_r_0",
- "type" : "REDUCE",
- "successfulAttempt" : "",
- "finishTime" : 0
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 299
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<task>
- <startTime>1326238777460</startTime>
- <finishTime>0</finishTime>
- <elapsedTime>0</elapsedTime>
- <progress>100.0</progress>
- <id>task_1326232085508_4_4_r_0</id>
- <state>RUNNING</state>
- <type>REDUCE</type>
- <successfulAttempt/>
-</task>
-+---+
-
-* Task Counters API
-
- With the task counters API, you can object a collection of resources that represent all the counters for that task.
-
-** URI
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <jobTaskCounters> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task id |
-*---------------+--------------+-------------------------------+
-| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <counterGroup> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| counterGroupName | string | The name of the counter group |
-*---------------+--------------+-------------------------------+
-| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <counter> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the counter |
-*---------------+--------------+-------------------------------+
-| value | long | The value of the counter |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobTaskCounters" : {
- "id" : "task_1326232085508_4_4_r_0",
- "taskCounterGroup" : [
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
- "counter" : [
- {
- "value" : 2363,
- "name" : "FILE_BYTES_READ"
- },
- {
- "value" : 54372,
- "name" : "FILE_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "FILE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_WRITE_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_READ"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "HDFS_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_WRITE_OPS"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "COMBINE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "COMBINE_OUTPUT_RECORDS"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_GROUPS"
- },
- {
- "value" : 2235,
- "name" : "REDUCE_SHUFFLE_BYTES"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "REDUCE_OUTPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "SPILLED_RECORDS"
- },
- {
- "value" : 1,
- "name" : "SHUFFLED_MAPS"
- },
- {
- "value" : 0,
- "name" : "FAILED_SHUFFLE"
- },
- {
- "value" : 1,
- "name" : "MERGED_MAP_OUTPUTS"
- },
- {
- "value" : 26,
- "name" : "GC_TIME_MILLIS"
- },
- {
- "value" : 860,
- "name" : "CPU_MILLISECONDS"
- },
- {
- "value" : 107839488,
- "name" : "PHYSICAL_MEMORY_BYTES"
- },
- {
- "value" : 1123147776,
- "name" : "VIRTUAL_MEMORY_BYTES"
- },
- {
- "value" : 57475072,
- "name" : "COMMITTED_HEAP_BYTES"
- }
- ]
- },
- {
- "counterGroupName" : "Shuffle Errors",
- "counter" : [
- {
- "value" : 0,
- "name" : "BAD_ID"
- },
- {
- "value" : 0,
- "name" : "CONNECTION"
- },
- {
- "value" : 0,
- "name" : "IO_ERROR"
- },
- {
- "value" : 0,
- "name" : "WRONG_LENGTH"
- },
- {
- "value" : 0,
- "name" : "WRONG_MAP"
- },
- {
- "value" : 0,
- "name" : "WRONG_REDUCE"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "BYTES_WRITTEN"
- }
- ]
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 2660
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobTaskCounters>
- <id>task_1326232085508_4_4_r_0</id>
- <taskCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
- <counter>
- <name>FILE_BYTES_READ</name>
- <value>2363</value>
- </counter>
- <counter>
- <name>FILE_BYTES_WRITTEN</name>
- <value>54372</value>
- </counter>
- <counter>
- <name>FILE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_WRITE_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_READ</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_WRITE_OPS</name>
- <value>0</value>
- </counter>
- </taskCounterGroup>
- <taskCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
- <counter>
- <name>COMBINE_INPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>COMBINE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_GROUPS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_SHUFFLE_BYTES</name>
- <value>2235</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_RECORDS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SPILLED_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SHUFFLED_MAPS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>FAILED_SHUFFLE</name>
- <value>0</value>
- </counter>
- <counter>
- <name>MERGED_MAP_OUTPUTS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>GC_TIME_MILLIS</name>
- <value>26</value>
- </counter>
- <counter>
- <name>CPU_MILLISECONDS</name>
- <value>860</value>
- </counter>
- <counter>
- <name>PHYSICAL_MEMORY_BYTES</name>
- <value>107839488</value>
- </counter>
- <counter>
- <name>VIRTUAL_MEMORY_BYTES</name>
- <value>1123147776</value>
- </counter>
- <counter>
- <name>COMMITTED_HEAP_BYTES</name>
- <value>57475072</value>
- </counter>
- </taskCounterGroup>
- <taskCounterGroup>
- <counterGroupName>Shuffle Errors</counterGroupName>
- <counter>
- <name>BAD_ID</name>
- <value>0</value>
- </counter>
- <counter>
- <name>CONNECTION</name>
- <value>0</value>
- </counter>
- <counter>
- <name>IO_ERROR</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_LENGTH</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_MAP</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_REDUCE</name>
- <value>0</value>
- </counter>
- </taskCounterGroup>
- <taskCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- </taskCounterGroup>
-</jobTaskCounters>
-+---+
-
-* Task Attempts API
-
- With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects.
-
-** URI
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <taskAttempts> object
-
- When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects.
- See also {{Task Attempt API}} for syntax of the task object.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects |
-*---------------+--------------+--------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "taskAttempts" : {
- "taskAttempt" : [
- {
- "elapsedMergeTime" : 47,
- "shuffleFinishTime" : 1326238780052,
- "assignedContainerId" : "container_1326232085508_0004_01_000003",
- "progress" : 100,
- "elapsedTime" : 0,
- "state" : "RUNNING",
- "elapsedShuffleTime" : 2592,
- "mergeFinishTime" : 1326238780099,
- "rack" : "/98.139.92.0",
- "elapsedReduceTime" : 0,
- "nodeHttpAddress" : "host.domain.com:8042",
- "type" : "REDUCE",
- "startTime" : 1326238777460,
- "id" : "attempt_1326232085508_4_4_r_0_0",
- "finishTime" : 0
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 807
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<taskAttempts>
- <taskAttempt xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="reduceTaskAttemptInfo">
- <startTime>1326238777460</startTime>
- <finishTime>0</finishTime>
- <elapsedTime>0</elapsedTime>
- <progress>100.0</progress>
- <id>attempt_1326232085508_4_4_r_0_0</id>
- <rack>/98.139.92.0</rack>
- <state>RUNNING</state>
- <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
- <type>REDUCE</type>
- <assignedContainerId>container_1326232085508_0004_01_000003</assignedContainerId>
- <shuffleFinishTime>1326238780052</shuffleFinishTime>
- <mergeFinishTime>1326238780099</mergeFinishTime>
- <elapsedShuffleTime>2592</elapsedShuffleTime>
- <elapsedMergeTime>47</elapsedMergeTime>
- <elapsedReduceTime>0</elapsedReduceTime>
- </taskAttempt>
-</taskAttempts>
-+---+
-
-* {Task Attempt API}
-
- A Task Attempt resource contains information about a particular task attempt within a job.
-
-** URI
-
- Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value.
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <taskAttempt> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task id |
-*---------------+--------------+--------------------------------+
-| rack | string | The rack |
-*---------------+--------------+--------------------------------+
-| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED|
-*---------------+--------------+--------------------------------+
-| type | string | The type of task |
-*---------------+--------------+--------------------------------+
-| assignedContainerId | string | The container id this attempt is assigned to|
-*---------------+--------------+--------------------------------+
-| nodeHttpAddress | string | The http address of the node this task attempt ran on |
-*---------------+--------------+--------------------------------+
-| diagnostics| string | The diagnostics message |
-*---------------+--------------+--------------------------------+
-| progress | float | The progress of the task attempt as a percent|
-*---------------+--------------+--------------------------------+
-| startTime | long | The time in which the task attempt started (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| finishTime | long | The time in which the task attempt finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| elapsedTime | long | The elapsed time since the task attempt started (in ms)|
-*---------------+--------------+--------------------------------+
-
- For reduce task attempts you also have the following fields:
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| mergeFinishTime | long | The time at which merge finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)|
-*---------------+--------------+--------------------------------+
-| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)|
-*---------------+--------------+--------------------------------+
-| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)|
-*---------------+--------------+--------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "taskAttempt" : {
- "elapsedMergeTime" : 47,
- "shuffleFinishTime" : 1326238780052,
- "assignedContainerId" : "container_1326232085508_0004_01_000003",
- "progress" : 100,
- "elapsedTime" : 0,
- "state" : "RUNNING",
- "elapsedShuffleTime" : 2592,
- "mergeFinishTime" : 1326238780099,
- "rack" : "/98.139.92.0",
- "elapsedReduceTime" : 0,
- "nodeHttpAddress" : "host.domain.com:8042",
- "startTime" : 1326238777460,
- "id" : "attempt_1326232085508_4_4_r_0_0",
- "type" : "REDUCE",
- "finishTime" : 0
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 691
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<taskAttempt>
- <startTime>1326238777460</startTime>
- <finishTime>0</finishTime>
- <elapsedTime>0</elapsedTime>
- <progress>100.0</progress>
- <id>attempt_1326232085508_4_4_r_0_0</id>
- <rack>/98.139.92.0</rack>
- <state>RUNNING</state>
- <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
- <type>REDUCE</type>
- <assignedContainerId>container_1326232085508_0004_01_000003</assignedContainerId>
- <shuffleFinishTime>1326238780052</shuffleFinishTime>
- <mergeFinishTime>1326238780099</mergeFinishTime>
- <elapsedShuffleTime>2592</elapsedShuffleTime>
- <elapsedMergeTime>47</elapsedMergeTime>
- <elapsedReduceTime>0</elapsedReduceTime>
-</taskAttempt>
-+---+
-
-* Task Attempt Counters API
-
- With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt.
-
-** URI
-
-------
- * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <jobTaskAttemptCounters> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task attempt id |
-*---------------+--------------+-------------------------------+
-| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <taskAttemptCounterGroup> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| counterGroupName | string | The name of the counter group |
-*---------------+--------------+-------------------------------+
-| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <counter> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the counter |
-*---------------+--------------+-------------------------------+
-| value | long | The value of the counter |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobTaskAttemptCounters" : {
- "taskAttemptCounterGroup" : [
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
- "counter" : [
- {
- "value" : 2363,
- "name" : "FILE_BYTES_READ"
- },
- {
- "value" : 54372,
- "name" : "FILE_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "FILE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_WRITE_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_READ"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "HDFS_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_WRITE_OPS"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "COMBINE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "COMBINE_OUTPUT_RECORDS"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_GROUPS"
- },
- {
- "value" : 2235,
- "name" : "REDUCE_SHUFFLE_BYTES"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "REDUCE_OUTPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "SPILLED_RECORDS"
- },
- {
- "value" : 1,
- "name" : "SHUFFLED_MAPS"
- },
- {
- "value" : 0,
- "name" : "FAILED_SHUFFLE"
- },
- {
- "value" : 1,
- "name" : "MERGED_MAP_OUTPUTS"
- },
- {
- "value" : 26,
- "name" : "GC_TIME_MILLIS"
- },
- {
- "value" : 860,
- "name" : "CPU_MILLISECONDS"
- },
- {
- "value" : 107839488,
- "name" : "PHYSICAL_MEMORY_BYTES"
- },
- {
- "value" : 1123147776,
- "name" : "VIRTUAL_MEMORY_BYTES"
- },
- {
- "value" : 57475072,
- "name" : "COMMITTED_HEAP_BYTES"
- }
- ]
- },
- {
- "counterGroupName" : "Shuffle Errors",
- "counter" : [
- {
- "value" : 0,
- "name" : "BAD_ID"
- },
- {
- "value" : 0,
- "name" : "CONNECTION"
- },
- {
- "value" : 0,
- "name" : "IO_ERROR"
- },
- {
- "value" : 0,
- "name" : "WRONG_LENGTH"
- },
- {
- "value" : 0,
- "name" : "WRONG_MAP"
- },
- {
- "value" : 0,
- "name" : "WRONG_REDUCE"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "BYTES_WRITTEN"
- }
- ]
- }
- ],
- "id" : "attempt_1326232085508_4_4_r_0_0"
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 2735
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobTaskAttemptCounters>
- <id>attempt_1326232085508_4_4_r_0_0</id>
- <taskAttemptCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
- <counter>
- <name>FILE_BYTES_READ</name>
- <value>2363</value>
- </counter>
- <counter>
- <name>FILE_BYTES_WRITTEN</name>
- <value>54372</value>
- </counter>
- <counter>
- <name>FILE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_WRITE_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_READ</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_WRITE_OPS</name>
- <value>0</value>
- </counter>
- </taskAttemptCounterGroup>
- <taskAttemptCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
- <counter>
- <name>COMBINE_INPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>COMBINE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_GROUPS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_SHUFFLE_BYTES</name>
- <value>2235</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_RECORDS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SPILLED_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SHUFFLED_MAPS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>FAILED_SHUFFLE</name>
- <value>0</value>
- </counter>
- <counter>
- <name>MERGED_MAP_OUTPUTS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>GC_TIME_MILLIS</name>
- <value>26</value>
- </counter>
- <counter>
- <name>CPU_MILLISECONDS</name>
- <value>860</value>
- </counter>
- <counter>
- <name>PHYSICAL_MEMORY_BYTES</name>
- <value>107839488</value>
- </counter>
- <counter>
- <name>VIRTUAL_MEMORY_BYTES</name>
- <value>1123147776</value>
- </counter>
- <counter>
- <name>COMMITTED_HEAP_BYTES</name>
- <value>57475072</value>
- </counter>
- </taskAttemptCounterGroup>
- <taskAttemptCounterGroup>
- <counterGroupName>Shuffle Errors</counterGroupName>
- <counter>
- <name>BAD_ID</name>
- <value>0</value>
- </counter>
- <counter>
- <name>CONNECTION</name>
- <value>0</value>
- </counter>
- <counter>
- <name>IO_ERROR</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_LENGTH</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_MAP</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_REDUCE</name>
- <value>0</value>
- </counter>
- </taskAttemptCounterGroup>
- <taskAttemptCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- </taskAttemptCounterGroup>
-</jobTaskAttemptCounters>
-+---+
-
[42/50] [abbrv] hadoop git commit: HADOOP-11593. Convert site
documentation from apt to markdown (stragglers) (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
new file mode 100644
index 0000000..44b5bfb
--- /dev/null
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -0,0 +1,864 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+#set ( $H3 = '###' )
+#set ( $H4 = '####' )
+#set ( $H5 = '#####' )
+
+Hadoop Key Management Server (KMS) - Documentation Sets
+=======================================================
+
+Hadoop KMS is a cryptographic key management server based on Hadoop's **KeyProvider** API.
+
+It provides a client and a server components which communicate over HTTP using a REST API.
+
+The client is a KeyProvider implementation interacts with the KMS using the KMS HTTP REST API.
+
+KMS and its client have built-in security and they support HTTP SPNEGO Kerberos authentication and HTTPS secure transport.
+
+KMS is a Java web-application and it runs using a pre-configured Tomcat bundled with the Hadoop distribution.
+
+KMS Client Configuration
+------------------------
+
+The KMS client `KeyProvider` uses the **kms** scheme, and the embedded URL must be the URL of the KMS. For example, for a KMS running on `http://localhost:16000/kms`, the KeyProvider URI is `kms://http@localhost:16000/kms`. And, for a KMS running on `https://localhost:16000/kms`, the KeyProvider URI is `kms://https@localhost:16000/kms`
+
+KMS
+---
+
+$H3 KMS Configuration
+
+Configure the KMS backing KeyProvider properties in the `etc/hadoop/kms-site.xml` configuration file:
+
+```xml
+ <property>
+ <name>hadoop.kms.key.provider.uri</name>
+ <value>jceks://file@/${user.home}/kms.keystore</value>
+ </property>
+
+ <property>
+ <name>hadoop.security.keystore.java-keystore-provider.password-file</name>
+ <value>kms.keystore.password</value>
+ </property>
+```
+
+The password file is looked up in the Hadoop's configuration directory via the classpath.
+
+NOTE: You need to restart the KMS for the configuration changes to take effect.
+
+$H3 KMS Cache
+
+KMS caches keys for short period of time to avoid excessive hits to the underlying key provider.
+
+The Cache is enabled by default (can be dissabled by setting the `hadoop.kms.cache.enable` boolean property to false)
+
+The cache is used with the following 3 methods only, `getCurrentKey()` and `getKeyVersion()` and `getMetadata()`.
+
+For the `getCurrentKey()` method, cached entries are kept for a maximum of 30000 millisecond regardless the number of times the key is being access (to avoid stale keys to be considered current).
+
+For the `getKeyVersion()` method, cached entries are kept with a default inactivity timeout of 600000 milliseconds (10 mins). This time out is configurable via the following property in the `etc/hadoop/kms-site.xml` configuration file:
+
+```xml
+ <property>
+ <name>hadoop.kms.cache.enable</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.cache.timeout.ms</name>
+ <value>600000</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.current.key.cache.timeout.ms</name>
+ <value>30000</value>
+ </property>
+```
+
+$H3 KMS Aggregated Audit logs
+
+Audit logs are aggregated for API accesses to the GET\_KEY\_VERSION, GET\_CURRENT\_KEY, DECRYPT\_EEK, GENERATE\_EEK operations.
+
+Entries are grouped by the (user,key,operation) combined key for a configurable aggregation interval after which the number of accesses to the specified end-point by the user for a given key is flushed to the audit log.
+
+The Aggregation interval is configured via the property :
+
+ <property>
+ <name>hadoop.kms.aggregation.delay.ms</name>
+ <value>10000</value>
+ </property>
+
+$H3 Start/Stop the KMS
+
+To start/stop KMS use KMS's bin/kms.sh script. For example:
+
+ hadoop-${project.version} $ sbin/kms.sh start
+
+NOTE: Invoking the script without any parameters list all possible parameters (start, stop, run, etc.). The `kms.sh` script is a wrapper for Tomcat's `catalina.sh` script that sets the environment variables and Java System properties required to run KMS.
+
+$H3 Embedded Tomcat Configuration
+
+To configure the embedded Tomcat go to the `share/hadoop/kms/tomcat/conf`.
+
+KMS pre-configures the HTTP and Admin ports in Tomcat's `server.xml` to 16000 and 16001.
+
+Tomcat logs are also preconfigured to go to Hadoop's `logs/` directory.
+
+The following environment variables (which can be set in KMS's `etc/hadoop/kms-env.sh` script) can be used to alter those values:
+
+* KMS_HTTP_PORT
+* KMS_ADMIN_PORT
+* KMS_MAX_THREADS
+* KMS_LOGNOTE: You need to restart the KMS for the configuration changes to take effect.
+
+$H3 Loading native libraries
+
+The following environment variable (which can be set in KMS's `etc/hadoop/kms-env.sh` script) can be used to specify the location of any required native libraries. For eg. Tomact native Apache Portable Runtime (APR) libraries:
+
+* JAVA_LIBRARY_PATH
+
+$H3 KMS Security Configuration
+
+$H4 Enabling Kerberos HTTP SPNEGO Authentication
+
+Configure the Kerberos `etc/krb5.conf` file with the information of your KDC server.
+
+Create a service principal and its keytab for the KMS, it must be an `HTTP` service principal.
+
+Configure KMS `etc/hadoop/kms-site.xml` with the correct security values, for example:
+
+```xml
+ <property>
+ <name>hadoop.kms.authentication.type</name>
+ <value>kerberos</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.keytab</name>
+ <value>${user.home}/kms.keytab</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.principal</name>
+ <value>HTTP/localhost</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.kerberos.name.rules</name>
+ <value>DEFAULT</value>
+ </property>
+```
+
+NOTE: You need to restart the KMS for the configuration changes to take effect.
+
+$H4 KMS Proxyuser Configuration
+
+Each proxyuser must be configured in `etc/hadoop/kms-site.xml` using the following properties:
+
+```xml
+ <property>
+ <name>hadoop.kms.proxyuser.#USER#.users</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.proxyuser.#USER#.groups</name>
+ <value>*</value>
+ </property>
+
+ <property>
+ <name>hadoop.kms.proxyuser.#USER#.hosts</name>
+ <value>*</value>
+ </property>
+```
+
+`#USER#` is the username of the proxyuser to configure.
+
+The `users` property indicates the users that can be impersonated.
+
+The `groups` property indicates the groups users being impersonated must belong to.
+
+At least one of the `users` or `groups` properties must be defined. If both are specified, then the configured proxyuser will be able to impersonate and user in the `users` list and any user belonging to one of the groups in the `groups` list.
+
+The `hosts` property indicates from which host the proxyuser can make impersonation requests.
+
+If `users`, `groups` or `hosts` has a `*`, it means there are no restrictions for the proxyuser regarding users, groups or hosts.
+
+$H4 KMS over HTTPS (SSL)
+
+To configure KMS to work over HTTPS the following 2 properties must be set in the `etc/hadoop/kms_env.sh` script (shown with default values):
+
+* KMS_SSL_KEYSTORE_FILE=$HOME/.keystore
+* KMS_SSL_KEYSTORE_PASS=password
+
+In the KMS `tomcat/conf` directory, replace the `server.xml` file with the provided `ssl-server.xml` file.
+
+You need to create an SSL certificate for the KMS. As the `kms` Unix user, using the Java `keytool` command to create the SSL certificate:
+
+ $ keytool -genkey -alias tomcat -keyalg RSA
+
+You will be asked a series of questions in an interactive prompt. It will create the keystore file, which will be named **.keystore** and located in the `kms` user home directory.
+
+The password you enter for "keystore password" must match the value of the `KMS_SSL_KEYSTORE_PASS` environment variable set in the `kms-env.sh` script in the configuration directory.
+
+The answer to "What is your first and last name?" (i.e. "CN") must be the hostname of the machine where the KMS will be running.
+
+NOTE: You need to restart the KMS for the configuration changes to take effect.
+
+$H4 KMS Access Control
+
+KMS ACLs configuration are defined in the KMS `etc/hadoop/kms-acls.xml` configuration file. This file is hot-reloaded when it changes.
+
+KMS supports both fine grained access control as well as blacklist for kms operations via a set ACL configuration properties.
+
+A user accessing KMS is first checked for inclusion in the Access Control List for the requested operation and then checked for exclusion in the Black list for the operation before access is granted.
+
+```xml
+<configuration>
+ <property>
+ <name>hadoop.kms.acl.CREATE</name>
+ <value>*</value>
+ <description>
+ ACL for create-key operations.
+ If the user is not in the GET ACL, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.CREATE</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for create-key operations.
+ If the user is in the Blacklist, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DELETE</name>
+ <value>*</value>
+ <description>
+ ACL for delete-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.DELETE</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for delete-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.ROLLOVER</name>
+ <value>*</value>
+ <description>
+ ACL for rollover-key operations.
+ If the user is not in the GET ACL, the key material is not returned
+ as part of the response.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.ROLLOVER</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for rollover-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET</name>
+ <value>*</value>
+ <description>
+ ACL for get-key-version and get-current-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.GET</name>
+ <value>hdfs,foo</value>
+ <description>
+ ACL for get-key-version and get-current-key operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_KEYS</name>
+ <value>*</value>
+ <description>
+ ACL for get-keys operation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.GET_KEYS</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for get-keys operation.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GET_METADATA</name>
+ <value>*</value>
+ <description>
+ ACL for get-key-metadata and get-keys-metadata operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.GET_METADATA</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for get-key-metadata and get-keys-metadata operations.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
+ <value>*</value>
+ <description>
+ Complimentary ACL for CREATE and ROLLOVER operation to allow the client
+ to provide the key material when creating or rolling a key.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.SET_KEY_MATERIAL</name>
+ <value>hdfs,foo</value>
+ <description>
+ Complimentary Blacklist for CREATE and ROLLOVER operation to allow the client
+ to provide the key material when creating or rolling a key.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.GENERATE_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for generateEncryptedKey
+ CryptoExtension operations
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.GENERATE_EEK</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for generateEncryptedKey
+ CryptoExtension operations
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.acl.DECRYPT_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for decrypt EncryptedKey
+ CryptoExtension operations
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.blacklist.DECRYPT_EEK</name>
+ <value>hdfs,foo</value>
+ <description>
+ Blacklist for decrypt EncryptedKey
+ CryptoExtension operations
+ </description>
+ </property>
+</configuration>
+```
+
+$H4 Key Access Control
+
+KMS supports access control for all non-read operations at the Key level. All Key Access operations are classified as :
+
+* MANAGEMENT - createKey, deleteKey, rolloverNewVersion
+* GENERATE_EEK - generateEncryptedKey, warmUpEncryptedKeys
+* DECRYPT_EEK - decryptEncryptedKey
+* READ - getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata, getCurrentKey
+* ALL - all of the above
+
+These can be defined in the KMS `etc/hadoop/kms-acls.xml` as follows
+
+For all keys for which a key access has not been explicitly configured, It is possible to configure a default key access control for a subset of the operation types.
+
+It is also possible to configure a "whitelist" key ACL for a subset of the operation types. The whitelist key ACL is a whitelist in addition to the explicit or default per-key ACL. That is, if no per-key ACL is explicitly set, a user will be granted access if they are present in the default per-key ACL or the whitelist key ACL. If a per-key ACL is explicitly set, a user will be granted access if they are present in the per-key ACL or the whitelist key ACL.
+
+If no ACL is configured for a specific key AND no default ACL is configured AND no root key ACL is configured for the requested operation, then access will be DENIED.
+
+**NOTE:** The default and whitelist key ACL does not support `ALL` operation qualifier.
+
+```xml
+ <property>
+ <name>key.acl.testKey1.MANAGEMENT</name>
+ <value>*</value>
+ <description>
+ ACL for create-key, deleteKey and rolloverNewVersion operations.
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey2.GENERATE_EEK</name>
+ <value>*</value>
+ <description>
+ ACL for generateEncryptedKey operations.
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey3.DECRYPT_EEK</name>
+ <value>admink3</value>
+ <description>
+ ACL for decryptEncryptedKey operations.
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey4.READ</name>
+ <value>*</value>
+ <description>
+ ACL for getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata,
+ getCurrentKey operations
+ </description>
+ </property>
+
+ <property>
+ <name>key.acl.testKey5.ALL</name>
+ <value>*</value>
+ <description>
+ ACL for ALL operations.
+ </description>
+ </property>
+
+ <property>
+ <name>whitelist.key.acl.MANAGEMENT</name>
+ <value>admin1</value>
+ <description>
+ whitelist ACL for MANAGEMENT operations for all keys.
+ </description>
+ </property>
+
+ <!--
+ 'testKey3' key ACL is defined. Since a 'whitelist'
+ key is also defined for DECRYPT_EEK, in addition to
+ admink3, admin1 can also perform DECRYPT_EEK operations
+ on 'testKey3'
+-->
+ <property>
+ <name>whitelist.key.acl.DECRYPT_EEK</name>
+ <value>admin1</value>
+ <description>
+ whitelist ACL for DECRYPT_EEK operations for all keys.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.MANAGEMENT</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for MANAGEMENT operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.GENERATE_EEK</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for GENERATE_EEK operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.DECRYPT_EEK</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for DECRYPT_EEK operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+
+ <property>
+ <name>default.key.acl.READ</name>
+ <value>user1,user2</value>
+ <description>
+ default ACL for READ operations for all keys that are not
+ explicitly defined.
+ </description>
+ </property>
+```
+
+$H3 KMS Delegation Token Configuration
+
+KMS delegation token secret manager can be configured with the following properties:
+
+```xml
+ <property>
+ <name>hadoop.kms.authentication.delegation-token.update-interval.sec</name>
+ <value>86400</value>
+ <description>
+ How often the master key is rotated, in seconds. Default value 1 day.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.delegation-token.max-lifetime.sec</name>
+ <value>604800</value>
+ <description>
+ Maximum lifetime of a delagation token, in seconds. Default value 7 days.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.delegation-token.renew-interval.sec</name>
+ <value>86400</value>
+ <description>
+ Renewal interval of a delagation token, in seconds. Default value 1 day.
+ </description>
+ </property>
+
+ <property>
+ <name>hadoop.kms.authentication.delegation-token.removal-scan-interval.sec</name>
+ <value>3600</value>
+ <description>
+ Scan interval to remove expired delegation tokens.
+ </description>
+ </property>
+```
+
+$H3 Using Multiple Instances of KMS Behind a Load-Balancer or VIP
+
+KMS supports multiple KMS instances behind a load-balancer or VIP for scalability and for HA purposes.
+
+When using multiple KMS instances behind a load-balancer or VIP, requests from the same user may be handled by different KMS instances.
+
+KMS instances behind a load-balancer or VIP must be specially configured to work properly as a single logical service.
+
+$H4 HTTP Kerberos Principals Configuration
+
+When KMS instances are behind a load-balancer or VIP, clients will use the hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the URL is used to construct the Kerberos service name of the server, `HTTP/#HOSTNAME#`. This means that all KMS instances must have a Kerberos service name with the load-balancer or VIP hostname.
+
+In order to be able to access directly a specific KMS instance, the KMS instance must also have Keberos service name with its own hostname. This is required for monitoring and admin purposes.
+
+Both Kerberos service principal credentials (for the load-balancer/VIP hostname and for the actual KMS instance hostname) must be in the keytab file configured for authentication. And the principal name specified in the configuration must be '\*'. For example:
+
+```xml
+ <property>
+ <name>hadoop.kms.authentication.kerberos.principal</name>
+ <value>*</value>
+ </property>
+```
+
+**NOTE:** If using HTTPS, the SSL certificate used by the KMS instance must be configured to support multiple hostnames (see Java 7 `keytool` SAN extension support for details on how to do this).
+
+$H4 HTTP Authentication Signature
+
+KMS uses Hadoop Authentication for HTTP authentication. Hadoop Authentication issues a signed HTTP Cookie once the client has authenticated successfully. This HTTP Cookie has an expiration time, after which it will trigger a new authentication sequence. This is done to avoid triggering the authentication on every HTTP request of a client.
+
+A KMS instance must verify the HTTP Cookie signatures signed by other KMS instances. To do this all KMS instances must share the signing secret.
+
+This secret sharing can be done using a Zookeeper service which is configured in KMS with the following properties in the `kms-site.xml`:
+
+```xml
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider</name>
+ <value>zookeeper</value>
+ <description>
+ Indicates how the secret to sign the authentication cookies will be
+ stored. Options are 'random' (default), 'string' and 'zookeeper'.
+ If using a setup with multiple KMS instances, 'zookeeper' should be used.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.path</name>
+ <value>/hadoop-kms/hadoop-auth-signature-secret</value>
+ <description>
+ The Zookeeper ZNode path where the KMS instances will store and retrieve
+ the secret from.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string</name>
+ <value>#HOSTNAME#:#PORT#,...</value>
+ <description>
+ The Zookeeper connection string, a list of hostnames and port comma
+ separated.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type</name>
+ <value>kerberos</value>
+ <description>
+ The Zookeeper authentication type, 'none' or 'sasl' (Kerberos).
+ </description>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name>
+ <value>/etc/hadoop/conf/kms.keytab</value>
+ <description>
+ The absolute path for the Kerberos keytab with the credentials to
+ connect to Zookeeper.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name>
+ <value>kms/#HOSTNAME#</value>
+ <description>
+ The Kerberos service principal used to connect to Zookeeper.
+ </description>
+ </property>
+```
+
+$H4 Delegation Tokens
+
+TBD
+
+$H3 KMS HTTP REST API
+
+$H4 Create a Key
+
+*REQUEST:*
+
+ POST http://HOST:PORT/kms/v1/keys
+ Content-Type: application/json
+
+ {
+ "name" : "<key-name>",
+ "cipher" : "<cipher>",
+ "length" : <length>, //int
+ "material" : "<material>", //base64
+ "description" : "<description>"
+ }
+
+*RESPONSE:*
+
+ 201 CREATED
+ LOCATION: http://HOST:PORT/kms/v1/key/<key-name>
+ Content-Type: application/json
+
+ {
+ "name" : "versionName",
+ "material" : "<material>", //base64, not present without GET ACL
+ }
+
+$H4 Rollover Key
+
+*REQUEST:*
+
+ POST http://HOST:PORT/kms/v1/key/<key-name>
+ Content-Type: application/json
+
+ {
+ "material" : "<material>",
+ }
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ {
+ "name" : "versionName",
+ "material" : "<material>", //base64, not present without GET ACL
+ }
+
+$H4 Delete Key
+
+*REQUEST:*
+
+ DELETE http://HOST:PORT/kms/v1/key/<key-name>
+
+*RESPONSE:*
+
+ 200 OK
+
+$H4 Get Key Metadata
+
+*REQUEST:*
+
+ GET http://HOST:PORT/kms/v1/key/<key-name>/_metadata
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ {
+ "name" : "<key-name>",
+ "cipher" : "<cipher>",
+ "length" : <length>, //int
+ "description" : "<description>",
+ "created" : <millis-epoc>, //long
+ "versions" : <versions> //int
+ }
+
+$H4 Get Current Key
+
+*REQUEST:*
+
+ GET http://HOST:PORT/kms/v1/key/<key-name>/_currentversion
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ {
+ "name" : "versionName",
+ "material" : "<material>", //base64
+ }
+
+$H4 Generate Encrypted Key for Current KeyVersion
+
+*REQUEST:*
+
+ GET http://HOST:PORT/kms/v1/key/<key-name>/_eek?eek_op=generate&num_keys=<number-of-keys-to-generate>
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+ [
+ {
+ "versionName" : "encryptionVersionName",
+ "iv" : "<iv>", //base64
+ "encryptedKeyVersion" : {
+ "versionName" : "EEK",
+ "material" : "<material>", //base64
+ }
+ },
+ {
+ "versionName" : "encryptionVersionName",
+ "iv" : "<iv>", //base64
+ "encryptedKeyVersion" : {
+ "versionName" : "EEK",
+ "material" : "<material>", //base64
+ }
+ },
+ ...
+ ]
+
+$H4 Decrypt Encrypted Key
+
+*REQUEST:*
+
+ POST http://HOST:PORT/kms/v1/keyversion/<version-name>/_eek?ee_op=decrypt
+ Content-Type: application/json
+
+ {
+ "name" : "<key-name>",
+ "iv" : "<iv>", //base64
+ "material" : "<material>", //base64
+ }
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ {
+ "name" : "EK",
+ "material" : "<material>", //base64
+ }
+
+$H4 Get Key Version
+
+*REQUEST:*
+
+ GET http://HOST:PORT/kms/v1/keyversion/<version-name>
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ {
+ "name" : "versionName",
+ "material" : "<material>", //base64
+ }
+
+$H4 Get Key Versions
+
+*REQUEST:*
+
+ GET http://HOST:PORT/kms/v1/key/<key-name>/_versions
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ [
+ {
+ "name" : "versionName",
+ "material" : "<material>", //base64
+ },
+ {
+ "name" : "versionName",
+ "material" : "<material>", //base64
+ },
+ ...
+ ]
+
+$H4 Get Key Names
+
+*REQUEST:*
+
+ GET http://HOST:PORT/kms/v1/keys/names
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ [
+ "<key-name>",
+ "<key-name>",
+ ...
+ ]
+
+$H4 Get Keys Metadata
+
+ GET http://HOST:PORT/kms/v1/keys/metadata?key=<key-name>&key=<key-name>,...
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ [
+ {
+ "name" : "<key-name>",
+ "cipher" : "<cipher>",
+ "length" : <length>, //int
+ "description" : "<description>",
+ "created" : <millis-epoc>, //long
+ "versions" : <versions> //int
+ },
+ {
+ "name" : "<key-name>",
+ "cipher" : "<cipher>",
+ "length" : <length>, //int
+ "description" : "<description>",
+ "created" : <millis-epoc>, //long
+ "versions" : <versions> //int
+ },
+ ...
+ ]
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-project/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/apt/index.apt.vm b/hadoop-project/src/site/apt/index.apt.vm
deleted file mode 100644
index 4167f4d..0000000
--- a/hadoop-project/src/site/apt/index.apt.vm
+++ /dev/null
@@ -1,73 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Apache Hadoop ${project.version}
- ---
- ---
- ${maven.build.timestamp}
-
-Apache Hadoop ${project.version}
-
- Apache Hadoop ${project.version} consists of significant
- improvements over the previous stable release (hadoop-1.x).
-
- Here is a short overview of the improvments to both HDFS and MapReduce.
-
- * {HDFS Federation}
-
- In order to scale the name service horizontally, federation uses multiple
- independent Namenodes/Namespaces. The Namenodes are federated, that is, the
- Namenodes are independent and don't require coordination with each other.
- The datanodes are used as common storage for blocks by all the Namenodes.
- Each datanode registers with all the Namenodes in the cluster. Datanodes
- send periodic heartbeats and block reports and handles commands from the
- Namenodes.
-
- More details are available in the
- {{{./hadoop-project-dist/hadoop-hdfs/Federation.html}HDFS Federation}}
- document.
-
- * {MapReduce NextGen aka YARN aka MRv2}
-
- The new architecture introduced in hadoop-0.23, divides the two major
- functions of the JobTracker: resource management and job life-cycle management
- into separate components.
-
- The new ResourceManager manages the global assignment of compute resources to
- applications and the per-application ApplicationMaster manages the
- application‚ scheduling and coordination.
-
- An application is either a single job in the sense of classic MapReduce jobs
- or a DAG of such jobs.
-
- The ResourceManager and per-machine NodeManager daemon, which manages the
- user processes on that machine, form the computation fabric.
-
- The per-application ApplicationMaster is, in effect, a framework specific
- library and is tasked with negotiating resources from the ResourceManager and
- working with the NodeManager(s) to execute and monitor the tasks.
-
- More details are available in the
- {{{./hadoop-yarn/hadoop-yarn-site/YARN.html}YARN}}
- document.
-
-Getting Started
-
- The Hadoop documentation includes the information you need to get started using
- Hadoop. Begin with the
- {{{./hadoop-project-dist/hadoop-common/SingleCluster.html}Single Node Setup}} which
- shows you how to set up a single-node Hadoop installation. Then move on to the
- {{{./hadoop-project-dist/hadoop-common/ClusterSetup.html}Cluster Setup}} to learn how
- to set up a multi-node Hadoop installation.
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-project/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/markdown/index.md.vm b/hadoop-project/src/site/markdown/index.md.vm
new file mode 100644
index 0000000..c3a93ad
--- /dev/null
+++ b/hadoop-project/src/site/markdown/index.md.vm
@@ -0,0 +1,72 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Apache Hadoop ${project.version}
+================================
+
+Apache Hadoop ${project.version} consists of significant
+improvements over the previous stable release (hadoop-1.x).
+
+Here is a short overview of the improvments to both HDFS and MapReduce.
+
+* HDFS Federation
+
+ In order to scale the name service horizontally, federation uses
+ multiple independent Namenodes/Namespaces. The Namenodes are
+ federated, that is, the Namenodes are independent and don't require
+ coordination with each other. The datanodes are used as common storage
+ for blocks by all the Namenodes. Each datanode registers with all the
+ Namenodes in the cluster. Datanodes send periodic heartbeats and block
+ reports and handles commands from the Namenodes.
+
+ More details are available in the
+ [HDFS Federation](./hadoop-project-dist/hadoop-hdfs/Federation.html)
+ document.
+
+* MapReduce NextGen aka YARN aka MRv2
+
+ The new architecture introduced in hadoop-0.23, divides the two major
+ functions of the JobTracker: resource management and job life-cycle
+ management into separate components.
+
+ The new ResourceManager manages the global assignment of compute
+ resources to applications and the per-application
+ ApplicationMaster manages the application‚ scheduling and
+ coordination.
+
+ An application is either a single job in the sense of classic
+ MapReduce jobs or a DAG of such jobs.
+
+ The ResourceManager and per-machine NodeManager daemon, which
+ manages the user processes on that machine, form the computation
+ fabric.
+
+ The per-application ApplicationMaster is, in effect, a framework
+ specific library and is tasked with negotiating resources from the
+ ResourceManager and working with the NodeManager(s) to execute and
+ monitor the tasks.
+
+ More details are available in the
+ [YARN](./hadoop-yarn/hadoop-yarn-site/YARN.html) document.
+
+Getting Started
+===============
+
+The Hadoop documentation includes the information you need to get started using
+Hadoop. Begin with the
+[Single Node Setup](./hadoop-project-dist/hadoop-common/SingleCluster.html)
+which shows you how to set up a single-node Hadoop installation.
+Then move on to the
+[Cluster Setup](./hadoop-project-dist/hadoop-common/ClusterSetup.html)
+to learn how to set up a multi-node Hadoop installation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm b/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm
deleted file mode 100644
index 29fbd33..0000000
--- a/hadoop-tools/hadoop-openstack/src/site/apt/index.apt.vm
+++ /dev/null
@@ -1,686 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop OpenStack Support: Swift Object Store
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc|section=1|fromDepth=0}
-
-Hadoop OpenStack Support: Swift Object Store
-
-* {Introduction}
-
- {{{http://www.openstack.org/}OpenStack}} is an open source cloud infrastructure
- which can be accessed
- from multiple public IaaS providers, and deployed privately. It offers
- infrastructure services such as VM hosting (Nova), authentication (Keystone)
- and storage of binary objects (Swift).
-
- This module enables Apache Hadoop applications -including MapReduce jobs,
- read and write data to and from instances of the
- {{{http://www.openstack.org/software/openstack-storage/}OpenStack Swift object store}}.
-
-* Features
-
- * Read and write of data stored in a Swift object store
-
- * Support of a pseudo-hierachical file system (directories, subdirectories and
- files)
-
- * Standard filesystem operations: <<<create>>>, <<<delete>>>, <<<mkdir>>>,
- <<<ls>>>, <<<mv>>>, <<<stat>>>.
-
- * Can act as a source of data in a MapReduce job, or a sink.
-
- * Support for multiple OpenStack services, and multiple containers from a
- single service.
-
- * Supports in-cluster and remote access to Swift data.
-
- * Supports OpenStack Keystone authentication with password or token.
-
- * Released under the Apache Software License
-
- * Tested against the Hadoop 3.x and 1.x branches, against multiple public
- OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
-
- * Tested against private OpenStack clusters, including scalability tests of
- large file uploads.
-
-* Using the Hadoop Swift Filesystem Client
-
-** Concepts: services and containers
-
- OpenStack swift is an <Object Store>; also known as a <blobstore>. It stores
- arbitrary binary objects by name in a <container>.
-
- The Hadoop Swift filesystem library adds another concept, the <service>, which
- defines which Swift blobstore hosts a container -and how to connect to it.
-
-** Containers and Objects
-
- * Containers are created by users with accounts on the Swift filestore, and hold
- <objects>.
-
- * Objects can be zero bytes long, or they can contain data.
-
- * Objects in the container can be up to 5GB; there is a special support for
- larger files than this, which merges multiple objects in to one.
-
- * Each object is referenced by it's <name>; there is no notion of directories.
-
- * You can use any characters in an object name that can be 'URL-encoded'; the
- maximum length of a name is 1034 characters -after URL encoding.
-
- * Names can have <<</>>> characters in them, which are used to create the illusion of
- a directory structure. For example <<<dir/dir2/name>>>. Even though this looks
- like a directory, <it is still just a name>. There is no requirement to have
- any entries in the container called <<<dir>>> or <<<dir/dir2>>>
-
- * That said. if the container has zero-byte objects that look like directory
- names above other objects, they can pretend to be directories. Continuing the
- example, a 0-byte object called <<<dir>>> would tell clients that it is a
- directory while <<<dir/dir2>>> or <<<dir/dir2/name>>> were present. This creates an
- illusion of containers holding a filesystem.
-
- Client applications talk to Swift over HTTP or HTTPS, reading, writing and
- deleting objects using standard HTTP operations (GET, PUT and DELETE,
- respectively). There is also a COPY operation, that creates a new object in the
- container, with a new name, containing the old data. There is no rename
- operation itself, objects need to be copied -then the original entry deleted.
-
-** Eventual Consistency
-
- The Swift Filesystem is *eventually consistent*: an operation on an object may
- not be immediately visible to that client, or other clients. This is a
- consequence of the goal of the filesystem: to span a set of machines, across
- multiple datacenters, in such a way that the data can still be available when
- many of them fail. (In contrast, the Hadoop HDFS filesystem is *immediately
- consistent*, but it does not span datacenters.)
-
- Eventual consistency can cause surprises for client applications that expect
- immediate consistency: after an object is deleted or overwritten, the object
- may still be visible -or the old data still retrievable. The Swift Filesystem
- client for Apache Hadoop attempts to handle this, in conjunction with the
- MapReduce engine, but there may be still be occasions when eventual consistency
- causes surprises.
-
-** Non-atomic "directory" operations.
-
- Hadoop expects some
- operations to be atomic, especially <<<rename()>>>, which is something
- the MapReduce layer relies on to commit the output of a job, renaming data
- from a temp directory to the final path. Because a rename
- is implemented as a copy of every blob under the directory's path, followed
- by a delete of the originals, the intermediate state of the operation
- will be visible to other clients. If two Reducer tasks to rename their temp
- directory to the final path, both operations may succeed, with the result that
- output directory contains mixed data. This can happen if MapReduce jobs
- are being run with <speculation> enabled and Swift used as the direct output
- of the MR job (it can also happen against Amazon S3).
-
- Other consequences of the non-atomic operations are:
-
- 1. If a program is looking for the presence of the directory before acting
- on the data -it may start prematurely. This can be avoided by using
- other mechanisms to co-ordinate the programs, such as the presence of a file
- that is written <after> any bulk directory operations.
-
- 2. A <<<rename()>>> or <<<delete()>>> operation may include files added under
- the source directory tree during the operation, may unintentionally delete
- it, or delete the 0-byte swift entries that mimic directories and act
- as parents for the files. Try to avoid doing this.
-
- The best ways to avoid all these problems is not using Swift as
- the filesystem between MapReduce jobs or other Hadoop workflows. It
- can act as a source of data, and a final destination, but it doesn't meet
- all of Hadoop's expectations of what a filesystem is -it's a <blobstore>.
-
-* Working with Swift Object Stores in Hadoop
-
- Once installed, the Swift FileSystem client can be used by any Hadoop application
- to read from or write to data stored in a Swift container.
-
- Data stored in Swift can be used as the direct input to a MapReduce job
- -simply use the <<<swift:>>> URL (see below) to declare the source of the data.
-
- This Swift Filesystem client is designed to work with multiple
- Swift object stores, both public and private. This allows the client to work
- with different clusters, reading and writing data to and from either of them.
-
- It can also work with the same object stores using multiple login details.
-
- These features are achieved by one basic concept: using a service name in
- the URI referring to a swift filesystem, and looking up all the connection and
- login details for that specific service. Different service names can be defined
- in the Hadoop XML configuration file, so defining different clusters, or
- providing different login details for the same object store(s).
-
-
-** Swift Filesystem URIs
-
- Hadoop uses URIs to refer to files within a filesystem. Some common examples
- are:
-
-+--
- local://etc/hosts
- hdfs://cluster1/users/example/data/set1
- hdfs://cluster2.example.org:8020/users/example/data/set1
-+--
-
- The Swift Filesystem Client adds a new URL type <<<swift>>>. In a Swift Filesystem
- URL, the hostname part of a URL identifies the container and the service to
- work with; the path the name of the object. Here are some examples
-
-+--
- swift://container.rackspace/my-object.csv
- swift://data.hpcloud/data/set1
- swift://dmitry.privatecloud/out/results
-+--
-
- In the last two examples, the paths look like directories: it is not, they are
- simply the objects named <<<data/set1>>> and <<<out/results>>> respectively.
-
-** Installing
-
- The <<<hadoop-openstack>>> JAR must be on the classpath of the Hadoop program trying to
- talk to the Swift service. If installed in the classpath of the Hadoop
- MapReduce service, then all programs started by the MR engine will pick up the
- JAR automatically. This is the easiest way to give all Hadoop jobs access to
- Swift.
-
- Alternatively, the JAR can be included as one of the JAR files that an
- application uses. This lets the Hadoop jobs work with a Swift object store even
- if the Hadoop cluster is not pre-configured for this.
-
- The library also depends upon the Apache HttpComponents library, which
- must also be on the classpath.
-
-** Configuring
-
- To talk to a swift service, the user must must provide:
-
- [[1]] The URL defining the container and the service.
-
- [[1]] In the cluster/job configuration, the login details of that service.
-
- Multiple service definitions can co-exist in the same configuration file: just
- use different names for them.
-
-*** Example: Rackspace US, in-cluster access using API key
-
- This service definition is for use in a Hadoop cluster deployed within Rackspace's
- US infrastructure.
-
-+--
- <property>
- <name>fs.swift.service.rackspace.auth.url</name>
- <value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
- <description>Rackspace US (multiregion)</description>
- </property>
-
- <property>
- <name>fs.swift.service.rackspace.username</name>
- <value>user4</value>
- </property>
-
- <property>
- <name>fs.swift.service.rackspace.region</name>
- <value>DFW</value>
- </property>
-
- <property>
- <name>fs.swift.service.rackspace.apikey</name>
- <value>fe806aa86dfffe2f6ed8</value>
- </property>
-+--
-
- Here the API key visible in the account settings API keys page is used to log
- in. No property for public/private access -the default is to use the private
- endpoint for Swift operations.
-
- This configuration also selects one of the regions, DFW, for its data.
-
- A reference to this service would use the <<<rackspace>>> service name:
-
----
- swift://hadoop-container.rackspace/
----
-
-*** Example: Rackspace UK: remote access with password authentication
-
- This connects to Rackspace's UK ("LON") datacenter.
-
-+--
- <property>
- <name>fs.swift.service.rackspaceuk.auth.url</name>
- <value>https://lon.identity.api.rackspacecloud.com/v2.0/tokens</value>
- <description>Rackspace UK</description>
- </property>
-
- <property>
- <name>fs.swift.service.rackspaceuk.username</name>
- <value>user4</value>
- </property>
-
- <property>
- <name>fs.swift.service.rackspaceuk.password</name>
- <value>insert-password-here/value>
- </property>
-
- <property>
- <name>fs.swift.service.rackspace.public</name>
- <value>true</value>
- </property>
-+--
-
- This is a public access point connection, using a password over an API key.
-
- A reference to this service would use the <<<rackspaceuk>>> service name:
-
-+--
- swift://hadoop-container.rackspaceuk/
-+--
-
- Because the public endpoint is used, if this service definition is used within
- the London datacenter, all accesses will be billed at the public
- upload/download rates, <irrespective of where the Hadoop cluster is>.
-
-*** Example: HP cloud service definition
-
- Here is an example that connects to the HP Cloud object store.
-
-+--
- <property>
- <name>fs.swift.service.hpcloud.auth.url</name>
- <value>https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
- </value>
- <description>HP Cloud</description>
- </property>
-
- <property>
- <name>fs.swift.service.hpcloud.tenant</name>
- <value>FE806AA86</value>
- </property>
-
- <property>
- <name>fs.swift.service.hpcloud.username</name>
- <value>FE806AA86DFFFE2F6ED8</value>
- </property>
-
- <property>
- <name>fs.swift.service.hpcloud.password</name>
- <value>secret-password-goes-here</value>
- </property>
-
- <property>
- <name>fs.swift.service.hpcloud.public</name>
- <value>true</value>
- </property>
-+--
-
- A reference to this service would use the <<<hpcloud>>> service name:
-
-+--
- swift://hadoop-container.hpcloud/
-+--
-
-** General Swift Filesystem configuration options
-
- Some configuration options apply to the Swift client, independent of
- the specific Swift filesystem chosen.
-
-*** Blocksize fs.swift.blocksize
-
- Swift does not break up files into blocks, except in the special case of files
- over 5GB in length. Accordingly, there isn't a notion of a "block size"
- to define where the data is kept.
-
- Hadoop's MapReduce layer depends on files declaring their block size,
- so that it knows how to partition work. Too small a blocksize means that
- many mappers work on small pieces of data; too large a block size means
- that only a few mappers get started.
-
- The block size value reported by Swift, therefore, controls the basic workload
- partioning of the MapReduce engine -and can be an important parameter to
- tune for performance of the cluster.
-
- The property has a unit of kilobytes; the default value is <<<32*1024>>>: 32 MB
-
-+--
- <property>
- <name>fs.swift.blocksize</name>
- <value>32768</value>
- </property>
-+--
-
- This blocksize has no influence on how files are stored in Swift; it only controls
- what the reported size of blocks are - a value used in Hadoop MapReduce to
- divide work.
-
- Note that the MapReduce engine's split logic can be tuned independently by setting
- the <<<mapred.min.split.size>>> and <<<mapred.max.split.size>>> properties,
- which can be done in specific job configurations.
-
-+--
- <property>
- <name>mapred.min.split.size</name>
- <value>524288</value>
- </property>
-
- <property>
- <name>mapred.max.split.size</name>
- <value>1048576</value>
- </property>
-+--
-
- In an Apache Pig script, these properties would be set as:
-
----
- mapred.min.split.size 524288
- mapred.max.split.size 1048576
----
-
-*** Partition size fs.swift.partsize
-
- The Swift filesystem client breaks very large files into partitioned files,
- uploading each as it progresses, and writing any remaning data and an XML
- manifest when a partitioned file is closed.
-
- The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that
- Swift can support.
-
- It is possible to set a smaller partition size, in the <<<fs.swift.partsize>>>
- option. This takes a value in KB.
-
-+--
- <property>
- <name>fs.swift.partsize</name>
- <value>1024</value>
- <description>upload every MB</description>
- </property>
-+--
-
- When should this value be changed from its default?
-
- While there is no need to ever change it for basic operation of
- the Swift filesystem client, it can be tuned
-
- * If a Swift filesystem is location aware, then breaking a file up into
- smaller partitions scatters the data round the cluster. For best performance,
- the property <<<fs.swift.blocksize>>> should be set to a smaller value than the
- partition size of files.
-
- * When writing to an unpartitioned file, the entire write is done in the
- <<<close()>>> operation. When a file is partitioned, the outstanding data to
- be written whenever the outstanding amount of data is greater than the
- partition size. This means that data will be written more incrementally
-
-*** Request size fs.swift.requestsize
-
- The Swift filesystem client reads files in HTTP GET operations, asking for
- a block of data at a time.
-
- The default value is 64KB. A larger value may be more efficient over faster
- networks, as it reduces the overhead of setting up the HTTP operation.
-
- However, if the file is read with many random accesses, requests for
- data will be made from different parts of the file -discarding some of the
- previously requested data. The benefits of larger request sizes may be wasted.
-
- The property <<<fs.swift.requestsize>>> sets the request size in KB.
-
-+--
- <property>
- <name>fs.swift.requestsize</name>
- <value>128</value>
- </property>
-+--
-
-*** Connection timeout fs.swift.connect.timeout
-
- This sets the timeout in milliseconds to connect to a Swift service.
-
-+--
- <property>
- <name>fs.swift.connect.timeout</name>
- <value>15000</value>
- </property>
-+--
-
- A shorter timeout means that connection failures are raised faster -but
- may trigger more false alarms. A longer timeout is more resilient to network
- problems -and may be needed when talking to remote filesystems.
-
-*** Connection timeout fs.swift.socket.timeout
-
- This sets the timeout in milliseconds to wait for data from a connected socket.
-
-+--
- <property>
- <name>fs.swift.socket.timeout</name>
- <value>60000</value>
- </property>
-+--
-
- A shorter timeout means that connection failures are raised faster -but
- may trigger more false alarms. A longer timeout is more resilient to network
- problems -and may be needed when talking to remote filesystems.
-
-*** Connection Retry Count fs.swift.connect.retry.count
-
- This sets the number of times to try to connect to a service whenever
- an HTTP request is made.
-
-+--
- <property>
- <name>fs.swift.connect.retry.count</name>
- <value>3</value>
- </property>
-+--
-
- The more retries, the more resilient it is to transient outages -and the
- less rapid it is at detecting and reporting server connectivity problems.
-
-*** Connection Throttle Delay fs.swift.connect.throttle.delay
-
- This property adds a delay between bulk file copy and delete operations,
- to prevent requests being throttled or blocked by the remote service
-
-+--
- <property>
- <name>fs.swift.connect.throttle.delay</name>
- <value>0</value>
- </property>
-+--
-
- It is measured in milliseconds; "0" means do not add any delay.
-
- Throttling is enabled on the public endpoints of some Swift services.
- If <<<rename()>>> or <<<delete()>>> operations fail with
- <<<SwiftThrottledRequestException>>>
- exceptions, try setting this property.
-
-*** HTTP Proxy
-
- If the client can only access the Swift filesystem via a web proxy
- server, the client configuration must specify the proxy via
- the <<<fs.swift.connect.proxy.host>>> and <<<fs.swift.connect.proxy.port>>>
- properties.
-
-+--
- <property>
- <name>fs.swift.proxy.host</name>
- <value>web-proxy</value>
- </property>
-
- <property>
- <name>fs.swift.proxy.port</name>
- <value>8088</value>
- </property>
-+--
-
- If the host is declared, the proxy port must be set to a valid integer value.
-
-
-** Troubleshooting
-
-*** ClassNotFoundException
-
- The <<<hadoop-openstack>>> JAR -or any dependencies- may not be on your classpath.
-
- If it is a remote MapReduce job that is failing, make sure that the JAR is
- installed on the servers in the cluster -or that the job submission process
- uploads the JAR file to the distributed cache.
-
-*** Failure to Authenticate
-
- A <<<SwiftAuthenticationFailedException>>> is thrown when the client
- cannot authenticate with the OpenStack keystone server. This could be
- because the URL in the service definition is wrong, or because
- the supplied credentials are invalid.
-
- [[1]] Check the authentication URL through <<<curl>>> or your browser
-
- [[1]] Use a Swift client such as CyberDuck to validate your credentials
-
- [[1]] If you have included a tenant ID, try leaving it out. Similarly,
- try adding it if you had not included it.
-
- [[1]] Try switching from API key authentication to password-based authentication,
- by setting the password.
-
- [[1]] Change your credentials. As with Amazon AWS clients, some credentials
- don't seem to like going over the network.
-
-*** Timeout connecting to the Swift Service
-
- This happens if the client application is running outside an OpenStack cluster,
- where it does not have access to the private hostname/IP address for filesystem
- operations. Set the <<<public>>> flag to true -but remember to set it to false
- for use in-cluster.
-
-** Warnings
-
- [[1]] Do not share your login details with anyone, which means do not log the
- details, or check the XML configuration files into any revision control system
- to which you do not have exclusive access.
-
- [[1]] Similarly, do not use your real account details in any documentation *or any
- bug reports submitted online*
-
- [[1]] Prefer the apikey authentication over passwords as it is easier
- to revoke a key -and some service providers allow you to set
- an automatic expiry date on a key when issued.
-
- [[1]] Do not use the public service endpoint from within a public OpenStack
- cluster, as it will run up large bills.
-
- [[1]] Remember: it's not a real filesystem or hierarchical directory structure.
- Some operations (directory rename and delete) take time and are not atomic or
- isolated from other operations taking place.
-
- [[1]] Append is not supported.
-
- [[1]] Unix-style permissions are not supported. All accounts with write access to
- a repository have unlimited access; the same goes for those with read access.
-
- [[1]] In the public clouds, do not make the containers public unless you are happy
- with anyone reading your data, and are prepared to pay the costs of their
- downloads.
-
-** Limits
-
- * Maximum length of an object path: 1024 characters
-
- * Maximum size of a binary object: no absolute limit. Files > 5GB are
- partitioned into separate files in the native filesystem, and merged during
- retrieval. <Warning:> the partitioned/large file support is the
- most complex part of the Hadoop/Swift FS integration, and, along with
- authentication, the most troublesome to support.
-
-** Testing the hadoop-openstack module
-
- The <<<hadoop-openstack>>> can be remotely tested against any public
- or private cloud infrastructure which supports the OpenStack Keystone
- authentication mechanism. It can also be tested against private
- OpenStack clusters. OpenStack Development teams are strongly encouraged to test
- the Hadoop swift filesystem client against any version of Swift that they
- are developing or deploying, to stress their cluster and to identify
- bugs early.
-
- The module comes with a large suite of JUnit tests -tests that are
- only executed if the source tree includes credentials to test against a
- specific cluster.
-
- After checking out the Hadoop source tree, create the file:
-
-+--
- hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
-+--
-
- Into this file, insert the credentials needed to bond to the test filesystem,
- as decribed above.
-
- Next set the property <<<test.fs.swift.name>>> to the URL of a
- swift container to test against. The tests expect exclusive access
- to this container -do not keep any other data on it, or expect it
- to be preserved.
-
-+--
- <property>
- <name>test.fs.swift.name</name>
- <value>swift://test.myswift/</value>
- </property>
-+--
-
- In the base hadoop directory, run:
-
-+--
- mvn clean install -DskipTests
-+--
-
- This builds a set of Hadoop JARs consistent with the <<<hadoop-openstack>>>
- module that is about to be tested.
-
- In the <<<hadoop-tools/hadoop-openstack>>> directory run
-
-+--
- mvn test -Dtest=TestSwiftRestClient
-+--
-
- This runs some simple tests which include authenticating
- against the remote swift service. If these tests fail, so will all
- the rest. If it does fail: check your authentication.
-
- Once this test succeeds, you can run the full test suite
-
-+--
- mvn test
-+--
-
- Be advised that these tests can take an hour or more, especially against a
- remote Swift service -or one that throttles bulk operations.
-
- Once the <<<auth-keys.xml>>> file is in place, the <<<mvn test>>> runs from
- the Hadoop source base directory will automatically run these OpenStack tests
- While this ensures that no regressions have occurred, it can also add significant
- time to test runs, and may run up bills, depending on who is providing\
- the Swift storage service. We recommend having a separate source tree
- set up purely for the Swift tests, and running it manually or by the CI tooling
- at a lower frequency than normal test runs.
-
- Finally: Apache Hadoop is an open source project. Contributions of code
- -including more tests- are very welcome.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-openstack/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/site/markdown/index.md b/hadoop-tools/hadoop-openstack/src/site/markdown/index.md
new file mode 100644
index 0000000..0eeb274
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/site/markdown/index.md
@@ -0,0 +1,544 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Hadoop OpenStack Support: Swift Object Store](#Hadoop_OpenStack_Support:_Swift_Object_Store)
+ * [Introduction](#Introduction)
+ * [Features](#Features)
+ * [Using the Hadoop Swift Filesystem Client](#Using_the_Hadoop_Swift_Filesystem_Client)
+ * [Concepts: services and containers](#Concepts:_services_and_containers)
+ * [Containers and Objects](#Containers_and_Objects)
+ * [Eventual Consistency](#Eventual_Consistency)
+ * [Non-atomic "directory" operations.](#Non-atomic_directory_operations.)
+ * [Working with Swift Object Stores in Hadoop](#Working_with_Swift_Object_Stores_in_Hadoop)
+ * [Swift Filesystem URIs](#Swift_Filesystem_URIs)
+ * [Installing](#Installing)
+ * [Configuring](#Configuring)
+ * [Example: Rackspace US, in-cluster access using API key](#Example:_Rackspace_US_in-cluster_access_using_API_key)
+ * [Example: Rackspace UK: remote access with password authentication](#Example:_Rackspace_UK:_remote_access_with_password_authentication)
+ * [Example: HP cloud service definition](#Example:_HP_cloud_service_definition)
+ * [General Swift Filesystem configuration options](#General_Swift_Filesystem_configuration_options)
+ * [Blocksize fs.swift.blocksize](#Blocksize_fs.swift.blocksize)
+ * [Partition size fs.swift.partsize](#Partition_size_fs.swift.partsize)
+ * [Request size fs.swift.requestsize](#Request_size_fs.swift.requestsize)
+ * [Connection timeout fs.swift.connect.timeout](#Connection_timeout_fs.swift.connect.timeout)
+ * [Connection timeout fs.swift.socket.timeout](#Connection_timeout_fs.swift.socket.timeout)
+ * [Connection Retry Count fs.swift.connect.retry.count](#Connection_Retry_Count_fs.swift.connect.retry.count)
+ * [Connection Throttle Delay fs.swift.connect.throttle.delay](#Connection_Throttle_Delay_fs.swift.connect.throttle.delay)
+ * [HTTP Proxy](#HTTP_Proxy)
+ * [Troubleshooting](#Troubleshooting)
+ * [ClassNotFoundException](#ClassNotFoundException)
+ * [Failure to Authenticate](#Failure_to_Authenticate)
+ * [Timeout connecting to the Swift Service](#Timeout_connecting_to_the_Swift_Service)
+ * [Warnings](#Warnings)
+ * [Limits](#Limits)
+ * [Testing the hadoop-openstack module](#Testing_the_hadoop-openstack_module)
+
+Hadoop OpenStack Support: Swift Object Store
+============================================
+
+Introduction
+------------
+
+[OpenStack](http://www.openstack.org/) is an open source cloud infrastructure which can be accessed from multiple public IaaS providers, and deployed privately. It offers infrastructure services such as VM hosting (Nova), authentication (Keystone) and storage of binary objects (Swift).
+
+This module enables Apache Hadoop applications -including MapReduce jobs, read and write data to and from instances of the [OpenStack Swift object store](http://www.openstack.org/software/openstack-storage/).
+
+Features
+--------
+
+* Read and write of data stored in a Swift object store
+
+* Support of a pseudo-hierachical file system (directories, subdirectories and
+ files)
+
+* Standard filesystem operations: `create`, `delete`, `mkdir`,
+ `ls`, `mv`, `stat`.
+
+* Can act as a source of data in a MapReduce job, or a sink.
+
+* Support for multiple OpenStack services, and multiple containers from a
+ single service.
+
+* Supports in-cluster and remote access to Swift data.
+
+* Supports OpenStack Keystone authentication with password or token.
+
+* Released under the Apache Software License
+
+* Tested against the Hadoop 3.x and 1.x branches, against multiple public
+ OpenStack clusters: Rackspace US, Rackspace UK, HP Cloud.
+
+* Tested against private OpenStack clusters, including scalability tests of
+ large file uploads.
+
+Using the Hadoop Swift Filesystem Client
+----------------------------------------
+
+### Concepts: services and containers
+
+OpenStack swift is an *Object Store*; also known as a *blobstore*. It stores arbitrary binary objects by name in a *container*.
+
+The Hadoop Swift filesystem library adds another concept, the *service*, which defines which Swift blobstore hosts a container -and how to connect to it.
+
+### Containers and Objects
+
+* Containers are created by users with accounts on the Swift filestore, and hold
+ *objects*.
+
+* Objects can be zero bytes long, or they can contain data.
+
+* Objects in the container can be up to 5GB; there is a special support for
+ larger files than this, which merges multiple objects in to one.
+
+* Each object is referenced by it's *name*; there is no notion of directories.
+
+* You can use any characters in an object name that can be 'URL-encoded'; the
+ maximum length of a name is 1034 characters -after URL encoding.
+
+* Names can have `/` characters in them, which are used to create the illusion of
+ a directory structure. For example `dir/dir2/name`. Even though this looks
+ like a directory, *it is still just a name*. There is no requirement to have
+ any entries in the container called `dir` or `dir/dir2`
+
+* That said. if the container has zero-byte objects that look like directory
+ names above other objects, they can pretend to be directories. Continuing the
+ example, a 0-byte object called `dir` would tell clients that it is a
+ directory while `dir/dir2` or `dir/dir2/name` were present. This creates an
+ illusion of containers holding a filesystem.
+
+Client applications talk to Swift over HTTP or HTTPS, reading, writing and deleting objects using standard HTTP operations (GET, PUT and DELETE, respectively). There is also a COPY operation, that creates a new object in the container, with a new name, containing the old data. There is no rename operation itself, objects need to be copied -then the original entry deleted.
+
+### Eventual Consistency
+
+The Swift Filesystem is \*eventually consistent\*: an operation on an object may not be immediately visible to that client, or other clients. This is a consequence of the goal of the filesystem: to span a set of machines, across multiple datacenters, in such a way that the data can still be available when many of them fail. (In contrast, the Hadoop HDFS filesystem is \*immediately consistent\*, but it does not span datacenters.)
+
+Eventual consistency can cause surprises for client applications that expect immediate consistency: after an object is deleted or overwritten, the object may still be visible -or the old data still retrievable. The Swift Filesystem client for Apache Hadoop attempts to handle this, in conjunction with the MapReduce engine, but there may be still be occasions when eventual consistency causes surprises.
+
+### Non-atomic "directory" operations.
+
+Hadoop expects some operations to be atomic, especially `rename()`, which is something the MapReduce layer relies on to commit the output of a job, renaming data from a temp directory to the final path. Because a rename is implemented as a copy of every blob under the directory's path, followed by a delete of the originals, the intermediate state of the operation will be visible to other clients. If two Reducer tasks to rename their temp directory to the final path, both operations may succeed, with the result that output directory contains mixed data. This can happen if MapReduce jobs are being run with *speculation* enabled and Swift used as the direct output of the MR job (it can also happen against Amazon S3).
+
+Other consequences of the non-atomic operations are:
+
+1. If a program is looking for the presence of the directory before acting
+ on the data -it may start prematurely. This can be avoided by using
+ other mechanisms to co-ordinate the programs, such as the presence of a file
+ that is written *after* any bulk directory operations.
+
+2. A `rename()` or `delete()` operation may include files added under
+ the source directory tree during the operation, may unintentionally delete
+ it, or delete the 0-byte swift entries that mimic directories and act
+ as parents for the files. Try to avoid doing this.
+
+The best ways to avoid all these problems is not using Swift as the filesystem between MapReduce jobs or other Hadoop workflows. It can act as a source of data, and a final destination, but it doesn't meet all of Hadoop's expectations of what a filesystem is -it's a *blobstore*.
+
+Working with Swift Object Stores in Hadoop
+------------------------------------------
+
+Once installed, the Swift FileSystem client can be used by any Hadoop application to read from or write to data stored in a Swift container.
+
+Data stored in Swift can be used as the direct input to a MapReduce job -simply use the `swift:` URL (see below) to declare the source of the data.
+
+This Swift Filesystem client is designed to work with multiple Swift object stores, both public and private. This allows the client to work with different clusters, reading and writing data to and from either of them.
+
+It can also work with the same object stores using multiple login details.
+
+These features are achieved by one basic concept: using a service name in the URI referring to a swift filesystem, and looking up all the connection and login details for that specific service. Different service names can be defined in the Hadoop XML configuration file, so defining different clusters, or providing different login details for the same object store(s).
+
+### Swift Filesystem URIs
+
+Hadoop uses URIs to refer to files within a filesystem. Some common examples are:
+
+ local://etc/hosts
+ hdfs://cluster1/users/example/data/set1
+ hdfs://cluster2.example.org:8020/users/example/data/set1
+
+The Swift Filesystem Client adds a new URL type `swift`. In a Swift Filesystem URL, the hostname part of a URL identifies the container and the service to work with; the path the name of the object. Here are some examples
+
+ swift://container.rackspace/my-object.csv
+ swift://data.hpcloud/data/set1
+ swift://dmitry.privatecloud/out/results
+
+In the last two examples, the paths look like directories: it is not, they are simply the objects named `data/set1` and `out/results` respectively.
+
+### Installing
+
+The `hadoop-openstack` JAR must be on the classpath of the Hadoop program trying to talk to the Swift service. If installed in the classpath of the Hadoop MapReduce service, then all programs started by the MR engine will pick up the JAR automatically. This is the easiest way to give all Hadoop jobs access to Swift.
+
+Alternatively, the JAR can be included as one of the JAR files that an application uses. This lets the Hadoop jobs work with a Swift object store even if the Hadoop cluster is not pre-configured for this.
+
+The library also depends upon the Apache HttpComponents library, which must also be on the classpath.
+
+### Configuring
+
+To talk to a swift service, the user must must provide:
+
+1. The URL defining the container and the service.
+
+2. In the cluster/job configuration, the login details of that service.
+
+Multiple service definitions can co-exist in the same configuration file: just use different names for them.
+
+#### Example: Rackspace US, in-cluster access using API key
+
+This service definition is for use in a Hadoop cluster deployed within Rackspace's US infrastructure.
+
+ <property>
+ <name>fs.swift.service.rackspace.auth.url</name>
+ <value>https://auth.api.rackspacecloud.com/v2.0/tokens</value>
+ <description>Rackspace US (multiregion)</description>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.username</name>
+ <value>user4</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.region</name>
+ <value>DFW</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.apikey</name>
+ <value>fe806aa86dfffe2f6ed8</value>
+ </property>
+
+Here the API key visible in the account settings API keys page is used to log in. No property for public/private access -the default is to use the private endpoint for Swift operations.
+
+This configuration also selects one of the regions, DFW, for its data.
+
+A reference to this service would use the `rackspace` service name:
+
+ swift://hadoop-container.rackspace/
+
+#### Example: Rackspace UK: remote access with password authentication
+
+This connects to Rackspace's UK ("LON") datacenter.
+
+ <property>
+ <name>fs.swift.service.rackspaceuk.auth.url</name>
+ <value>https://lon.identity.api.rackspacecloud.com/v2.0/tokens</value>
+ <description>Rackspace UK</description>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspaceuk.username</name>
+ <value>user4</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspaceuk.password</name>
+ <value>insert-password-here/value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.rackspace.public</name>
+ <value>true</value>
+ </property>
+
+This is a public access point connection, using a password over an API key.
+
+A reference to this service would use the `rackspaceuk` service name:
+
+ swift://hadoop-container.rackspaceuk/
+
+Because the public endpoint is used, if this service definition is used within the London datacenter, all accesses will be billed at the public upload/download rates, *irrespective of where the Hadoop cluster is*.
+
+#### Example: HP cloud service definition
+
+Here is an example that connects to the HP Cloud object store.
+
+ <property>
+ <name>fs.swift.service.hpcloud.auth.url</name>
+ <value>https://region-a.geo-1.identity.hpcloudsvc.com:35357/v2.0/tokens
+ </value>
+ <description>HP Cloud</description>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.tenant</name>
+ <value>FE806AA86</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.username</name>
+ <value>FE806AA86DFFFE2F6ED8</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.password</name>
+ <value>secret-password-goes-here</value>
+ </property>
+
+ <property>
+ <name>fs.swift.service.hpcloud.public</name>
+ <value>true</value>
+ </property>
+
+A reference to this service would use the `hpcloud` service name:
+
+ swift://hadoop-container.hpcloud/
+
+### General Swift Filesystem configuration options
+
+Some configuration options apply to the Swift client, independent of the specific Swift filesystem chosen.
+
+#### Blocksize fs.swift.blocksize
+
+Swift does not break up files into blocks, except in the special case of files over 5GB in length. Accordingly, there isn't a notion of a "block size" to define where the data is kept.
+
+Hadoop's MapReduce layer depends on files declaring their block size, so that it knows how to partition work. Too small a blocksize means that many mappers work on small pieces of data; too large a block size means that only a few mappers get started.
+
+The block size value reported by Swift, therefore, controls the basic workload partioning of the MapReduce engine -and can be an important parameter to tune for performance of the cluster.
+
+The property has a unit of kilobytes; the default value is `32*1024`: 32 MB
+
+ <property>
+ <name>fs.swift.blocksize</name>
+ <value>32768</value>
+ </property>
+
+This blocksize has no influence on how files are stored in Swift; it only controls what the reported size of blocks are - a value used in Hadoop MapReduce to divide work.
+
+Note that the MapReduce engine's split logic can be tuned independently by setting the `mapred.min.split.size` and `mapred.max.split.size` properties, which can be done in specific job configurations.
+
+ <property>
+ <name>mapred.min.split.size</name>
+ <value>524288</value>
+ </property>
+
+ <property>
+ <name>mapred.max.split.size</name>
+ <value>1048576</value>
+ </property>
+
+In an Apache Pig script, these properties would be set as:
+
+ mapred.min.split.size 524288
+ mapred.max.split.size 1048576
+
+#### Partition size fs.swift.partsize
+
+The Swift filesystem client breaks very large files into partitioned files, uploading each as it progresses, and writing any remaning data and an XML manifest when a partitioned file is closed.
+
+The partition size defaults to 4608 MB; 4.5GB, the maximum filesize that Swift can support.
+
+It is possible to set a smaller partition size, in the `fs.swift.partsize` option. This takes a value in KB.
+
+ <property>
+ <name>fs.swift.partsize</name>
+ <value>1024</value>
+ <description>upload every MB</description>
+ </property>
+
+When should this value be changed from its default?
+
+While there is no need to ever change it for basic operation of the Swift filesystem client, it can be tuned
+
+* If a Swift filesystem is location aware, then breaking a file up into
+ smaller partitions scatters the data round the cluster. For best performance,
+ the property `fs.swift.blocksize` should be set to a smaller value than the
+ partition size of files.
+
+* When writing to an unpartitioned file, the entire write is done in the
+ `close()` operation. When a file is partitioned, the outstanding data to
+ be written whenever the outstanding amount of data is greater than the
+ partition size. This means that data will be written more incrementally
+
+#### Request size fs.swift.requestsize
+
+The Swift filesystem client reads files in HTTP GET operations, asking for a block of data at a time.
+
+The default value is 64KB. A larger value may be more efficient over faster networks, as it reduces the overhead of setting up the HTTP operation.
+
+However, if the file is read with many random accesses, requests for data will be made from different parts of the file -discarding some of the previously requested data. The benefits of larger request sizes may be wasted.
+
+The property `fs.swift.requestsize` sets the request size in KB.
+
+ <property>
+ <name>fs.swift.requestsize</name>
+ <value>128</value>
+ </property>
+
+#### Connection timeout fs.swift.connect.timeout
+
+This sets the timeout in milliseconds to connect to a Swift service.
+
+ <property>
+ <name>fs.swift.connect.timeout</name>
+ <value>15000</value>
+ </property>
+
+A shorter timeout means that connection failures are raised faster -but may trigger more false alarms. A longer timeout is more resilient to network problems -and may be needed when talking to remote filesystems.
+
+#### Connection timeout fs.swift.socket.timeout
+
+This sets the timeout in milliseconds to wait for data from a connected socket.
+
+ <property>
+ <name>fs.swift.socket.timeout</name>
+ <value>60000</value>
+ </property>
+
+A shorter timeout means that connection failures are raised faster -but may trigger more false alarms. A longer timeout is more resilient to network problems -and may be needed when talking to remote filesystems.
+
+#### Connection Retry Count fs.swift.connect.retry.count
+
+This sets the number of times to try to connect to a service whenever an HTTP request is made.
+
+ <property>
+ <name>fs.swift.connect.retry.count</name>
+ <value>3</value>
+ </property>
+
+The more retries, the more resilient it is to transient outages -and the less rapid it is at detecting and reporting server connectivity problems.
+
+#### Connection Throttle Delay fs.swift.connect.throttle.delay
+
+This property adds a delay between bulk file copy and delete operations, to prevent requests being throttled or blocked by the remote service
+
+ <property>
+ <name>fs.swift.connect.throttle.delay</name>
+ <value>0</value>
+ </property>
+
+It is measured in milliseconds; "0" means do not add any delay.
+
+Throttling is enabled on the public endpoints of some Swift services. If `rename()` or `delete()` operations fail with `SwiftThrottledRequestException` exceptions, try setting this property.
+
+#### HTTP Proxy
+
+If the client can only access the Swift filesystem via a web proxy server, the client configuration must specify the proxy via the `fs.swift.connect.proxy.host` and `fs.swift.connect.proxy.port` properties.
+
+ <property>
+ <name>fs.swift.proxy.host</name>
+ <value>web-proxy</value>
+ </property>
+
+ <property>
+ <name>fs.swift.proxy.port</name>
+ <value>8088</value>
+ </property>
+
+If the host is declared, the proxy port must be set to a valid integer value.
+
+### Troubleshooting
+
+#### ClassNotFoundException
+
+The `hadoop-openstack` JAR -or any dependencies- may not be on your classpath.
+
+If it is a remote MapReduce job that is failing, make sure that the JAR is installed on the servers in the cluster -or that the job submission process uploads the JAR file to the distributed cache.
+
+#### Failure to Authenticate
+
+A `SwiftAuthenticationFailedException` is thrown when the client cannot authenticate with the OpenStack keystone server. This could be because the URL in the service definition is wrong, or because the supplied credentials are invalid.
+
+1. Check the authentication URL through `curl` or your browser
+
+2. Use a Swift client such as CyberDuck to validate your credentials
+
+3. If you have included a tenant ID, try leaving it out. Similarly,
+ try adding it if you had not included it.
+
+4. Try switching from API key authentication to password-based authentication,
+ by setting the password.
+
+5. Change your credentials. As with Amazon AWS clients, some credentials
+ don't seem to like going over the network.
+
+#### Timeout connecting to the Swift Service
+
+This happens if the client application is running outside an OpenStack cluster, where it does not have access to the private hostname/IP address for filesystem operations. Set the `public` flag to true -but remember to set it to false for use in-cluster.
+
+### Warnings
+
+1. Do not share your login details with anyone, which means do not log the
+ details, or check the XML configuration files into any revision control system
+ to which you do not have exclusive access.
+
+2. Similarly, do not use your real account details in any
+ documentation \*or any bug reports submitted online\*
+
+3. Prefer the apikey authentication over passwords as it is easier
+ to revoke a key -and some service providers allow you to set
+ an automatic expiry date on a key when issued.
+
+4. Do not use the public service endpoint from within a public OpenStack
+ cluster, as it will run up large bills.
+
+5. Remember: it's not a real filesystem or hierarchical directory structure.
+ Some operations (directory rename and delete) take time and are not atomic or
+ isolated from other operations taking place.
+
+6. Append is not supported.
+
+7. Unix-style permissions are not supported. All accounts with write access to
+ a repository have unlimited access; the same goes for those with read access.
+
+8. In the public clouds, do not make the containers public unless you are happy
+ with anyone reading your data, and are prepared to pay the costs of their
+ downloads.
+
+### Limits
+
+* Maximum length of an object path: 1024 characters
+
+* Maximum size of a binary object: no absolute limit. Files \> 5GB are
+ partitioned into separate files in the native filesystem, and merged during
+ retrieval. *Warning:* the partitioned/large file support is the
+ most complex part of the Hadoop/Swift FS integration, and, along with
+ authentication, the most troublesome to support.
+
+### Testing the hadoop-openstack module
+
+The `hadoop-openstack` can be remotely tested against any public or private cloud infrastructure which supports the OpenStack Keystone authentication mechanism. It can also be tested against private OpenStack clusters. OpenStack Development teams are strongly encouraged to test the Hadoop swift filesystem client against any version of Swift that they are developing or deploying, to stress their cluster and to identify bugs early.
+
+The module comes with a large suite of JUnit tests -tests that are only executed if the source tree includes credentials to test against a specific cluster.
+
+After checking out the Hadoop source tree, create the file:
+
+ hadoop-tools/hadoop-openstack/src/test/resources/auth-keys.xml
+
+Into this file, insert the credentials needed to bond to the test filesystem, as decribed above.
+
+Next set the property `test.fs.swift.name` to the URL of a swift container to test against. The tests expect exclusive access to this container -do not keep any other data on it, or expect it to be preserved.
+
+ <property>
+ <name>test.fs.swift.name</name>
+ <value>swift://test.myswift/</value>
+ </property>
+
+In the base hadoop directory, run:
+
+ mvn clean install -DskipTests
+
+This builds a set of Hadoop JARs consistent with the `hadoop-openstack` module that is about to be tested.
+
+In the `hadoop-tools/hadoop-openstack` directory run
+
+ mvn test -Dtest=TestSwiftRestClient
+
+This runs some simple tests which include authenticating against the remote swift service. If these tests fail, so will all the rest. If it does fail: check your authentication.
+
+Once this test succeeds, you can run the full test suite
+
+ mvn test
+
+Be advised that these tests can take an hour or more, especially against a remote Swift service -or one that throttles bulk operations.
+
+Once the `auth-keys.xml` file is in place, the `mvn test` runs from the Hadoop source base directory will automatically run these OpenStack tests While this ensures that no regressions have occurred, it can also add significant time to test runs, and may run up bills, depending on who is providingthe Swift storage service. We recommend having a separate source tree set up purely for the Swift tests, and running it manually or by the CI tooling at a lower frequency than normal test runs.
+
+Finally: Apache Hadoop is an open source project. Contributions of code -including more tests- are very welcome.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css b/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css
new file mode 100644
index 0000000..f830baa
--- /dev/null
+++ b/hadoop-tools/hadoop-openstack/src/site/resources/css/site.css
@@ -0,0 +1,30 @@
+/*
+* Licensed to the Apache Software Foundation (ASF) under one or more
+* contributor license agreements. See the NOTICE file distributed with
+* this work for additional information regarding copyright ownership.
+* The ASF licenses this file to You under the Apache License, Version 2.0
+* (the "License"); you may not use this file except in compliance with
+* the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+#banner {
+ height: 93px;
+ background: none;
+}
+
+#bannerLeft img {
+ margin-left: 30px;
+ margin-top: 10px;
+}
+
+#bannerRight img {
+ margin: 17px;
+}
+
[31/50] [abbrv] hadoop git commit: HADOOP-11596. Allow
smart-apply-patch.sh to add new files in binary git patches (raviprak)
Posted by zj...@apache.org.
HADOOP-11596. Allow smart-apply-patch.sh to add new files in binary git patches (raviprak)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13d1ba99
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13d1ba99
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13d1ba99
Branch: refs/heads/YARN-2928
Commit: 13d1ba9965236381e9014fce22120b999c36189a
Parents: db66062
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Tue Feb 17 11:13:47 2015 -0800
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Tue Feb 17 11:13:47 2015 -0800
----------------------------------------------------------------------
dev-support/smart-apply-patch.sh | 4 ++--
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d1ba99/dev-support/smart-apply-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/smart-apply-patch.sh b/dev-support/smart-apply-patch.sh
index 3542fb4..03bc4f8 100755
--- a/dev-support/smart-apply-patch.sh
+++ b/dev-support/smart-apply-patch.sh
@@ -38,8 +38,8 @@ is_git_diff_with_prefix() {
fi
if [[ "$line" =~ ^\+\+\+\ ]] ||
[[ "$line" =~ ^\-\-\-\ ]]; then
- if ! [[ "$line" =~ ^....[ab]/ ]]; then
- return 1 # All +++ and --- lines must start with a/ or b/.
+ if ! [[ "$line" =~ ^....[ab]/ || "$line" =~ ^..../dev/null ]]; then
+ return 1 # All +++ and --- lines must start with a/ or b/ or be /dev/null.
fi
fi
done < $1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13d1ba99/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 6e43872..0de835a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -612,6 +612,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
+ HADOOP-11596. Allow smart-apply-patch.sh to add new files in binary git
+ patches (raviprak)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
[38/50] [abbrv] hadoop git commit: YARN-3207. Secondary filter
matches entites which do not have the key being filtered for. Contributed by
Zhijie Shen
Posted by zj...@apache.org.
YARN-3207. Secondary filter matches entites which do not have the key
being filtered for. Contributed by Zhijie Shen
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57db50cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57db50cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57db50cb
Branch: refs/heads/YARN-2928
Commit: 57db50cbe3ce42618ad6d6869ae337d15b261f4e
Parents: 685af8a
Author: Xuan <xg...@apache.org>
Authored: Tue Feb 17 18:17:29 2015 -0800
Committer: Xuan <xg...@apache.org>
Committed: Tue Feb 17 18:17:29 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java | 2 +-
.../hadoop/yarn/server/timeline/TimelineStoreTestUtils.java | 3 +++
3 files changed, 7 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57db50cb/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 66543cd..cbba046 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -608,6 +608,9 @@ Release 2.7.0 - UNRELEASED
YARN-2749. Fix some testcases from TestLogAggregationService fails in trunk.
(Xuan Gong via junping_du)
+ YARN-3207. Secondary filter matches entites which do not have the key being
+ filtered for. (Zhijie Shen via xgong)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57db50cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
index 5f153bd..9fd2cfc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/LeveldbTimelineStore.java
@@ -672,7 +672,7 @@ public class LeveldbTimelineStore extends AbstractService
if (v == null) {
Set<Object> vs = entity.getPrimaryFilters()
.get(filter.getName());
- if (vs != null && !vs.contains(filter.getValue())) {
+ if (vs == null || !vs.contains(filter.getValue())) {
filterPassed = false;
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57db50cb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
index 6f15b92..c99786d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/TimelineStoreTestUtils.java
@@ -767,6 +767,9 @@ public class TimelineStoreTestUtils {
entities = getEntitiesWithFilters("type_1", userFilter, badTestingFilters);
assertEquals(0, entities.size());
+
+ entities = getEntitiesWithFilters("type_5", null, badTestingFilters);
+ assertEquals(0, entities.size());
}
public void testGetEvents() throws IOException {
[07/50] [abbrv] hadoop git commit: YARN-1299. Improve a log message
in AppSchedulingInfo by adding application id. Contributed by Ashutosh Jindal
and devaraj.
Posted by zj...@apache.org.
YARN-1299. Improve a log message in AppSchedulingInfo by adding application id. Contributed by Ashutosh Jindal and devaraj.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9aae81c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9aae81c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9aae81c9
Branch: refs/heads/YARN-2928
Commit: 9aae81c93421874b726c7b6ff970895c429e502d
Parents: 447bd7b
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Feb 17 01:48:22 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Feb 17 01:48:22 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9aae81c9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7161dc3..bb145eb 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -298,6 +298,9 @@ Release 2.7.0 - UNRELEASED
YARN-3203. Correct a log message in AuxServices. (Brahma Reddy Battula
via ozawa)
+ YARN-1299. Improve a log message in AppSchedulingInfo by adding application
+ id. (Ashutosh Jindal and devaraj via ozawa)
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9aae81c9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 3ade7f7..a9a459f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -176,7 +176,8 @@ public class AppSchedulingInfo {
// Similarly, deactivate application?
if (request.getNumContainers() <= 0) {
- LOG.info("checking for deactivate... ");
+ LOG.info("checking for deactivate of application :"
+ + this.applicationId);
checkForDeactivation();
}
[13/50] [abbrv] hadoop git commit: HDFS-7798. Checkpointing failure
caused by shared KerberosAuthenticator. (Chengbing Liu via yliu)
Posted by zj...@apache.org.
HDFS-7798. Checkpointing failure caused by shared KerberosAuthenticator. (Chengbing Liu via yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/500e6a0f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/500e6a0f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/500e6a0f
Branch: refs/heads/YARN-2928
Commit: 500e6a0f46d14a591d0ec082b6d26ee59bdfdf76
Parents: 9729b24
Author: yliu <yl...@apache.org>
Authored: Tue Feb 17 07:46:33 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Tue Feb 17 07:46:33 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java | 7 ++-----
2 files changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/500e6a0f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index cc24dc4..8b234fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -957,6 +957,9 @@ Release 2.7.0 - UNRELEASED
HDFS-4625. BKJM doesn't take advantage of speculative reads. (Rakesh R
via aajisaka)
+ HDFS-7798. Checkpointing failure caused by shared KerberosAuthenticator.
+ (Chengbing Liu via yliu)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/500e6a0f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
index 00e9e98..8a743b6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/URLConnectionFactory.java
@@ -49,9 +49,6 @@ import com.google.common.annotations.VisibleForTesting;
public class URLConnectionFactory {
private static final Log LOG = LogFactory.getLog(URLConnectionFactory.class);
- /** SPNEGO authenticator */
- private static final KerberosUgiAuthenticator AUTH = new KerberosUgiAuthenticator();
-
/**
* Timeout for socket connects and reads
*/
@@ -161,8 +158,8 @@ public class URLConnectionFactory {
}
UserGroupInformation.getCurrentUser().checkTGTAndReloginFromKeytab();
final AuthenticatedURL.Token authToken = new AuthenticatedURL.Token();
- return new AuthenticatedURL(AUTH, connConfigurator).openConnection(url,
- authToken);
+ return new AuthenticatedURL(new KerberosUgiAuthenticator(),
+ connConfigurator).openConnection(url, authToken);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("open URL connection");
[16/50] [abbrv] hadoop git commit: HDFS-7797. Add audit log for
setQuota operation (Rakesh R via umamahesh)
Posted by zj...@apache.org.
HDFS-7797. Add audit log for setQuota operation (Rakesh R via umamahesh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f24a5678
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f24a5678
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f24a5678
Branch: refs/heads/YARN-2928
Commit: f24a56787a15e89a7c1e777b8043ab9ae8792505
Parents: 2f0f756
Author: Uma Maheswara Rao G <um...@apache.org>
Authored: Tue Feb 17 20:11:11 2015 +0530
Committer: Uma Maheswara Rao G <um...@apache.org>
Committed: Tue Feb 17 20:11:11 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 7 ++++++-
2 files changed, 8 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24a5678/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 8b234fe..fcf5994 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -637,6 +637,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7604. Track and display failed DataNode storage locations in NameNode.
(cnauroth)
+ HDFS-7797. Add audit log for setQuota operation (Rakesh R via umamahesh)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f24a5678/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 84ab179..06d7bd0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -3814,14 +3814,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
throws IOException {
checkOperation(OperationCategory.WRITE);
writeLock();
+ boolean success = false;
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set quota on " + src);
FSDirAttrOp.setQuota(dir, src, nsQuota, ssQuota, type);
+ success = true;
} finally {
writeUnlock();
+ if (success) {
+ getEditLog().logSync();
+ }
+ logAuditEvent(success, "setQuota", src);
}
- getEditLog().logSync();
}
/** Persist all metadata about this file.
[05/50] [abbrv] hadoop git commit: YARN-2749. Fix some testcases from
TestLogAggregationService fails in trunk. (Contributed by Xuan Gong)
Posted by zj...@apache.org.
YARN-2749. Fix some testcases from TestLogAggregationService fails in trunk. (Contributed by Xuan Gong)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab0b958a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab0b958a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab0b958a
Branch: refs/heads/YARN-2928
Commit: ab0b958a522d502426b91b6e4ab6dd29caccc372
Parents: 3338f6d
Author: Junping Du <ju...@apache.org>
Authored: Sun Feb 15 06:46:32 2015 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Sun Feb 15 06:46:32 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../logaggregation/AppLogAggregatorImpl.java | 15 +++++++++++++++
2 files changed, 18 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab0b958a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e9a7c58..e00e447 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -599,6 +599,9 @@ Release 2.7.0 - UNRELEASED
YARN-2899. Run TestDockerContainerExecutorWithMocks on Linux only.
(Ming Ma via cnauroth)
+ YARN-2749. Fix some testcases from TestLogAggregationService fails in trunk.
+ (Xuan Gong via junping_du)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab0b958a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 20887b6..8eb00f4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -117,6 +117,8 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
private final int retentionSize;
private final long rollingMonitorInterval;
private final NodeId nodeId;
+ // This variable is only for testing
+ private final AtomicBoolean waiting = new AtomicBoolean(false);
private final Map<ContainerId, ContainerLogAggregator> containerLogAggregators =
new HashMap<ContainerId, ContainerLogAggregator>();
@@ -391,6 +393,7 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
while (!this.appFinishing.get() && !this.aborted.get()) {
synchronized(this) {
try {
+ waiting.set(true);
if (this.rollingMonitorInterval > 0) {
wait(this.rollingMonitorInterval * 1000);
if (this.appFinishing.get() || this.aborted.get()) {
@@ -507,7 +510,19 @@ public class AppLogAggregatorImpl implements AppLogAggregator {
@Private
@VisibleForTesting
+ // This is only used for testing.
+ // This will wake the log aggregation thread that is waiting for
+ // rollingMonitorInterval.
+ // To use this method, make sure the log aggregation thread is running
+ // and waiting for rollingMonitorInterval.
public synchronized void doLogAggregationOutOfBand() {
+ while(!waiting.get()) {
+ try {
+ wait(200);
+ } catch (InterruptedException e) {
+ // Do Nothing
+ }
+ }
LOG.info("Do OutOfBand log aggregation");
this.notifyAll();
}
[24/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
deleted file mode 100644
index e011563..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapredCommands.apt.vm
+++ /dev/null
@@ -1,233 +0,0 @@
-~~ Licensed to the Apache Software Foundation (ASF) under one or more
-~~ contributor license agreements. See the NOTICE file distributed with
-~~ this work for additional information regarding copyright ownership.
-~~ The ASF licenses this file to You under the Apache License, Version 2.0
-~~ (the "License"); you may not use this file except in compliance with
-~~ the License. You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- MapReduce Commands Guide
- ---
- ---
- ${maven.build.timestamp}
-
-MapReduce Commands Guide
-
-%{toc|section=1|fromDepth=2|toDepth=4}
-
-* Overview
-
- MapReduce commands are invoked by the <<<bin/mapred>>> script. Running the
- script without any arguments prints the description for all commands.
-
- Usage: <<<mapred [--config confdir] [--loglevel loglevel] COMMAND>>>
-
- MapReduce has an option parsing framework that employs parsing generic
- options as well as running classes.
-
-*-------------------------+---------------------------------------------------+
-|| COMMAND_OPTIONS || Description |
-*-------------------------+---------------------------------------------------+
-| --config confdir | Overwrites the default Configuration directory. Default
-| | is $\{HADOOP_PREFIX\}/conf.
-*-------------------------+---------------------------------------------------+
-| --loglevel loglevel | Overwrites the log level. Valid log levels are FATAL,
-| | ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO.
-*-------------------------+---------------------------------------------------+
-| COMMAND COMMAND_OPTIONS | Various commands with their options are described
-| | in the following sections. The commands have been
-| | grouped into {{User Commands}} and
-| | {{Administration Commands}}.
-*-------------------------+---------------------------------------------------+
-
-* User Commands
-
- Commands useful for users of a hadoop cluster.
-
-** <<<pipes>>>
-
- Runs a pipes job.
-
- Usage: <<<mapred pipes [-conf <path>] [-jobconf <key=value>, <key=value>,
- ...] [-input <path>] [-output <path>] [-jar <jar file>] [-inputformat
- <class>] [-map <class>] [-partitioner <class>] [-reduce <class>] [-writer
- <class>] [-program <executable>] [-reduces <num>]>>>
-
-*----------------------------------------+------------------------------------+
-|| COMMAND_OPTION || Description
-*----------------------------------------+------------------------------------+
-| -conf <path> | Configuration for job
-*----------------------------------------+------------------------------------+
-| -jobconf <key=value>, <key=value>, ... | Add/override configuration for job
-*----------------------------------------+------------------------------------+
-| -input <path> | Input directory
-*----------------------------------------+------------------------------------+
-| -output <path> | Output directory
-*----------------------------------------+------------------------------------+
-| -jar <jar file> | Jar filename
-*----------------------------------------+------------------------------------+
-| -inputformat <class> | InputFormat class
-*----------------------------------------+------------------------------------+
-| -map <class> | Java Map class
-*----------------------------------------+------------------------------------+
-| -partitioner <class> | Java Partitioner
-*----------------------------------------+------------------------------------+
-| -reduce <class> | Java Reduce class
-*----------------------------------------+------------------------------------+
-| -writer <class> | Java RecordWriter
-*----------------------------------------+------------------------------------+
-| -program <executable> | Executable URI
-*----------------------------------------+------------------------------------+
-| -reduces <num> | Number of reduces
-*----------------------------------------+------------------------------------+
-
-** <<<job>>>
-
- Command to interact with Map Reduce Jobs.
-
- Usage: <<<mapred job
- | [{{{../../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options}GENERIC_OPTIONS}}]
- | [-submit <job-file>]
- | [-status <job-id>]
- | [-counter <job-id> <group-name> <counter-name>]
- | [-kill <job-id>]
- | [-events <job-id> <from-event-#> <#-of-events>]
- | [-history [all] <jobOutputDir>] | [-list [all]]
- | [-kill-task <task-id>] | [-fail-task <task-id>]
- | [-set-priority <job-id> <priority>]>>>
-
-*------------------------------+---------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------------------+---------------------------------------------+
-| -submit <job-file> | Submits the job.
-*------------------------------+---------------------------------------------+
-| -status <job-id> | Prints the map and reduce completion
- | percentage and all job counters.
-*------------------------------+---------------------------------------------+
-| -counter <job-id> <group-name> <counter-name> | Prints the counter value.
-*------------------------------+---------------------------------------------+
-| -kill <job-id> | Kills the job.
-*------------------------------+---------------------------------------------+
-| -events <job-id> <from-event-#> <#-of-events> | Prints the events' details
- | received by jobtracker for the given range.
-*------------------------------+---------------------------------------------+
-| -history [all]<jobOutputDir> | Prints job details, failed and killed tip
- | details. More details about the job such as
- | successful tasks and task attempts made for
- | each task can be viewed by specifying the
- | [all] option.
-*------------------------------+---------------------------------------------+
-| -list [all] | Displays jobs which are yet to complete.
- | <<<-list all>>> displays all jobs.
-*------------------------------+---------------------------------------------+
-| -kill-task <task-id> | Kills the task. Killed tasks are NOT counted
- | against failed attempts.
-*------------------------------+---------------------------------------------+
-| -fail-task <task-id> | Fails the task. Failed tasks are counted
- | against failed attempts.
-*------------------------------+---------------------------------------------+
-| -set-priority <job-id> <priority> | Changes the priority of the job. Allowed
- | priority values are VERY_HIGH, HIGH, NORMAL,
- | LOW, VERY_LOW
-*------------------------------+---------------------------------------------+
-
-** <<<queue>>>
-
- command to interact and view Job Queue information
-
- Usage: <<<mapred queue [-list] | [-info <job-queue-name> [-showJobs]]
- | [-showacls]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| -list | Gets list of Job Queues configured in the system.
- | Along with scheduling information associated with the job
- | queues.
-*-----------------+-----------------------------------------------------------+
-| -info <job-queue-name> [-showJobs] | Displays the job queue information and
- | associated scheduling information of particular job queue.
- | If <<<-showJobs>>> options is present a list of jobs
- | submitted to the particular job queue is displayed.
-*-----------------+-----------------------------------------------------------+
-| -showacls | Displays the queue name and associated queue operations
- | allowed for the current user. The list consists of only
- | those queues to which the user has access.
-*-----------------+-----------------------------------------------------------+
-
-** <<<classpath>>>
-
- Prints the class path needed to get the Hadoop jar and the required
- libraries.
-
- Usage: <<<mapred classpath>>>
-
-** <<<distcp>>>
-
- Copy file or directories recursively. More information can be found at
- {{{./DistCp.html}Hadoop DistCp Guide}}.
-
-** <<<archive>>>
-
- Creates a hadoop archive. More information can be found at
- {{{./HadoopArchives.html}Hadoop Archives Guide}}.
-
-** <<<version>>>
-
- Prints the version.
-
- Usage: <<<mapred version>>>
-
-* Administration Commands
-
- Commands useful for administrators of a hadoop cluster.
-
-** <<<historyserver>>>
-
- Start JobHistoryServer.
-
- Usage: <<<mapred historyserver>>>
-
-** <<<hsadmin>>>
-
- Runs a MapReduce hsadmin client for execute JobHistoryServer administrative
- commands.
-
- Usage: <<<mapred hsadmin
- [-refreshUserToGroupsMappings] |
- [-refreshSuperUserGroupsConfiguration] |
- [-refreshAdminAcls] |
- [-refreshLoadedJobCache] |
- [-refreshLogRetentionSettings] |
- [-refreshJobRetentionSettings] |
- [-getGroups [username]] | [-help [cmd]]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| -refreshUserToGroupsMappings | Refresh user-to-groups mappings
-*-----------------+-----------------------------------------------------------+
-| -refreshSuperUserGroupsConfiguration| Refresh superuser proxy groups mappings
-*-----------------+-----------------------------------------------------------+
-| -refreshAdminAcls | Refresh acls for administration of Job history server
-*-----------------+-----------------------------------------------------------+
-| -refreshLoadedJobCache | Refresh loaded job cache of Job history server
-*-----------------+-----------------------------------------------------------+
-| -refreshJobRetentionSettings|Refresh job history period, job cleaner settings
-*-----------------+-----------------------------------------------------------+
-| -refreshLogRetentionSettings | Refresh log retention period and log retention
-| | check interval
-*-----------------+-----------------------------------------------------------+
-| -getGroups [username] | Get the groups which given user belongs to
-*-----------------+-----------------------------------------------------------+
-| -help [cmd] | Displays help for the given command or all commands if none is
-| | specified.
-*-----------------+-----------------------------------------------------------+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm
deleted file mode 100644
index 06d8022..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/PluggableShuffleAndPluggableSort.apt.vm
+++ /dev/null
@@ -1,98 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Map Reduce Next Generation-${project.version} - Pluggable Shuffle and Pluggable Sort
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop MapReduce Next Generation - Pluggable Shuffle and Pluggable Sort
-
-* Introduction
-
- The pluggable shuffle and pluggable sort capabilities allow replacing the
- built in shuffle and sort logic with alternate implementations. Example use
- cases for this are: using a different application protocol other than HTTP
- such as RDMA for shuffling data from the Map nodes to the Reducer nodes; or
- replacing the sort logic with custom algorithms that enable Hash aggregation
- and Limit-N query.
-
- <<IMPORTANT:>> The pluggable shuffle and pluggable sort capabilities are
- experimental and unstable. This means the provided APIs may change and break
- compatibility in future versions of Hadoop.
-
-* Implementing a Custom Shuffle and a Custom Sort
-
- A custom shuffle implementation requires a
- <<<org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices.AuxiliaryService>>>
- implementation class running in the NodeManagers and a
- <<<org.apache.hadoop.mapred.ShuffleConsumerPlugin>>> implementation class
- running in the Reducer tasks.
-
- The default implementations provided by Hadoop can be used as references:
-
- * <<<org.apache.hadoop.mapred.ShuffleHandler>>>
-
- * <<<org.apache.hadoop.mapreduce.task.reduce.Shuffle>>>
-
- A custom sort implementation requires a <<<org.apache.hadoop.mapred.MapOutputCollector>>>
- implementation class running in the Mapper tasks and (optionally, depending
- on the sort implementation) a <<<org.apache.hadoop.mapred.ShuffleConsumerPlugin>>>
- implementation class running in the Reducer tasks.
-
- The default implementations provided by Hadoop can be used as references:
-
- * <<<org.apache.hadoop.mapred.MapTask$MapOutputBuffer>>>
-
- * <<<org.apache.hadoop.mapreduce.task.reduce.Shuffle>>>
-
-* Configuration
-
- Except for the auxiliary service running in the NodeManagers serving the
- shuffle (by default the <<<ShuffleHandler>>>), all the pluggable components
- run in the job tasks. This means, they can be configured on per job basis.
- The auxiliary service servicing the Shuffle must be configured in the
- NodeManagers configuration.
-
-** Job Configuration Properties (on per job basis):
-
-*--------------------------------------+---------------------+-----------------+
-| <<Property>> | <<Default Value>> | <<Explanation>> |
-*--------------------------------------+---------------------+-----------------+
-| <<<mapreduce.job.reduce.shuffle.consumer.plugin.class>>> | <<<org.apache.hadoop.mapreduce.task.reduce.Shuffle>>> | The <<<ShuffleConsumerPlugin>>> implementation to use |
-*--------------------------------------+---------------------+-----------------+
-| <<<mapreduce.job.map.output.collector.class>>> | <<<org.apache.hadoop.mapred.MapTask$MapOutputBuffer>>> | The <<<MapOutputCollector>>> implementation(s) to use |
-*--------------------------------------+---------------------+-----------------+
-
- These properties can also be set in the <<<mapred-site.xml>>> to change the default values for all jobs.
-
- The collector class configuration may specify a comma-separated list of collector implementations.
- In this case, the map task will attempt to instantiate each in turn until one of the
- implementations successfully initializes. This can be useful if a given collector
- implementation is only compatible with certain types of keys or values, for example.
-
-** NodeManager Configuration properties, <<<yarn-site.xml>>> in all nodes:
-
-*--------------------------------------+---------------------+-----------------+
-| <<Property>> | <<Default Value>> | <<Explanation>> |
-*--------------------------------------+---------------------+-----------------+
-| <<<yarn.nodemanager.aux-services>>> | <<<...,mapreduce_shuffle>>> | The auxiliary service name |
-*--------------------------------------+---------------------+-----------------+
-| <<<yarn.nodemanager.aux-services.mapreduce_shuffle.class>>> | <<<org.apache.hadoop.mapred.ShuffleHandler>>> | The auxiliary service class to use |
-*--------------------------------------+---------------------+-----------------+
-
- <<IMPORTANT:>> If setting an auxiliary service in addition the default
- <<<mapreduce_shuffle>>> service, then a new service key should be added to the
- <<<yarn.nodemanager.aux-services>>> property, for example <<<mapred.shufflex>>>.
- Then the property defining the corresponding class must be
- <<<yarn.nodemanager.aux-services.mapreduce_shufflex.class>>>.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistributedCacheDeploy.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistributedCacheDeploy.md.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistributedCacheDeploy.md.vm
new file mode 100644
index 0000000..36ad8fc
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/DistributedCacheDeploy.md.vm
@@ -0,0 +1,119 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+#set ( $H3 = '###' )
+#set ( $H4 = '####' )
+#set ( $H5 = '#####' )
+
+Hadoop: Distributed Cache Deploy
+================================
+
+Introduction
+------------
+
+The MapReduce application framework has rudimentary support for deploying a new version of the MapReduce framework via the distributed cache. By setting the appropriate configuration properties, users can run a different version of MapReduce than the one initially deployed to the cluster. For example, cluster administrators can place multiple versions of MapReduce in HDFS and configure `mapred-site.xml` to specify which version jobs will use by default. This allows the administrators to perform a rolling upgrade of the MapReduce framework under certain conditions.
+
+Preconditions and Limitations
+-----------------------------
+
+The support for deploying the MapReduce framework via the distributed cache currently does not address the job client code used to submit and query jobs. It also does not address the `ShuffleHandler` code that runs as an auxilliary service within each NodeManager. As a result the following limitations apply to MapReduce versions that can be successfully deployed via the distributed cache in a rolling upgrade fashion:
+
+* The MapReduce version must be compatible with the job client code used to
+ submit and query jobs. If it is incompatible then the job client must be
+ upgraded separately on any node from which jobs using the new MapReduce
+ version will be submitted or queried.
+
+* The MapReduce version must be compatible with the configuration files used
+ by the job client submitting the jobs. If it is incompatible with that
+ configuration (e.g.: a new property must be set or an existing property
+ value changed) then the configuration must be updated first.
+
+* The MapReduce version must be compatible with the `ShuffleHandler`
+ version running on the nodes in the cluster. If it is incompatible then the
+ new `ShuffleHandler` code must be deployed to all the nodes in the
+ cluster, and the NodeManagers must be restarted to pick up the new
+ `ShuffleHandler` code.
+
+Deploying a New MapReduce Version via the Distributed Cache
+-----------------------------------------------------------
+
+Deploying a new MapReduce version consists of three steps:
+
+1. Upload the MapReduce archive to a location that can be accessed by the
+ job submission client. Ideally the archive should be on the cluster's default
+ filesystem at a publicly-readable path. See the archive location discussion
+ below for more details.
+
+2. Configure `mapreduce.application.framework.path` to point to the
+ location where the archive is located. As when specifying distributed cache
+ files for a job, this is a URL that also supports creating an alias for the
+ archive if a URL fragment is specified. For example,
+ `hdfs:/mapred/framework/hadoop-mapreduce-${project.version}.tar.gz#mrframework`
+ will be localized as `mrframework` rather than
+ `hadoop-mapreduce-${project.version}.tar.gz`.
+
+3. Configure `mapreduce.application.classpath` to set the proper
+ classpath to use with the MapReduce archive configured above. NOTE: An error
+ occurs if `mapreduce.application.framework.path` is configured but
+ `mapreduce.application.classpath` does not reference the base name of the
+ archive path or the alias if an alias was specified.
+
+$H3 Location of the MapReduce Archive and How It Affects Job Performance
+
+Note that the location of the MapReduce archive can be critical to job submission and job startup performance. If the archive is not located on the cluster's default filesystem then it will be copied to the job staging directory for each job and localized to each node where the job's tasks run. This will slow down job submission and task startup performance.
+
+If the archive is located on the default filesystem then the job client will not upload the archive to the job staging directory for each job submission. However if the archive path is not readable by all cluster users then the archive will be localized separately for each user on each node where tasks execute. This can cause unnecessary duplication in the distributed cache.
+
+When working with a large cluster it can be important to increase the replication factor of the archive to increase its availability. This will spread the load when the nodes in the cluster localize the archive for the first time.
+
+MapReduce Archives and Classpath Configuration
+----------------------------------------------
+
+Setting a proper classpath for the MapReduce archive depends upon the composition of the archive and whether it has any additional dependencies. For example, the archive can contain not only the MapReduce jars but also the necessary YARN, HDFS, and Hadoop Common jars and all other dependencies. In that case, `mapreduce.application.classpath` would be configured to something like the following example, where the archive basename is hadoop-mapreduce-${project.version}.tar.gz and the archive is organized internally similar to the standard Hadoop distribution archive:
+
+`$HADOOP_CONF_DIR,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/lib/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/common/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/common/lib/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/yarn/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/yarn/lib/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/hdfs/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/hdfs/lib/*`
+
+Another possible approach is to have the archive consist of just the MapReduce jars and have the remaining dependencies picked up from the Hadoop distribution installed on the nodes. In that case, the above example would change to something like the following:
+
+`$HADOOP_CONF_DIR,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/*,$PWD/hadoop-mapreduce-${project.version}.tar.gz/hadoop-mapreduce-${project.version}/share/hadoop/mapreduce/lib/*,$HADOOP_COMMON_HOME/share/hadoop/common/*,$HADOOP_COMMON_HOME/share/hadoop/common/lib/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/*,$HADOOP_HDFS_HOME/share/hadoop/hdfs/lib/*,$HADOOP_YARN_HOME/share/hadoop/yarn/*,$HADOOP_YARN_HOME/share/hadoop/yarn/lib/*`
+
+$H3 NOTE:
+
+If shuffle encryption is also enabled in the cluster, then we could meet the problem that MR job get failed with exception like below:
+
+ 2014-10-10 02:17:16,600 WARN [fetcher#1] org.apache.hadoop.mapreduce.task.reduce.Fetcher: Failed to connect to junpingdu-centos5-3.cs1cloud.internal:13562 with 1 map outputs
+ javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target
+ at com.sun.net.ssl.internal.ssl.Alerts.getSSLException(Alerts.java:174)
+ at com.sun.net.ssl.internal.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1731)
+ at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:241)
+ at com.sun.net.ssl.internal.ssl.Handshaker.fatalSE(Handshaker.java:235)
+ at com.sun.net.ssl.internal.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1206)
+ at com.sun.net.ssl.internal.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:136)
+ at com.sun.net.ssl.internal.ssl.Handshaker.processLoop(Handshaker.java:593)
+ at com.sun.net.ssl.internal.ssl.Handshaker.process_record(Handshaker.java:529)
+ at com.sun.net.ssl.internal.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:925)
+ at com.sun.net.ssl.internal.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1170)
+ at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1197)
+ at com.sun.net.ssl.internal.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1181)
+ at sun.net.www.protocol.https.HttpsClient.afterConnect(HttpsClient.java:434)
+ at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.setNewClient(AbstractDelegateHttpsURLConnection.java:81)
+ at sun.net.www.protocol.https.AbstractDelegateHttpsURLConnection.setNewClient(AbstractDelegateHttpsURLConnection.java:61)
+ at sun.net.www.protocol.http.HttpURLConnection.writeRequests(HttpURLConnection.java:584)
+ at sun.net.www.protocol.http.HttpURLConnection.getInputStream(HttpURLConnection.java:1193)
+ at java.net.HttpURLConnection.getResponseCode(HttpURLConnection.java:379)
+ at sun.net.www.protocol.https.HttpsURLConnectionImpl.getResponseCode(HttpsURLConnectionImpl.java:318)
+ at org.apache.hadoop.mapreduce.task.reduce.Fetcher.verifyConnection(Fetcher.java:427)
+ ....
+
+This is because MR client (deployed from HDFS) cannot access ssl-client.xml in local FS under directory of $HADOOP\_CONF\_DIR. To fix the problem, we can add the directory with ssl-client.xml to the classpath of MR which is specified in "mapreduce.application.classpath" as mentioned above. To avoid MR application being affected by other local configurations, it is better to create a dedicated directory for putting ssl-client.xml, e.g. a sub-directory under $HADOOP\_CONF\_DIR, like: $HADOOP\_CONF\_DIR/security.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
new file mode 100644
index 0000000..58fd52a
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/EncryptedShuffle.md
@@ -0,0 +1,255 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop: Encrypted Shuffle
+=========================
+
+Introduction
+------------
+
+The Encrypted Shuffle capability allows encryption of the MapReduce shuffle using HTTPS and with optional client authentication (also known as bi-directional HTTPS, or HTTPS with client certificates). It comprises:
+
+* A Hadoop configuration setting for toggling the shuffle between HTTP and
+ HTTPS.
+
+* A Hadoop configuration settings for specifying the keystore and truststore
+ properties (location, type, passwords) used by the shuffle service and the
+ reducers tasks fetching shuffle data.
+
+* A way to re-load truststores across the cluster (when a node is added or
+ removed).
+
+Configuration
+-------------
+
+### **core-site.xml** Properties
+
+To enable encrypted shuffle, set the following properties in core-site.xml of all nodes in the cluster:
+
+| **Property** | **Default Value** | **Explanation** |
+|:---- |:---- |:---- |
+| `hadoop.ssl.require.client.cert` | `false` | Whether client certificates are required |
+| `hadoop.ssl.hostname.verifier` | `DEFAULT` | The hostname verifier to provide for HttpsURLConnections. Valid values are: **DEFAULT**, **STRICT**, **STRICT\_I6**, **DEFAULT\_AND\_LOCALHOST** and **ALLOW\_ALL** |
+| `hadoop.ssl.keystores.factory.class` | `org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory` | The KeyStoresFactory implementation to use |
+| `hadoop.ssl.server.conf` | `ssl-server.xml` | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
+| `hadoop.ssl.client.conf` | `ssl-client.xml` | Resource file from which ssl server keystore information will be extracted. This file is looked up in the classpath, typically it should be in Hadoop conf/ directory |
+| `hadoop.ssl.enabled.protocols` | `TLSv1` | The supported SSL protocols (JDK6 can use **TLSv1**, JDK7+ can use **TLSv1,TLSv1.1,TLSv1.2**) |
+
+**IMPORTANT:** Currently requiring client certificates should be set to false. Refer the [Client Certificates](#Client_Certificates) section for details.
+
+**IMPORTANT:** All these properties should be marked as final in the cluster configuration files.
+
+#### Example:
+
+```xml
+ <property>
+ <name>hadoop.ssl.require.client.cert</name>
+ <value>false</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.hostname.verifier</name>
+ <value>DEFAULT</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.keystores.factory.class</name>
+ <value>org.apache.hadoop.security.ssl.FileBasedKeyStoresFactory</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.server.conf</name>
+ <value>ssl-server.xml</value>
+ <final>true</final>
+ </property>
+
+ <property>
+ <name>hadoop.ssl.client.conf</name>
+ <value>ssl-client.xml</value>
+ <final>true</final>
+ </property>
+```
+
+### `mapred-site.xml` Properties
+
+To enable encrypted shuffle, set the following property in mapred-site.xml of all nodes in the cluster:
+
+| **Property** | **Default Value** | **Explanation** |
+|:---- |:---- |:---- |
+| `mapreduce.shuffle.ssl.enabled` | `false` | Whether encrypted shuffle is enabled |
+
+**IMPORTANT:** This property should be marked as final in the cluster configuration files.
+
+#### Example:
+
+```xml
+ <property>
+ <name>mapreduce.shuffle.ssl.enabled</name>
+ <value>true</value>
+ <final>true</final>
+ </property>
+```
+
+The Linux container executor should be set to prevent job tasks from reading the server keystore information and gaining access to the shuffle server certificates.
+
+Refer to Hadoop Kerberos configuration for details on how to do this.
+
+Keystore and Truststore Settings
+--------------------------------
+
+Currently `FileBasedKeyStoresFactory` is the only `KeyStoresFactory` implementation. The `FileBasedKeyStoresFactory` implementation uses the following properties, in the **ssl-server.xml** and **ssl-client.xml** files, to configure the keystores and truststores.
+
+### `ssl-server.xml` (Shuffle server) Configuration:
+
+The mapred user should own the **ssl-server.xml** file and have exclusive read access to it.
+
+| **Property** | **Default Value** | **Explanation** |
+|:---- |:---- |:---- |
+| `ssl.server.keystore.type` | `jks` | Keystore file type |
+| `ssl.server.keystore.location` | NONE | Keystore file location. The mapred user should own this file and have exclusive read access to it. |
+| `ssl.server.keystore.password` | NONE | Keystore file password |
+| `ssl.server.truststore.type` | `jks` | Truststore file type |
+| `ssl.server.truststore.location` | NONE | Truststore file location. The mapred user should own this file and have exclusive read access to it. |
+| `ssl.server.truststore.password` | NONE | Truststore file password |
+| `ssl.server.truststore.reload.interval` | 10000 | Truststore reload interval, in milliseconds |
+
+#### Example:
+
+```xml
+<configuration>
+
+ <!-- Server Certificate Store -->
+ <property>
+ <name>ssl.server.keystore.type</name>
+ <value>jks</value>
+ </property>
+ <property>
+ <name>ssl.server.keystore.location</name>
+ <value>${user.home}/keystores/server-keystore.jks</value>
+ </property>
+ <property>
+ <name>ssl.server.keystore.password</name>
+ <value>serverfoo</value>
+ </property>
+
+ <!-- Server Trust Store -->
+ <property>
+ <name>ssl.server.truststore.type</name>
+ <value>jks</value>
+ </property>
+ <property>
+ <name>ssl.server.truststore.location</name>
+ <value>${user.home}/keystores/truststore.jks</value>
+ </property>
+ <property>
+ <name>ssl.server.truststore.password</name>
+ <value>clientserverbar</value>
+ </property>
+ <property>
+ <name>ssl.server.truststore.reload.interval</name>
+ <value>10000</value>
+ </property>
+</configuration>
+```
+
+### `ssl-client.xml` (Reducer/Fetcher) Configuration:
+
+The mapred user should own the **ssl-client.xml** file and it should have default permissions.
+
+| **Property** | **Default Value** | **Explanation** |
+|:---- |:---- |:---- |
+| `ssl.client.keystore.type` | `jks` | Keystore file type |
+| `ssl.client.keystore.location` | NONE | Keystore file location. The mapred user should own this file and it should have default permissions. |
+| `ssl.client.keystore.password` | NONE | Keystore file password |
+| `ssl.client.truststore.type` | `jks` | Truststore file type |
+| `ssl.client.truststore.location` | NONE | Truststore file location. The mapred user should own this file and it should have default permissions. |
+| `ssl.client.truststore.password` | NONE | Truststore file password |
+| `ssl.client.truststore.reload.interval` | 10000 | Truststore reload interval, in milliseconds |
+
+#### Example:
+
+```xml
+<configuration>
+
+ <!-- Client certificate Store -->
+ <property>
+ <name>ssl.client.keystore.type</name>
+ <value>jks</value>
+ </property>
+ <property>
+ <name>ssl.client.keystore.location</name>
+ <value>${user.home}/keystores/client-keystore.jks</value>
+ </property>
+ <property>
+ <name>ssl.client.keystore.password</name>
+ <value>clientfoo</value>
+ </property>
+
+ <!-- Client Trust Store -->
+ <property>
+ <name>ssl.client.truststore.type</name>
+ <value>jks</value>
+ </property>
+ <property>
+ <name>ssl.client.truststore.location</name>
+ <value>${user.home}/keystores/truststore.jks</value>
+ </property>
+ <property>
+ <name>ssl.client.truststore.password</name>
+ <value>clientserverbar</value>
+ </property>
+ <property>
+ <name>ssl.client.truststore.reload.interval</name>
+ <value>10000</value>
+ </property>
+</configuration>
+```
+
+Activating Encrypted Shuffle
+----------------------------
+
+When you have made the above configuration changes, activate Encrypted Shuffle by re-starting all NodeManagers.
+
+**IMPORTANT:** Using encrypted shuffle will incur in a significant performance impact. Users should profile this and potentially reserve 1 or more cores for encrypted shuffle.
+
+Client Certificates
+-------------------
+
+Using Client Certificates does not fully ensure that the client is a reducer task for the job. Currently, Client Certificates (their private key) keystore files must be readable by all users submitting jobs to the cluster. This means that a rogue job could read such those keystore files and use the client certificates in them to establish a secure connection with a Shuffle server. However, unless the rogue job has a proper JobToken, it won't be able to retrieve shuffle data from the Shuffle server. A job, using its own JobToken, can only retrieve shuffle data that belongs to itself.
+
+Reloading Truststores
+---------------------
+
+By default the truststores will reload their configuration every 10 seconds. If a new truststore file is copied over the old one, it will be re-read, and its certificates will replace the old ones. This mechanism is useful for adding or removing nodes from the cluster, or for adding or removing trusted clients. In these cases, the client or NodeManager certificate is added to (or removed from) all the truststore files in the system, and the new configuration will be picked up without you having to restart the NodeManager daemons.
+
+Debugging
+---------
+
+**NOTE:** Enable debugging only for troubleshooting, and then only for jobs running on small amounts of data. It is very verbose and slows down jobs by several orders of magnitude. (You might need to increase mapred.task.timeout to prevent jobs from failing because tasks run so slowly.)
+
+To enable SSL debugging in the reducers, set `-Djavax.net.debug=all` in the `mapreduce.reduce.child.java.opts` property; for example:
+
+ <property>
+ <name>mapred.reduce.child.java.opts</name>
+ <value>-Xmx-200m -Djavax.net.debug=all</value>
+ </property>
+
+You can do this on a per-job basis, or by means of a cluster-wide setting in the `mapred-site.xml` file.
+
+To set this property in NodeManager, set it in the `yarn-env.sh` file:
+
+ YARN_NODEMANAGER_OPTS="-Djavax.net.debug=all"
[18/50] [abbrv] hadoop git commit: HADOOP-11570.
S3AInputStream.close() downloads the remaining bytes of the object from S3.
(Dan Hecht via stevel).
Posted by zj...@apache.org.
HADOOP-11570. S3AInputStream.close() downloads the remaining bytes of the object from S3. (Dan Hecht via stevel).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/826267f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/826267f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/826267f7
Branch: refs/heads/YARN-2928
Commit: 826267f789df657c62f7f5909e5a0b1a7b102c34
Parents: f0412de
Author: Steve Loughran <st...@apache.org>
Authored: Tue Feb 17 16:36:32 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Feb 17 16:36:44 2015 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/fs/s3a/S3AInputStream.java | 20 ++++++++++++--------
2 files changed, 15 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/826267f7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c3aafe8..0d1ef36 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -939,6 +939,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11000. HAServiceProtocol's health state is incorrectly transitioned
to SERVICE_NOT_RESPONDING (Ming Ma via vinayakumarb)
+ HADOOP-11570. S3AInputStream.close() downloads the remaining bytes of
+ the object from S3. (Dan Hecht via stevel).
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/826267f7/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index 4c56b82..685026e 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -37,14 +37,13 @@ public class S3AInputStream extends FSInputStream {
private long pos;
private boolean closed;
private S3ObjectInputStream wrappedStream;
- private S3Object wrappedObject;
private FileSystem.Statistics stats;
private AmazonS3Client client;
private String bucket;
private String key;
private long contentLength;
public static final Logger LOG = S3AFileSystem.LOG;
-
+ public static final long CLOSE_THRESHOLD = 4096;
public S3AInputStream(String bucket, String key, long contentLength, AmazonS3Client client,
FileSystem.Statistics stats) {
@@ -55,12 +54,11 @@ public class S3AInputStream extends FSInputStream {
this.stats = stats;
this.pos = 0;
this.closed = false;
- this.wrappedObject = null;
this.wrappedStream = null;
}
private void openIfNeeded() throws IOException {
- if (wrappedObject == null) {
+ if (wrappedStream == null) {
reopen(0);
}
}
@@ -90,8 +88,7 @@ public class S3AInputStream extends FSInputStream {
GetObjectRequest request = new GetObjectRequest(bucket, key);
request.setRange(pos, contentLength-1);
- wrappedObject = client.getObject(request);
- wrappedStream = wrappedObject.getObjectContent();
+ wrappedStream = client.getObject(request).getObjectContent();
if (wrappedStream == null) {
throw new IOException("Null IO stream");
@@ -192,8 +189,15 @@ public class S3AInputStream extends FSInputStream {
public synchronized void close() throws IOException {
super.close();
closed = true;
- if (wrappedObject != null) {
- wrappedObject.close();
+ if (wrappedStream != null) {
+ if (contentLength - pos <= CLOSE_THRESHOLD) {
+ // Close, rather than abort, so that the http connection can be reused.
+ wrappedStream.close();
+ } else {
+ // Abort, rather than just close, the underlying stream. Otherwise, the
+ // remaining object payload is read from S3 while closing the stream.
+ wrappedStream.abort();
+ }
}
}
[09/50] [abbrv] hadoop git commit: YARN-1299. Improve a log message
in AppSchedulingInfo by adding application id. Contributed by Ashutosh Jindal
and Devaraj K.
Posted by zj...@apache.org.
YARN-1299. Improve a log message in AppSchedulingInfo by adding application id. Contributed by Ashutosh Jindal and Devaraj K.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/556386a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/556386a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/556386a0
Branch: refs/heads/YARN-2928
Commit: 556386a07084b70a5d2ae0c2bd4445a348306db8
Parents: 3f32357
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Feb 17 01:48:22 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Feb 17 01:53:31 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556386a0/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7161dc3..66543cd 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -298,6 +298,9 @@ Release 2.7.0 - UNRELEASED
YARN-3203. Correct a log message in AuxServices. (Brahma Reddy Battula
via ozawa)
+ YARN-1299. Improve a log message in AppSchedulingInfo by adding application
+ id. (Ashutosh Jindal and Devaraj K via ozawa)
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/556386a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index 3ade7f7..a9a459f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -176,7 +176,8 @@ public class AppSchedulingInfo {
// Similarly, deactivate application?
if (request.getNumContainers() <= 0) {
- LOG.info("checking for deactivate... ");
+ LOG.info("checking for deactivate of application :"
+ + this.applicationId);
checkForDeactivation();
}
[32/50] [abbrv] hadoop git commit: HADOOP-11596 moving entry in
CHANGES.TXT up to improvements there,
as it wasn't in 2.7 & it was confusing diffs
Posted by zj...@apache.org.
HADOOP-11596 moving entry in CHANGES.TXT up to improvements there, as it wasn't in 2.7 & it was confusing diffs
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6be56709
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6be56709
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6be56709
Branch: refs/heads/YARN-2928
Commit: 6be567094983057fa3532f301f7a37667eeae25a
Parents: 13d1ba9
Author: Steve Loughran <st...@apache.org>
Authored: Tue Feb 17 20:06:05 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Feb 17 20:06:05 2015 +0000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6be56709/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0de835a..405e821 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -178,6 +178,9 @@ Trunk (Unreleased)
HADOOP-11575. Daemon log documentation is misleading
(Naganarasimha G R via aw)
+ HADOOP-11596. Allow smart-apply-patch.sh to add new files in binary git
+ patches (raviprak)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
@@ -612,9 +615,6 @@ Release 2.7.0 - UNRELEASED
HADOOP-11522. Update S3A Documentation. (Thomas Demoor via stevel)
- HADOOP-11596. Allow smart-apply-patch.sh to add new files in binary git
- patches (raviprak)
-
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
[08/50] [abbrv] hadoop git commit: Revert "YARN-1299. Improve a log
message in AppSchedulingInfo by adding application id. Contributed by
Ashutosh Jindal and devaraj."
Posted by zj...@apache.org.
Revert "YARN-1299. Improve a log message in AppSchedulingInfo by adding application id. Contributed by Ashutosh Jindal and devaraj."
This reverts commit 9aae81c93421874b726c7b6ff970895c429e502d.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f32357c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f32357c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f32357c
Branch: refs/heads/YARN-2928
Commit: 3f32357c368f4efac33835d719641c961f93a0be
Parents: 9aae81c
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Feb 17 01:52:22 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Feb 17 01:52:22 2015 +0900
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ---
.../yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java | 3 +--
2 files changed, 1 insertion(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f32357c/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index bb145eb..7161dc3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -298,9 +298,6 @@ Release 2.7.0 - UNRELEASED
YARN-3203. Correct a log message in AuxServices. (Brahma Reddy Battula
via ozawa)
- YARN-1299. Improve a log message in AppSchedulingInfo by adding application
- id. (Ashutosh Jindal and devaraj via ozawa)
-
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f32357c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
index a9a459f..3ade7f7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
@@ -176,8 +176,7 @@ public class AppSchedulingInfo {
// Similarly, deactivate application?
if (request.getNumContainers() <= 0) {
- LOG.info("checking for deactivate of application :"
- + this.applicationId);
+ LOG.info("checking for deactivate... ");
checkForDeactivation();
}
[10/50] [abbrv] hadoop git commit: MAPREDUCE-6225. Fix new findbug
warnings in hadoop-mapreduce-client-core. Contributed by Varun Saxena
Posted by zj...@apache.org.
MAPREDUCE-6225. Fix new findbug warnings in hadoop-mapreduce-client-core. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/814afa46
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/814afa46
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/814afa46
Branch: refs/heads/YARN-2928
Commit: 814afa46efef201cb782072432fc744e1cb9c463
Parents: 556386a
Author: Junping Du <ju...@apache.org>
Authored: Mon Feb 16 09:38:05 2015 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Mon Feb 16 09:38:05 2015 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapred/IndexCache.java | 2 +-
.../org/apache/hadoop/mapred/TaskLogAppender.java | 2 +-
.../hadoop/mapred/lib/CombineFileRecordReader.java | 5 -----
.../org/apache/hadoop/mapreduce/JobSubmitter.java | 15 ++++++---------
.../mapreduce/lib/fieldsel/FieldSelectionHelper.java | 2 +-
.../mapreduce/lib/input/CombineFileRecordReader.java | 5 -----
.../mapreduce/security/SecureShuffleUtils.java | 15 ++++++++++-----
.../hadoop/mapreduce/task/reduce/InMemoryReader.java | 4 +---
.../hadoop/mapreduce/util/ResourceBundles.java | 2 +-
10 files changed, 24 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index bb9e105..9ef7a32 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -294,6 +294,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6256. Removed unused private methods in o.a.h.mapreduce.Job.java.
(Naganarasimha G R via ozawa)
+ MAPREDUCE-6225. Fix new findbug warnings in hadoop-mapreduce-client-core.
+ (Varun Saxena via junping_du)
+
OPTIMIZATIONS
MAPREDUCE-6169. MergeQueue should release reference to the current item
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
index 54add3a..c3db951 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/IndexCache.java
@@ -145,7 +145,7 @@ class IndexCache {
*/
public void removeMap(String mapId) {
IndexInformation info = cache.get(mapId);
- if (info == null || ((info != null) && isUnderConstruction(info))) {
+ if (info == null || isUnderConstruction(info)) {
return;
}
info = cache.remove(mapId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java
index 2162a26..d10b764 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/TaskLogAppender.java
@@ -75,7 +75,7 @@ public class TaskLogAppender extends FileAppender implements Flushable {
if (maxEvents == null) {
String propValue = System.getProperty(LOGSIZE_PROPERTY, "0");
- setTotalLogFileSize(Long.valueOf(propValue));
+ setTotalLogFileSize(Long.parseLong(propValue));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
index f54f176..a25eb29 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/lib/CombineFileRecordReader.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapred.lib;
import java.io.*;
import java.lang.reflect.*;
-import org.apache.hadoop.fs.FileSystem;
-
import org.apache.hadoop.mapred.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -49,9 +47,7 @@ public class CombineFileRecordReader<K, V> implements RecordReader<K, V> {
protected CombineFileSplit split;
protected JobConf jc;
protected Reporter reporter;
- protected Class<RecordReader<K, V>> rrClass;
protected Constructor<RecordReader<K, V>> rrConstructor;
- protected FileSystem fs;
protected int idx;
protected long progress;
@@ -106,7 +102,6 @@ public class CombineFileRecordReader<K, V> implements RecordReader<K, V> {
throws IOException {
this.split = split;
this.jc = job;
- this.rrClass = rrClass;
this.reporter = reporter;
this.idx = 0;
this.curReader = null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index ba496ee..75357f7 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -390,10 +390,12 @@ class JobSubmitter {
short replication = (short)conf.getInt(Job.SUBMIT_REPLICATION, 10);
copyAndConfigureFiles(job, jobSubmitDir, replication);
- // Set the working directory
- if (job.getWorkingDirectory() == null) {
- job.setWorkingDirectory(jtFs.getWorkingDirectory());
- }
+ // Get the working directory. If not set, sets it to filesystem working dir
+ // This code has been added so that working directory reset before running
+ // the job. This is necessary for backward compatibility as other systems
+ // might use the public API JobConf#setWorkingDirectory to reset the working
+ // directory.
+ job.getWorkingDirectory();
}
/**
@@ -773,11 +775,6 @@ class JobSubmitter {
if (!log4jPropertyFile.isEmpty()) {
short replication = (short)conf.getInt(Job.SUBMIT_REPLICATION, 10);
copyLog4jPropertyFile(job, jobSubmitDir, replication);
-
- // Set the working directory
- if (job.getWorkingDirectory() == null) {
- job.setWorkingDirectory(jtFs.getWorkingDirectory());
- }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
index 11d6ee2..6e22fe9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/fieldsel/FieldSelectionHelper.java
@@ -90,7 +90,7 @@ public class FieldSelectionHelper {
}
pos = fieldSpec.indexOf('-');
if (pos < 0) {
- Integer fn = new Integer(fieldSpec);
+ Integer fn = Integer.valueOf(fieldSpec);
fieldList.add(fn);
} else {
String start = fieldSpec.substring(0, pos);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
index 767f79a..f71b946 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/input/CombineFileRecordReader.java
@@ -21,8 +21,6 @@ package org.apache.hadoop.mapreduce.lib.input;
import java.io.*;
import java.lang.reflect.*;
-import org.apache.hadoop.fs.FileSystem;
-
import org.apache.hadoop.mapreduce.*;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -46,9 +44,7 @@ public class CombineFileRecordReader<K, V> extends RecordReader<K, V> {
Integer.class};
protected CombineFileSplit split;
- protected Class<? extends RecordReader<K,V>> rrClass;
protected Constructor<? extends RecordReader<K,V>> rrConstructor;
- protected FileSystem fs;
protected TaskAttemptContext context;
protected int idx;
@@ -111,7 +107,6 @@ public class CombineFileRecordReader<K, V> extends RecordReader<K, V> {
throws IOException {
this.split = split;
this.context = context;
- this.rrClass = rrClass;
this.idx = 0;
this.curReader = null;
this.progress = 0;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
index 040a3e3..60919d0 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/security/SecureShuffleUtils.java
@@ -21,8 +21,8 @@ package org.apache.hadoop.mapreduce.security;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintStream;
+import java.io.UnsupportedEncodingException;
import java.net.URL;
-
import javax.crypto.SecretKey;
import javax.servlet.http.HttpServletRequest;
@@ -141,10 +141,15 @@ public class SecureShuffleUtils {
*/
public static String toHex(byte[] ba) {
ByteArrayOutputStream baos = new ByteArrayOutputStream();
- PrintStream ps = new PrintStream(baos);
- for (byte b : ba) {
- ps.printf("%x", b);
+ String strHex = "";
+ try {
+ PrintStream ps = new PrintStream(baos, false, "UTF-8");
+ for (byte b : ba) {
+ ps.printf("%x", b);
+ }
+ strHex = baos.toString("UTF-8");
+ } catch (UnsupportedEncodingException e) {
}
- return baos.toString();
+ return strHex;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
index b246d24..8a6ab44 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/task/reduce/InMemoryReader.java
@@ -79,10 +79,8 @@ public class InMemoryReader<K, V> extends Reader<K, V> {
File dumpFile = new File("../output/" + taskAttemptId + ".dump");
System.err.println("Dumping corrupt map-output of " + taskAttemptId +
" to " + dumpFile.getAbsolutePath());
- try {
- FileOutputStream fos = new FileOutputStream(dumpFile);
+ try (FileOutputStream fos = new FileOutputStream(dumpFile)) {
fos.write(buffer, 0, bufferSize);
- fos.close();
} catch (IOException ioe) {
System.err.println("Failed to dump map-output of " + taskAttemptId);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/814afa46/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java
index 52addcf..4a11740 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ResourceBundles.java
@@ -59,7 +59,7 @@ public class ResourceBundles {
catch (Exception e) {
return defaultValue;
}
- return value == null ? defaultValue : value;
+ return value;
}
private static String getLookupKey(String key, String suffix) {
[41/50] [abbrv] hadoop git commit: HADOOP-11593. Convert site
documentation from apt to markdown (stragglers) (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm b/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
deleted file mode 100644
index a8b408c..0000000
--- a/hadoop-tools/hadoop-sls/src/site/apt/SchedulerLoadSimulator.apt.vm
+++ /dev/null
@@ -1,439 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Yarn Scheduler Load Simulator (SLS)
- ---
- ---
- ${maven.build.timestamp}
-
-Yarn Scheduler Load Simulator (SLS)
-
-%{toc|section=1|fromDepth=0}
-
-* Overview
-
-** Overview
-
- The Yarn scheduler is a fertile area of interest with different
- implementations, e.g., Fifo, Capacity and Fair schedulers. Meanwhile, several
- optimizations are also made to improve scheduler performance for different
- scenarios and workload. Each scheduler algorithm has its own set of features,
- and drives scheduling decisions by many factors, such as fairness, capacity
- guarantee, resource availability, etc. It is very important to evaluate a
- scheduler algorithm very well before we deploy in a production cluster.
- Unfortunately, currently it is non-trivial to evaluate a scheduler algorithm.
- Evaluating in a real cluster is always time and cost consuming, and it is
- also very hard to find a large-enough cluster. Hence, a simulator which can
- predict how well a scheduler algorithm for some specific workload would be
- quite useful.
-
- The Yarn Scheduler Load Simulator (SLS) is such a tool, which can simulate
- large-scale Yarn clusters and application loads in a single machine.This
- simulator would be invaluable in furthering Yarn by providing a tool for
- researchers and developers to prototype new scheduler features and predict
- their behavior and performance with reasonable amount of confidence,
- thereby aiding rapid innovation.
-
- The simulator will exercise the real Yarn <<<ResourceManager>>> removing the
- network factor by simulating <<<NodeManagers>>> and <<<ApplicationMasters>>>
- via handling and dispatching <<<NM>>>/<<<AMs>>> heartbeat events from within
- the same JVM. To keep tracking of scheduler behavior and performance, a
- scheduler wrapper will wrap the real scheduler.
-
- The size of the cluster and the application load can be loaded from
- configuration files, which are generated from job history files directly by
- adopting {{{https://hadoop.apache.org/docs/stable/rumen.html}Apache Rumen}}.
-
- The simulator will produce real time metrics while executing, including:
-
- * Resource usages for whole cluster and each queue, which can be utilized to
- configure cluster and queue's capacity.
-
- * The detailed application execution trace (recorded in relation to simulated
- time), which can be analyzed to understand/validate the scheduler behavior
- (individual jobs turn around time, throughput, fairness, capacity guarantee,
- etc.).
-
- * Several key metrics of scheduler algorithm, such as time cost of each
- scheduler operation (allocate, handle, etc.), which can be utilized by Hadoop
- developers to find the code spots and scalability limits.
-
-** Goals
-
- * Exercise the scheduler at scale without a real cluster using real job
- traces.
-
- * Being able to simulate real workloads.
-
-** Architecture
-
- The following figure illustrates the implementation architecture of the
- simulator.
-
-[images/sls_arch.png] The architecture of the simulator
-
- The simulator takes input of workload traces, and fetches the cluster and
- applications information. For each NM and AM, the simulator builds a simulator
- to simulate their running. All NM/AM simulators run in a thread pool. The
- simulator reuses Yarn Resource Manager, and builds a wrapper out of the
- scheduler. The Scheduler Wrapper can track the scheduler behaviors and
- generates several logs, which are the outputs of the simulator and can be
- further analyzed.
-
-** Usecases
-
- * Engineering
-
- * Verify correctness of scheduler algorithm under load
-
- * Cheap/practical way for finding code hotspots/critical-path.
-
- * Validate the impact of changes and new features.
-
- * Determine what drives the scheduler scalability limits.
-
- []
-
- * QA
-
- * Validate scheduler behavior for "large" clusters and several workload
- profiles.
-
- * Solutions/Sales.
-
- * Sizing model for predefined/typical workloads.
-
- * Cluster sizing tool using real customer data (job traces).
-
- * Determine minimum SLAs under a particular workload.
-
-* Usage
-
- This section will show how to use the simulator. Here let <<<$HADOOP_ROOT>>>
- represent the Hadoop install directory. If you build Hadoop yourself,
- <<<$HADOOP_ROOT>>> is <<<hadoop-dist/target/hadoop-$VERSION>>>. The simulator
- is located at <<<$HADOOP_ROOT/share/hadoop/tools/sls>>>. The fold <<<sls>>>
- containers four directories: <<<bin>>>, <<<html>>>, <<<sample-conf>>>, and
- <<<sample-data>>>
-
- * <<<bin>>>: contains running scripts for the simulator.
-
- * <<<html>>>: contains several html/css/js files we needed for real-time
- tracking.
-
- * <<<sample-conf>>>: specifies the simulator configurations.
-
- * <<<sample-data>>>: provides an example rumen trace, which can be used to
- generate inputs of the simulator.
-
- []
-
- The following sections will describe how to use the simulator step by step.
- Before start, make sure that command <<<hadoop>>> is included in your
- <<<$PATH>>> environment parameter.
-
-** Step 1: Configure Hadoop and the simulator
-
- Before we start, make sure Hadoop and the simulator are configured well.
- All configuration files for Hadoop and the simulator should be placed in
- directory <<<$HADOOP_ROOT/etc/hadoop>>>, where the <<<ResourceManager>>>
- and Yarn scheduler load their configurations. Directory
- <<<$HADOOP_ROOT/share/hadoop/tools/sls/sample-conf/>>> provides several
- example configurations, that can be used to start a demo.
-
- For configuration of Hadoop and Yarn scheduler, users can refer to Yarn’s
- website ({{{http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/}
- http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/}}).
-
- For the simulator, it loads configuration information from file
- <<<$HADOOP_ROOT/etc/hadoop/sls-runner.xml>>>.
-
- Here we illustrate each configuration parameter in <<<sls-runner.xml>>>.
- Note that <<<$HADOOP_ROOT/share/hadoop/tools/sls/sample-conf/sls-runner.xml>>>
- contains all the default values for these configuration parameters.
-
- * <<<yarn.sls.runner.pool.size>>>
-
- The simulator uses a thread pool to simulate the <<<NM>>> and <<<AM>>> running
- , and this parameter specifies the number of threads in the pool.
-
- * <<<yarn.sls.nm.memory.mb>>>
-
- The total memory for each <<<NMSimulator>>>.
-
- * <<<yarn.sls.nm.vcores>>>
-
- The total vCores for each <<<NMSimulator>>>.
-
- * <<<yarn.sls.nm.heartbeat.interval.ms>>>
-
- The heartbeat interval for each <<<NMSimulator>>>.
-
- * <<<yarn.sls.am.heartbeat.interval.ms>>>
-
- The heartbeat interval for each <<<AMSimulator>>>.
-
- * <<<yarn.sls.am.type.mapreduce>>>
-
- The <<<AMSimulator>>> implementation for MapReduce-like applications.
- Users can specify implementations for other type of applications.
-
- * <<<yarn.sls.container.memory.mb>>>
-
- The memory required for each container simulator.
-
- * <<<yarn.sls.container.vcores>>>
-
- The vCores required for each container simulator.
-
- * <<<yarn.sls.runner.metrics.switch>>>
-
- The simulator introduces {{{http://metrics.codahale.com/}Metrics}} to measure
- the behaviors of critical components and operations. This field specifies
- whether we open (<<<ON>>>) or close (<<<OFF>>>) the Metrics running.
-
- * <<<yarn.sls.metrics.web.address.port>>>
-
- The port used by simulator to provide real-time tracking. The default value is
- 10001.
-
- * <<<org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler>>>
-
- The implementation of scheduler metrics of Fifo Scheduler.
-
- * <<<org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler>>>
-
- The implementation of scheduler metrics of Fair Scheduler.
-
- * <<<org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler>>>
-
- The implementation of scheduler metrics of Capacity Scheduler.
-
-** Step 2: Run the simulator
-
- The simulator supports two types of input files: the rumen traces and its own
- input traces. The script to start the simulator is <<<slsrun.sh>>>.
-
-+----+
-$ cd $HADOOP_ROOT/share/hadoop/tools/sls
-$ bin/slsrun.sh
- --input-rumen|--input-sls=<TRACE_FILE1,TRACE_FILE2,...>
- --output-dir=<SLS_SIMULATION_OUTPUT_DIRECTORY> [--nodes=<SLS_NODES_FILE>]
- [--track-jobs=<JOBID1,JOBID2,...>] [--print-simulation]
-+----+
-
- * <<<--input-rumen>>>: The input rumen trace files. Users can input multiple
- files, separated by comma. One example trace is provided in
- <<<$HADOOP_ROOT/share/hadoop/tools/sls/sample-data/2jobs2min-rumen-jh.json>>>.
-
- * <<<--input-sls>>>: Simulator its own file format. The simulator also
- provides a tool to convert rumen traces to sls traces (<<<rumen2sls.sh>>>).
- Refer to appendix for an example of sls input json file.
-
- * <<<--output-dir>>>: The output directory for generated running logs and
- metrics.
-
- * <<<--nodes>>>: The cluster topology. By default, the simulator will use the
- topology fetched from the input json files. Users can specifies a new topology
- by setting this parameter. Refer to the appendix for the topology file format.
-
- * <<<--track-jobs>>>: The particular jobs that will be tracked during
- simulator running, spearated by comma.
-
- * <<<--print-simulation>>>: Whether to print out simulation information
- before simulator running, including number of nodes, applications, tasks,
- and information for each application.
-
- In comparison to rumen format, here the sls format is much simpler and users
- can easily generate various workload. The simulator also provides a tool to
- convert rumen traces to sls traces.
-
-+----+
-$ bin/rumen2sls.sh
- --rumen-file=<RUMEN_FILE>
- --output-dir=<SLS_OUTPUT_DIRECTORY>
- [--output-prefix=<SLS_FILE_PREFIX>]
-+----+
-
- * <<<--rumen-file>>>: The rumen format file. One example trace is provided
- in directory <<<sample-data>>>.
-
- * <<<--output-dir>>>: The output directory of generated simulation traces.
- Two files will be generated in this output directory, including one trace
- file including all job and task information, and another file showing the
- topology information.
-
- * <<<--output-prefix>>>: The prefix of the generated files. The default value
- is ”sls”, and the two generated files are <<<sls-jobs.json>>> and
- <<<sls-nodes.json>>>.
-
-* Metrics
-
- The Yarn Scheduler Load Simulator has integrated
- {{{http://metrics.codahale.com/}Metrics}} to measure the behaviors of critical
- components and operations, including running applications and containers,
- cluster available resources, scheduler operation timecost, et al. If the
- switch <<<yarn.sls.runner.metrics.switch>>> is set <<<ON>>>, <<<Metrics>>>
- will run and output it logs in <<<--output-dir>>> directory specified by users.
- Users can track these information during simulator running, and can also
- analyze these logs after running to evaluate the scheduler performance.
-
-** Real-time Tracking
-
- The simulator provides an interface for tracking its running in real-time.
- Users can go to <<<http://host:port/simulate>>> to track whole running,
- and <<<http://host:port/track>>> to track a particular job or queue. Here
- the <<<host>>> is the place when we run the simulator, and <<<port>>> is
- the value configured by <<<yarn.sls.metrics.web.address.port>>> (default value
- is 10001).
-
- Here we'll illustrate each chart shown in the webpage.
-
- The first figure describes the number of running applications and containers.
-
-[images/sls_running_apps_containers.png] Number of running applications/containers
-
- The second figure describes the allocated and available resources (memory)
- in the cluster.
-
-[images/sls_cluster_memory.png] Cluster Resource (Memory)
-
- The third figure describes the allocated resource for each queue. Here we have
- three queues: sls_queue_1, sls_queue_2, and sls_queue_3.The first two queues
- are configured with 25% share, while the last one has 50% share.
-
-[images/sls_queue_allocated_memory.png] Queue Allocated Resource (Memory)
-
- The fourth figure describes the timecost for each scheduler operation.
-
-[images/sls_scheduler_operation_timecost.png] Scheduler Opertion Timecost
-
- Finally, we measure the memory used by the simulator.
-
-[images/sls_JVM.png] JVM Memory
-
- The simulator also provides an interface for tracking some particular
- jobs and queues. Go to <<<http://<Host>:<Port>/track>>> to get these
- information.
-
- Here the first figure illustrates the resource usage information for queue
- <<<SLS_Queue_1>>>.
-
-[images/sls_track_queue.png] Tracking Queue <<<sls_queue_3>>>
-
- The second figure illustrates the resource usage information for job
- <<<job_1369942127770_0653>>>.
-
-[images/sls_track_job.png] Tracking Job <<<job_1369942127770_0653>>>
-
-** Offline Analysis
-
- After the simulator finishes, all logs are saved in the output directory
- specified by <<<--output-dir>>> in
- <<<$HADOOP_ROOT/share/hadoop/tools/sls/bin/slsrun.sh>>>.
-
- * File <<<realtimetrack.json>>>: records all real-time tracking logs every 1
- second.
-
- * File <<<jobruntime.csv>>>: records all jobs’ start and end time in the
- simulator.
-
- * Folder <<<metrics>>>: logs generated by the Metrics.
-
- []
-
- Users can also reproduce those real-time tracking charts in offline mode.
- Just upload the <<<realtimetrack.json>>> to
- <<<$HADOOP_ROOT/share/hadoop/tools/sls/html/showSimulationTrace.html>>>.
- For browser security problem, need to put files <<<realtimetrack.json>>> and
- <<<showSimulationTrace.html>>> in the same directory.
-
-* Appendix
-
-** Resources
-
- {{{https://issues.apache.org/jira/browse/YARN-1021}YARN-1021}} is the main
- JIRA that introduces Yarn Scheduler Load Simulator to Hadoop Yarn project.
-
-** SLS JSON input file format
-
- Here we provide an example format of the sls json file, which contains 2 jobs.
- The first job has 3 map tasks and the second one has 2 map tasks.
-
-+----+
-{
- "am.type" : "mapreduce",
- "job.start.ms" : 0,
- "job.end.ms" : 95375,
- "job.queue.name" : "sls_queue_1",
- "job.id" : "job_1",
- "job.user" : "default",
- "job.tasks" : [ {
- "container.host" : "/default-rack/node1",
- "container.start.ms" : 6664,
- "container.end.ms" : 23707,
- "container.priority" : 20,
- "container.type" : "map"
- }, {
- "container.host" : "/default-rack/node3",
- "container.start.ms" : 6665,
- "container.end.ms" : 21593,
- "container.priority" : 20,
- "container.type" : "map"
- }, {
- "container.host" : "/default-rack/node2",
- "container.start.ms" : 68770,
- "container.end.ms" : 86613,
- "container.priority" : 20,
- "container.type" : "map"
- } ]
-}
-{
- "am.type" : "mapreduce",
- "job.start.ms" : 105204,
- "job.end.ms" : 197256,
- "job.queue.name" : "sls_queue_2",
- "job.id" : "job_2",
- "job.user" : "default",
- "job.tasks" : [ {
- "container.host" : "/default-rack/node1",
- "container.start.ms" : 111822,
- "container.end.ms" : 133985,
- "container.priority" : 20,
- "container.type" : "map"
- }, {
- "container.host" : "/default-rack/node2",
- "container.start.ms" : 111788,
- "container.end.ms" : 131377,
- "container.priority" : 20,
- "container.type" : "map"
- } ]
-}
-+----+
-
-** Simulator input topology file format
-
- Here is an example input topology file which has 3 nodes organized in 1 rack.
-
-+----+
-{
- "rack" : "default-rack",
- "nodes" : [ {
- "node" : "node1"
- }, {
- "node" : "node2"
- }, {
- "node" : "node3"
- }]
-}
-+----+
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
new file mode 100644
index 0000000..ca179ee
--- /dev/null
+++ b/hadoop-tools/hadoop-sls/src/site/markdown/SchedulerLoadSimulator.md
@@ -0,0 +1,357 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Yarn Scheduler Load Simulator (SLS)
+===================================
+
+* [Yarn Scheduler Load Simulator (SLS)](#Yarn_Scheduler_Load_Simulator_SLS)
+ * [Overview](#Overview)
+ * [Overview](#Overview)
+ * [Goals](#Goals)
+ * [Architecture](#Architecture)
+ * [Usecases](#Usecases)
+ * [Usage](#Usage)
+ * [Step 1: Configure Hadoop and the simulator](#Step_1:_Configure_Hadoop_and_the_simulator)
+ * [Step 2: Run the simulator](#Step_2:_Run_the_simulator)
+ * [Metrics](#Metrics)
+ * [Real-time Tracking](#Real-time_Tracking)
+ * [Offline Analysis](#Offline_Analysis)
+ * [Appendix](#Appendix)
+ * [Resources](#Resources)
+ * [SLS JSON input file format](#SLS_JSON_input_file_format)
+ * [Simulator input topology file format](#Simulator_input_topology_file_format)
+
+Overview
+--------
+
+### Overview
+
+The Yarn scheduler is a fertile area of interest with different implementations, e.g., Fifo, Capacity and Fair schedulers. Meanwhile, several optimizations are also made to improve scheduler performance for different scenarios and workload. Each scheduler algorithm has its own set of features, and drives scheduling decisions by many factors, such as fairness, capacity guarantee, resource availability, etc. It is very important to evaluate a scheduler algorithm very well before we deploy in a production cluster. Unfortunately, currently it is non-trivial to evaluate a scheduler algorithm. Evaluating in a real cluster is always time and cost consuming, and it is also very hard to find a large-enough cluster. Hence, a simulator which can predict how well a scheduler algorithm for some specific workload would be quite useful.
+
+The Yarn Scheduler Load Simulator (SLS) is such a tool, which can simulate large-scale Yarn clusters and application loads in a single machine.This simulator would be invaluable in furthering Yarn by providing a tool for researchers and developers to prototype new scheduler features and predict their behavior and performance with reasonable amount of confidence, thereby aiding rapid innovation.
+o
+The simulator will exercise the real Yarn `ResourceManager` removing the network factor by simulating `NodeManagers` and `ApplicationMasters` via handling and dispatching `NM`/`AMs` heartbeat events from within the same JVM. To keep tracking of scheduler behavior and performance, a scheduler wrapper will wrap the real scheduler.
+
+The size of the cluster and the application load can be loaded from configuration files, which are generated from job history files directly by adopting [Apache Rumen](https://hadoop.apache.org/docs/stable/rumen.html).
+
+The simulator will produce real time metrics while executing, including:
+
+* Resource usages for whole cluster and each queue, which can be utilized to
+ configure cluster and queue's capacity.
+
+* The detailed application execution trace (recorded in relation to simulated
+ time), which can be analyzed to understand/validate the scheduler behavior
+ (individual jobs turn around time, throughput, fairness, capacity guarantee,
+ etc.).
+
+* Several key metrics of scheduler algorithm, such as time cost of each
+ scheduler operation (allocate, handle, etc.), which can be utilized by Hadoop
+ developers to find the code spots and scalability limits.
+
+### Goals
+
+* Exercise the scheduler at scale without a real cluster using real job
+ traces.
+
+* Being able to simulate real workloads.
+
+### Architecture
+
+The following figure illustrates the implementation architecture of the simulator.
+
+
+
+The simulator takes input of workload traces, and fetches the cluster and applications information. For each NM and AM, the simulator builds a simulator to simulate their running. All NM/AM simulators run in a thread pool. The simulator reuses Yarn Resource Manager, and builds a wrapper out of the scheduler. The Scheduler Wrapper can track the scheduler behaviors and generates several logs, which are the outputs of the simulator and can be further analyzed.
+
+### Usecases
+
+* Engineering
+ * Verify correctness of scheduler algorithm under load
+ * Cheap/practical way for finding code hotspots/critical-path.
+ * Validate the impact of changes and new features.
+ * Determine what drives the scheduler scalability limits.
+
+* QA
+ * Validate scheduler behavior for "large" clusters and several workload profiles.
+
+* Solutions/Sales.
+ * Sizing model for predefined/typical workloads.
+ * Cluster sizing tool using real customer data (job traces).
+ * Determine minimum SLAs under a particular workload.
+
+Usage
+-----
+
+This section will show how to use the simulator. Here let `$HADOOP_ROOT` represent the Hadoop install directory. If you build Hadoop yourself, `$HADOOP_ROOT` is `hadoop-dist/target/hadoop-$VERSION`. The simulator is located at `$HADOOP_ROOT/share/hadoop/tools/sls`. The fold `sls` containers four directories: `bin`, `html`, `sample-conf`, and `sample-data`
+
+* `bin`: contains running scripts for the simulator.
+
+* `html`: contains several html/css/js files we needed for real-time tracking.
+
+* `sample-conf`: specifies the simulator configurations.
+
+* `sample-data`: provides an example rumen trace, which can be used to
+ generate inputs of the simulator.
+
+The following sections will describe how to use the simulator step by step. Before start, make sure that command `hadoop` is included in your `$PATH` environment parameter.
+
+### Step 1: Configure Hadoop and the simulator
+
+Before we start, make sure Hadoop and the simulator are configured well. All configuration files for Hadoop and the simulator should be placed in directory `$HADOOP_ROOT/etc/hadoop`, where the `ResourceManager` and Yarn scheduler load their configurations. Directory `$HADOOP_ROOT/share/hadoop/tools/sls/sample-conf/` provides several example configurations, that can be used to start a demo.
+
+For configuration of Hadoop and Yarn scheduler, users can refer to Yarn’s website (<http://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/>).
+
+For the simulator, it loads configuration information from file `$HADOOP_ROOT/etc/hadoop/sls-runner.xml`.
+
+Here we illustrate each configuration parameter in `sls-runner.xml`. Note that `$HADOOP_ROOT/share/hadoop/tools/sls/sample-conf/sls-runner.xml` contains all the default values for these configuration parameters.
+
+* `yarn.sls.runner.pool.size`
+
+ The simulator uses a thread pool to simulate the `NM` and `AM` running,
+ and this parameter specifies the number of threads in the pool.
+
+* `yarn.sls.nm.memory.mb`
+
+ The total memory for each `NMSimulator`.
+
+* `yarn.sls.nm.vcores`
+
+ The total vCores for each `NMSimulator`.
+
+* `yarn.sls.nm.heartbeat.interval.ms`
+
+ The heartbeat interval for each `NMSimulator`.
+
+* `yarn.sls.am.heartbeat.interval.ms`
+
+ The heartbeat interval for each `AMSimulator`.
+
+* `yarn.sls.am.type.mapreduce`
+
+ The `AMSimulator` implementation for MapReduce-like applications.
+ Users can specify implementations for other type of applications.
+
+* `yarn.sls.container.memory.mb`
+
+ The memory required for each container simulator.
+
+* `yarn.sls.container.vcores`
+
+ The vCores required for each container simulator.
+
+* `yarn.sls.runner.metrics.switch`
+
+ The simulator introduces [Metrics](http://metrics.codahale.com/) to measure
+ the behaviors of critical components and operations. This field specifies
+ whether we open (`ON`) or close (`OFF`) the Metrics running.
+
+* `yarn.sls.metrics.web.address.port`
+
+ The port used by simulator to provide real-time tracking. The default value is
+ 10001.
+
+* `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler`
+
+ The implementation of scheduler metrics of Fifo Scheduler.
+
+* `org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`
+
+ The implementation of scheduler metrics of Fair Scheduler.
+
+* `org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler`
+
+ The implementation of scheduler metrics of Capacity Scheduler.
+
+### Step 2: Run the simulator
+
+The simulator supports two types of input files: the rumen traces and its own input traces. The script to start the simulator is `slsrun.sh`.
+
+ $ cd $HADOOP_ROOT/share/hadoop/tools/sls
+ $ bin/slsrun.sh
+ --input-rumen |--input-sls=<TRACE_FILE1,TRACE_FILE2,...>
+ --output-dir=<SLS_SIMULATION_OUTPUT_DIRECTORY> [--nodes=<SLS_NODES_FILE>]
+ [--track-jobs=<JOBID1,JOBID2,...>] [--print-simulation]
+
+* `--input-rumen`: The input rumen trace files. Users can input multiple
+ files, separated by comma. One example trace is provided in
+ `$HADOOP_ROOT/share/hadoop/tools/sls/sample-data/2jobs2min-rumen-jh.json`.
+
+* `--input-sls`: Simulator its own file format. The simulator also
+ provides a tool to convert rumen traces to sls traces (`rumen2sls.sh`).
+ Refer to appendix for an example of sls input json file.
+
+* `--output-dir`: The output directory for generated running logs and
+ metrics.
+
+* `--nodes`: The cluster topology. By default, the simulator will use the
+ topology fetched from the input json files. Users can specifies a new topology
+ by setting this parameter. Refer to the appendix for the topology file format.
+
+* `--track-jobs`: The particular jobs that will be tracked during
+ simulator running, spearated by comma.
+
+* `--print-simulation`: Whether to print out simulation information
+ before simulator running, including number of nodes, applications, tasks,
+ and information for each application.
+
+ In comparison to rumen format, here the sls format is much simpler and users
+ can easily generate various workload. The simulator also provides a tool to
+ convert rumen traces to sls traces.
+
+ $ bin/rumen2sls.sh
+ --rumen-file=<RUMEN_FILE>
+ --output-dir=<SLS_OUTPUT_DIRECTORY>
+ [--output-prefix=<SLS_FILE_PREFIX>]
+
+* `--rumen-file`: The rumen format file. One example trace is provided
+ in directory `sample-data`.
+
+* `--output-dir`: The output directory of generated simulation traces.
+ Two files will be generated in this output directory, including one trace
+ file including all job and task information, and another file showing the
+ topology information.
+
+* `--output-prefix`: The prefix of the generated files. The default value
+ is "sls", and the two generated files are `sls-jobs.json` and
+ `sls-nodes.json`.
+
+Metrics
+-------
+
+The Yarn Scheduler Load Simulator has integrated [Metrics](http://metrics.codahale.com/) to measure the behaviors of critical components and operations, including running applications and containers, cluster available resources, scheduler operation timecost, et al. If the switch `yarn.sls.runner.metrics.switch` is set `ON`, `Metrics` will run and output it logs in `--output-dir` directory specified by users. Users can track these information during simulator running, and can also analyze these logs after running to evaluate the scheduler performance.
+
+### Real-time Tracking
+
+The simulator provides an interface for tracking its running in real-time. Users can go to `http://host:port/simulate` to track whole running, and `http://host:port/track` to track a particular job or queue. Here the `host` is the place when we run the simulator, and `port` is the value configured by `yarn.sls.metrics.web.address.port` (default value is 10001).
+
+Here we'll illustrate each chart shown in the webpage.
+
+The first figure describes the number of running applications and containers.
+
+
+
+The second figure describes the allocated and available resources (memory) in the cluster.
+
+
+
+The third figure describes the allocated resource for each queue. Here we have three queues: sls\_queue\_1, sls\_queue\_2, and sls\_queue\_3.The first two queues are configured with 25% share, while the last one has 50% share.
+
+
+
+The fourth figure describes the timecost for each scheduler operation.
+
+
+
+Finally, we measure the memory used by the simulator.
+
+
+
+The simulator also provides an interface for tracking some particular jobs and queues. Go to `http://<Host>:<Port>/track` to get these information.
+
+Here the first figure illustrates the resource usage information for queue `SLS_Queue_1`.
+
+
+
+The second figure illustrates the resource usage information for job `job_1369942127770_0653`.
+
+
+
+### Offline Analysis
+
+After the simulator finishes, all logs are saved in the output directory specified by `--output-dir` in `$HADOOP_ROOT/share/hadoop/tools/sls/bin/slsrun.sh`.
+
+* File `realtimetrack.json`: records all real-time tracking logs every 1
+ second.
+
+* File `jobruntime.csv`: records all jobs’ start and end time in the
+ simulator.
+
+* Folder `metrics`: logs generated by the Metrics.
+
+Users can also reproduce those real-time tracking charts in offline mode. Just upload the `realtimetrack.json` to `$HADOOP_ROOT/share/hadoop/tools/sls/html/showSimulationTrace.html`. For browser security problem, need to put files `realtimetrack.json` and `showSimulationTrace.html` in the same directory.
+
+Appendix
+--------
+
+### Resources
+
+[YARN-1021](https://issues.apache.org/jira/browse/YARN-1021) is the main JIRA that introduces Yarn Scheduler Load Simulator to Hadoop Yarn project.
+
+### SLS JSON input file format
+
+Here we provide an example format of the sls json file, which contains 2 jobs. The first job has 3 map tasks and the second one has 2 map tasks.
+
+ {
+ "am.type" : "mapreduce",
+ "job.start.ms" : 0,
+ "job.end.ms" : 95375,
+ "job.queue.name" : "sls_queue_1",
+ "job.id" : "job_1",
+ "job.user" : "default",
+ "job.tasks" : [ {
+ "container.host" : "/default-rack/node1",
+ "container.start.ms" : 6664,
+ "container.end.ms" : 23707,
+ "container.priority" : 20,
+ "container.type" : "map"
+ }, {
+ "container.host" : "/default-rack/node3",
+ "container.start.ms" : 6665,
+ "container.end.ms" : 21593,
+ "container.priority" : 20,
+ "container.type" : "map"
+ }, {
+ "container.host" : "/default-rack/node2",
+ "container.start.ms" : 68770,
+ "container.end.ms" : 86613,
+ "container.priority" : 20,
+ "container.type" : "map"
+ } ]
+ }
+ {
+ "am.type" : "mapreduce",
+ "job.start.ms" : 105204,
+ "job.end.ms" : 197256,
+ "job.queue.name" : "sls_queue_2",
+ "job.id" : "job_2",
+ "job.user" : "default",
+ "job.tasks" : [ {
+ "container.host" : "/default-rack/node1",
+ "container.start.ms" : 111822,
+ "container.end.ms" : 133985,
+ "container.priority" : 20,
+ "container.type" : "map"
+ }, {
+ "container.host" : "/default-rack/node2",
+ "container.start.ms" : 111788,
+ "container.end.ms" : 131377,
+ "container.priority" : 20,
+ "container.type" : "map"
+ } ]
+ }
+
+### Simulator input topology file format
+
+Here is an example input topology file which has 3 nodes organized in 1 rack.
+
+ {
+ "rack" : "default-rack",
+ "nodes" : [ {
+ "node" : "node1"
+ }, {
+ "node" : "node2"
+ }, {
+ "node" : "node3"
+ }]
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-streaming/src/site/apt/HadoopStreaming.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/site/apt/HadoopStreaming.apt.vm b/hadoop-tools/hadoop-streaming/src/site/apt/HadoopStreaming.apt.vm
deleted file mode 100644
index 8be92b5..0000000
--- a/hadoop-tools/hadoop-streaming/src/site/apt/HadoopStreaming.apt.vm
+++ /dev/null
@@ -1,792 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Streaming
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Streaming
-
-%{toc|section=1|fromDepth=0|toDepth=4}
-
-* Hadoop Streaming
-
- Hadoop streaming is a utility that comes with the Hadoop distribution. The
- utility allows you to create and run Map/Reduce jobs with any executable or
- script as the mapper and/or the reducer. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -input myInputDirs \
- -output myOutputDir \
- -mapper /bin/cat \
- -reducer /usr/bin/wc
-+---+
-
-* How Streaming Works
-
- In the above example, both the mapper and the reducer are executables that
- read the input from stdin (line by line) and emit the output to stdout. The
- utility will create a Map/Reduce job, submit the job to an appropriate
- cluster, and monitor the progress of the job until it completes.
-
- When an executable is specified for mappers, each mapper task will launch the
- executable as a separate process when the mapper is initialized. As the
- mapper task runs, it converts its inputs into lines and feed the lines to the
- stdin of the process. In the meantime, the mapper collects the line oriented
- outputs from the stdout of the process and converts each line into a
- key/value pair, which is collected as the output of the mapper. By default,
- the <prefix of a line up to the first tab character> is the <<<key>>> and the
- rest of the line (excluding the tab character) will be the <<<value>>>. If
- there is no tab character in the line, then entire line is considered as key
- and the value is null. However, this can be customized by setting
- <<<-inputformat>>> command option, as discussed later.
-
- When an executable is specified for reducers, each reducer task will launch
- the executable as a separate process then the reducer is initialized. As the
- reducer task runs, it converts its input key/values pairs into lines and
- feeds the lines to the stdin of the process. In the meantime, the reducer
- collects the line oriented outputs from the stdout of the process, converts
- each line into a key/value pair, which is collected as the output of the
- reducer. By default, the prefix of a line up to the first tab character is
- the key and the rest of the line (excluding the tab character) is the value.
- However, this can be customized by setting <<<-outputformat>>> command
- option, as discussed later.
-
- This is the basis for the communication protocol between the Map/Reduce
- framework and the streaming mapper/reducer.
-
- User can specify <<<stream.non.zero.exit.is.failure>>> as <<<true>>> or
- <<<false>>> to make a streaming task that exits with a non-zero status to be
- <<<Failure>>> or <<<Success>>> respectively. By default, streaming tasks
- exiting with non-zero status are considered to be failed tasks.
-
-* Streaming Command Options
-
- Streaming supports streaming command options as well as
- {{{Generic_Command_Options}generic command options}}. The general command
- line syntax is shown below.
-
- <<Note:>> Be sure to place the generic options before the streaming options,
- otherwise the command will fail. For an example, see
- {{{Making_Archives_Available_to_Tasks}Making Archives Available to Tasks}}.
-
-+---+
-hadoop command [genericOptions] [streamingOptions]
-+---+
-
- The Hadoop streaming command options are listed here:
-
-*-------------*--------------------*------------------------------------------*
-|| Parameter || Optional/Required || Description |
-*-------------+--------------------+------------------------------------------+
-| -input directoryname or filename | Required | Input location for mapper
-*-------------+--------------------+------------------------------------------+
-| -output directoryname | Required | Output location for reducer
-*-------------+--------------------+------------------------------------------+
-| -mapper executable or JavaClassName | Required | Mapper executable
-*-------------+--------------------+------------------------------------------+
-| -reducer executable or JavaClassName | Required | Reducer executable
-*-------------+--------------------+------------------------------------------+
-| -file filename | Optional | Make the mapper, reducer, or combiner executable
-| | | available locally on the compute nodes
-*-------------+--------------------+------------------------------------------+
-| -inputformat JavaClassName | Optional | Class you supply should return
-| | | key/value pairs of Text class. If not
-| | | specified, TextInputFormat is used as
-| | | the default
-*-------------+--------------------+------------------------------------------+
-| -outputformat JavaClassName | Optional | Class you supply should take
-| | | key/value pairs of Text class. If
-| | | not specified, TextOutputformat is
-| | | used as the default
-*-------------+--------------------+------------------------------------------+
-| -partitioner JavaClassName | Optional | Class that determines which reduce a
-| | | key is sent to
-*-------------+--------------------+------------------------------------------+
-| -combiner streamingCommand | Optional | Combiner executable for map output
-| or JavaClassName | |
-*-------------+--------------------+------------------------------------------+
-| -cmdenv name=value | Optional | Pass environment variable to streaming
-| | | commands
-*-------------+--------------------+------------------------------------------+
-| -inputreader | Optional | For backwards-compatibility: specifies a record
-| | | reader class (instead of an input format class)
-*-------------+--------------------+------------------------------------------+
-| -verbose | Optional | Verbose output
-*-------------+--------------------+------------------------------------------+
-| -lazyOutput | Optional | Create output lazily. For example, if the output
-| | | format is based on FileOutputFormat, the output file
-| | | is created only on the first call to Context.write
-*-------------+--------------------+------------------------------------------+
-| -numReduceTasks | Optional | Specify the number of reducers
-*-------------+--------------------+------------------------------------------+
-| -mapdebug | Optional | Script to call when map task fails
-*-------------+--------------------+------------------------------------------+
-| -reducedebug | Optional | Script to call when reduce task fails
-*-------------+--------------------+------------------------------------------+
-
-** Specifying a Java Class as the Mapper/Reducer
-
- You can supply a Java class as the mapper and/or the reducer.
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -input myInputDirs \
- -output myOutputDir \
- -inputformat org.apache.hadoop.mapred.KeyValueTextInputFormat \
- -mapper org.apache.hadoop.mapred.lib.IdentityMapper \
- -reducer /usr/bin/wc
-+---+
-
- You can specify <<<stream.non.zero.exit.is.failure>>> as <<<true>>> or
- <<<false>>> to make a streaming task that exits with a non-zero status to be
- <<<Failure>>> or <<<Success>>> respectively. By default, streaming tasks
- exiting with non-zero status are considered to be failed tasks.
-
-** Packaging Files With Job Submissions
-
- You can specify any executable as the mapper and/or the reducer. The
- executables do not need to pre-exist on the machines in the cluster; however,
- if they don't, you will need to use "-file" option to tell the framework to
- pack your executable files as a part of job submission. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -input myInputDirs \
- -output myOutputDir \
- -mapper myPythonScript.py \
- -reducer /usr/bin/wc \
- -file myPythonScript.py
-+---+
-
- The above example specifies a user defined Python executable as the mapper.
- The option "-file myPythonScript.py" causes the python executable shipped
- to the cluster machines as a part of job submission.
-
- In addition to executable files, you can also package other auxiliary files
- (such as dictionaries, configuration files, etc) that may be used by the
- mapper and/or the reducer. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -input myInputDirs \
- -output myOutputDir \
- -mapper myPythonScript.py \
- -reducer /usr/bin/wc \
- -file myPythonScript.py \
- -file myDictionary.txt
-+---+
-
-** Specifying Other Plugins for Jobs
-
- Just as with a normal Map/Reduce job, you can specify other plugins for a
- streaming job:
-
-+---+
- -inputformat JavaClassName
- -outputformat JavaClassName
- -partitioner JavaClassName
- -combiner streamingCommand or JavaClassName
-+---+
-
- The class you supply for the input format should return key/value pairs of
- Text class. If you do not specify an input format class, the TextInputFormat
- is used as the default. Since the TextInputFormat returns keys of
- LongWritable class, which are actually not part of the input data, the keys
- will be discarded; only the values will be piped to the streaming mapper.
-
- The class you supply for the output format is expected to take key/value
- pairs of Text class. If you do not specify an output format class, the
- TextOutputFormat is used as the default.
-
-** Setting Environment Variables
-
- To set an environment variable in a streaming command use:
-
-+---+
- -cmdenv EXAMPLE_DIR=/home/example/dictionaries/
-+---+
-
-* Generic Command Options
-
- Streaming supports {{{Streaming_Command_Options}streaming command options}}
- as well as generic command options. The general command line syntax is shown
- below.
-
- <<Note:>> Be sure to place the generic options before the streaming options,
- otherwise the command will fail. For an example, see
- {{{Making_Archives_Available_to_Tasks}Making Archives Available to Tasks}}.
-
-+---+
-hadoop command [genericOptions] [streamingOptions]
-+---+
-
- The Hadoop generic command options you can use with streaming are listed
- here:
-
-*-------------*--------------------*------------------------------------------*
-|| Parameter || Optional/Required || Description |
-*-------------+--------------------+------------------------------------------+
-| -conf configuration_file | Optional | Specify an application configuration
-| | | file
-*-------------+--------------------+------------------------------------------+
-| -D property=value | Optional | Use value for given property
-*-------------+--------------------+------------------------------------------+
-| -fs host:port or local | Optional | Specify a namenode
-*-------------+--------------------+------------------------------------------+
-| -files | Optional | Specify comma-separated files to be copied to the
-| | | Map/Reduce cluster
-*-------------+--------------------+------------------------------------------+
-| -libjars | Optional | Specify comma-separated jar files to include in the
-| | | classpath
-*-------------+--------------------+------------------------------------------+
-| -archives | Optional | Specify comma-separated archives to be unarchived on
-| | | the compute machines
-*-------------+--------------------+------------------------------------------+
-
-** Specifying Configuration Variables with the -D Option
-
- You can specify additional configuration variables by using
- "-D \<property\>=\<value\>".
-
-*** Specifying Directories
-
- To change the local temp directory use:
-
-+---+
- -D dfs.data.dir=/tmp
-+---+
-
- To specify additional local temp directories use:
-
-+---+
- -D mapred.local.dir=/tmp/local
- -D mapred.system.dir=/tmp/system
- -D mapred.temp.dir=/tmp/temp
-+---+
-
- <<Note:>> For more details on job configuration parameters see:
- {{{./mapred-default.xml}mapred-default.xml}}
-
-*** Specifying Map-Only Jobs
-
- Often, you may want to process input data using a map function only. To do
- this, simply set <<<mapreduce.job.reduces>>> to zero. The Map/Reduce
- framework will not create any reducer tasks. Rather, the outputs of the
- mapper tasks will be the final output of the job.
-
-+---+
- -D mapreduce.job.reduces=0
-+---+
-
- To be backward compatible, Hadoop Streaming also supports the "-reducer NONE"
- option, which is equivalent to "-D mapreduce.job.reduces=0".
-
-*** Specifying the Number of Reducers
-
- To specify the number of reducers, for example two, use:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -D mapreduce.job.reduces=2 \
- -input myInputDirs \
- -output myOutputDir \
- -mapper /bin/cat \
- -reducer /usr/bin/wc
-+---+
-
-*** Customizing How Lines are Split into Key/Value Pairs
-
- As noted earlier, when the Map/Reduce framework reads a line from the stdout
- of the mapper, it splits the line into a key/value pair. By default, the
- prefix of the line up to the first tab character is the key and the rest of
- the line (excluding the tab character) is the value.
-
- However, you can customize this default. You can specify a field separator
- other than the tab character (the default), and you can specify the nth
- (n >= 1) character rather than the first character in a line (the default) as
- the separator between the key and value. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -D stream.map.output.field.separator=. \
- -D stream.num.map.output.key.fields=4 \
- -input myInputDirs \
- -output myOutputDir \
- -mapper /bin/cat \
- -reducer /bin/cat
-+---+
-
- In the above example, "-D stream.map.output.field.separator=." specifies "."
- as the field separator for the map outputs, and the prefix up to the fourth
- "." in a line will be the key and the rest of the line (excluding the fourth
- ".") will be the value. If a line has less than four "."s, then the whole
- line will be the key and the value will be an empty Text object (like the one
- created by new Text("")).
-
- Similarly, you can use "-D stream.reduce.output.field.separator=SEP" and
- "-D stream.num.reduce.output.fields=NUM" to specify the nth field separator
- in a line of the reduce outputs as the separator between the key and the
- value.
-
- Similarly, you can specify "stream.map.input.field.separator" and
- "stream.reduce.input.field.separator" as the input separator for Map/Reduce
- inputs. By default the separator is the tab character.
-
-** Working with Large Files and Archives
-
- The -files and -archives options allow you to make files and archives
- available to the tasks. The argument is a URI to the file or archive that you
- have already uploaded to HDFS. These files and archives are cached across
- jobs. You can retrieve the host and fs_port values from the fs.default.name
- config variable.
-
- <<Note:>> The -files and -archives options are generic options. Be sure to
- place the generic options before the command options, otherwise the command
- will fail.
-
-*** Making Files Available to Tasks
-
- The -files option creates a symlink in the current working directory of the
- tasks that points to the local copy of the file.
-
- In this example, Hadoop automatically creates a symlink named testfile.txt in
- the current working directory of the tasks. This symlink points to the local
- copy of testfile.txt.
-
-+---+
--files hdfs://host:fs_port/user/testfile.txt
-+---+
-
- User can specify a different symlink name for -files using #.
-
-+---+
--files hdfs://host:fs_port/user/testfile.txt#testfile
-+---+
-
- Multiple entries can be specified like this:
-
-+---+
--files hdfs://host:fs_port/user/testfile1.txt,hdfs://host:fs_port/user/testfile2.txt
-+---+
-
-*** Making Archives Available to Tasks
-
- The -archives option allows you to copy jars locally to the current working
- directory of tasks and automatically unjar the files.
-
- In this example, Hadoop automatically creates a symlink named testfile.jar in
- the current working directory of tasks. This symlink points to the directory
- that stores the unjarred contents of the uploaded jar file.
-
-+---+
--archives hdfs://host:fs_port/user/testfile.jar
-+---+
-
- User can specify a different symlink name for -archives using #.
-
-+---+
--archives hdfs://host:fs_port/user/testfile.tgz#tgzdir
-+---+
-
- In this example, the input.txt file has two lines specifying the names of the
- two files: cachedir.jar/cache.txt and cachedir.jar/cache2.txt. "cachedir.jar"
- is a symlink to the archived directory, which has the files "cache.txt" and
- "cache2.txt".
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -archives 'hdfs://hadoop-nn1.example.com/user/me/samples/cachefile/cachedir.jar' \
- -D mapreduce.job.maps=1 \
- -D mapreduce.job.reduces=1 \
- -D mapreduce.job.name="Experiment" \
- -input "/user/me/samples/cachefile/input.txt" \
- -output "/user/me/samples/cachefile/out" \
- -mapper "xargs cat" \
- -reducer "cat"
-
-$ ls test_jar/
-cache.txt cache2.txt
-
-$ jar cvf cachedir.jar -C test_jar/ .
-added manifest
-adding: cache.txt(in = 30) (out= 29)(deflated 3%)
-adding: cache2.txt(in = 37) (out= 35)(deflated 5%)
-
-$ hdfs dfs -put cachedir.jar samples/cachefile
-
-$ hdfs dfs -cat /user/me/samples/cachefile/input.txt
-cachedir.jar/cache.txt
-cachedir.jar/cache2.txt
-
-$ cat test_jar/cache.txt
-This is just the cache string
-
-$ cat test_jar/cache2.txt
-This is just the second cache string
-
-$ hdfs dfs -ls /user/me/samples/cachefile/out
-Found 2 items
--rw-r--r-- 1 me supergroup 0 2013-11-14 17:00 /user/me/samples/cachefile/out/_SUCCESS
--rw-r--r-- 1 me supergroup 69 2013-11-14 17:00 /user/me/samples/cachefile/out/part-00000
-
-$ hdfs dfs -cat /user/me/samples/cachefile/out/part-00000
-This is just the cache string
-This is just the second cache string
-+---+
-
-* More Usage Examples
-
-** Hadoop Partitioner Class
-
- Hadoop has a library class,
- {{{../../api/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.html}
- KeyFieldBasedPartitioner}}, that is useful for many applications. This class
- allows the Map/Reduce framework to partition the map outputs based on certain
- key fields, not the whole keys. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -D stream.map.output.field.separator=. \
- -D stream.num.map.output.key.fields=4 \
- -D map.output.key.field.separator=. \
- -D mapreduce.partition.keypartitioner.options=-k1,2 \
- -D mapreduce.job.reduces=12 \
- -input myInputDirs \
- -output myOutputDir \
- -mapper /bin/cat \
- -reducer /bin/cat \
- -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner
-+---+
-
- Here, <-D stream.map.output.field.separator=.> and
- <-D stream.num.map.output.key.fields=4> are as explained in previous example.
- The two variables are used by streaming to identify the key/value pair of
- mapper.
-
- The map output keys of the above Map/Reduce job normally have four fields
- separated by ".". However, the Map/Reduce framework will partition the map
- outputs by the first two fields of the keys using the
- <-D mapred.text.key.partitioner.options=-k1,2> option. Here,
- <-D map.output.key.field.separator=.> specifies the separator for the
- partition. This guarantees that all the key/value pairs with the same first
- two fields in the keys will be partitioned into the same reducer.
-
- <This is effectively equivalent to specifying the first two fields as the
- primary key and the next two fields as the secondary. The primary key is used
- for partitioning, and the combination of the primary and secondary keys is
- used for sorting.> A simple illustration is shown here:
-
- Output of map (the keys)
-
-+---+
-11.12.1.2
-11.14.2.3
-11.11.4.1
-11.12.1.1
-11.14.2.2
-+---+
-
- Partition into 3 reducers (the first 2 fields are used as keys for partition)
-
-+---+
-11.11.4.1
------------
-11.12.1.2
-11.12.1.1
------------
-11.14.2.3
-11.14.2.2
-+---+
-
- Sorting within each partition for the reducer(all 4 fields used for sorting)
-
-+---+
-11.11.4.1
------------
-11.12.1.1
-11.12.1.2
------------
-11.14.2.2
-11.14.2.3
-+---+
-
-** Hadoop Comparator Class
-
- Hadoop has a library class,
- {{{../../api/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.html}
- KeyFieldBasedComparator}}, that is useful for many applications. This class
- provides a subset of features provided by the Unix/GNU Sort. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -D mapreduce.job.output.key.comparator.class=org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator \
- -D stream.map.output.field.separator=. \
- -D stream.num.map.output.key.fields=4 \
- -D mapreduce.map.output.key.field.separator=. \
- -D mapreduce.partition.keycomparator.options=-k2,2nr \
- -D mapreduce.job.reduces=1 \
- -input myInputDirs \
- -output myOutputDir \
- -mapper /bin/cat \
- -reducer /bin/cat
-+---+
-
- The map output keys of the above Map/Reduce job normally have four fields
- separated by ".". However, the Map/Reduce framework will sort the outputs by
- the second field of the keys using the
- <-D mapreduce.partition.keycomparator.options=-k2,2nr> option. Here, <-n>
- specifies that the sorting is numerical sorting and <-r> specifies that the
- result should be reversed. A simple illustration is shown below:
-
- Output of map (the keys)
-
-+---+
-11.12.1.2
-11.14.2.3
-11.11.4.1
-11.12.1.1
-11.14.2.2
-+---+
-
- Sorting output for the reducer (where second field used for sorting)
-
-+---+
-11.14.2.3
-11.14.2.2
-11.12.1.2
-11.12.1.1
-11.11.4.1
-+---+
-
-** Hadoop Aggregate Package
-
- Hadoop has a library package called
- {{{../../org/apache/hadoop/mapred/lib/aggregate/package-summary.html}
- Aggregate}}. Aggregate provides a special reducer class and a special
- combiner class, and a list of simple aggregators that perform aggregations
- such as "sum", "max", "min" and so on over a sequence of values. Aggregate
- allows you to define a mapper plugin class that is expected to generate
- "aggregatable items" for each input key/value pair of the mappers. The
- combiner/reducer will aggregate those aggregatable items by invoking the
- appropriate aggregators.
-
- To use Aggregate, simply specify "-reducer aggregate":
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -input myInputDirs \
- -output myOutputDir \
- -mapper myAggregatorForKeyCount.py \
- -reducer aggregate \
- -file myAggregatorForKeyCount.py \
-+---+
-
- The python program myAggregatorForKeyCount.py looks like:
-
-+---+
-#!/usr/bin/python
-
-import sys;
-
-def generateLongCountToken(id):
- return "LongValueSum:" + id + "\t" + "1"
-
-def main(argv):
- line = sys.stdin.readline();
- try:
- while line:
- line = line[:-1];
- fields = line.split("\t");
- print generateLongCountToken(fields[0]);
- line = sys.stdin.readline();
- except "end of file":
- return None
-if __name__ == "__main__":
- main(sys.argv)
-+---+
-
-** Hadoop Field Selection Class
-
- Hadoop has a library class,
- {{{../../api/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.html}
- FieldSelectionMapReduce}}, that effectively allows you to process text data
- like the unix "cut" utility. The map function defined in the class treats
- each input key/value pair as a list of fields. You can specify the field
- separator (the default is the tab character). You can select an arbitrary
- list of fields as the map output key, and an arbitrary list of fields as the
- map output value. Similarly, the reduce function defined in the class treats
- each input key/value pair as a list of fields. You can select an arbitrary
- list of fields as the reduce output key, and an arbitrary list of fields as
- the reduce output value. For example:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -D mapreduce.map.output.key.field.separator=. \
- -D mapreduce.partition.keypartitioner.options=-k1,2 \
- -D mapreduce.fieldsel.data.field.separator=. \
- -D mapreduce.fieldsel.map.output.key.value.fields.spec=6,5,1-3:0- \
- -D mapreduce.fieldsel.reduce.output.key.value.fields.spec=0-2:5- \
- -D mapreduce.map.output.key.class=org.apache.hadoop.io.Text \
- -D mapreduce.job.reduces=12 \
- -input myInputDirs \
- -output myOutputDir \
- -mapper org.apache.hadoop.mapred.lib.FieldSelectionMapReduce \
- -reducer org.apache.hadoop.mapred.lib.FieldSelectionMapReduce \
- -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner
-+---+
-
- The option "-D
- mapreduce.fieldsel.map.output.key.value.fields.spec=6,5,1-3:0-" specifies
- key/value selection for the map outputs. Key selection spec and value
- selection spec are separated by ":". In this case, the map output key will
- consist of fields 6, 5, 1, 2, and 3. The map output value will consist of all
- fields (0- means field 0 and all the subsequent fields).
-
- The option "-D mapreduce.fieldsel.reduce.output.key.value.fields.spec=0-2:5-"
- specifies key/value selection for the reduce outputs. In this case, the
- reduce output key will consist of fields 0, 1, 2 (corresponding to the
- original fields 6, 5, 1). The reduce output value will consist of all fields
- starting from field 5 (corresponding to all the original fields).
-
-* Frequently Asked Questions
-
-** How do I use Hadoop Streaming to run an arbitrary set of (semi) independent
- tasks?
-
- Often you do not need the full power of Map Reduce, but only need to run
- multiple instances of the same program - either on different parts of the
- data, or on the same data, but with different parameters. You can use Hadoop
- Streaming to do this.
-
-** How do I process files, one per map?
-
- As an example, consider the problem of zipping (compressing) a set of files
- across the hadoop cluster. You can achieve this by using Hadoop Streaming
- and custom mapper script:
-
- * Generate a file containing the full HDFS path of the input files. Each map
- task would get one file name as input.
-
- * Create a mapper script which, given a filename, will get the file to local
- disk, gzip the file and put it back in the desired output directory.
-
-** How many reducers should I use?
-
- See MapReduce Tutorial for details: {{{./MapReduceTutorial.html#Reducer}
- Reducer}}
-
-** If I set up an alias in my shell script, will that work after -mapper?
-
- For example, say I do: alias c1='cut -f1'. Will -mapper "c1" work?
-
- Using an alias will not work, but variable substitution is allowed as shown
- in this example:
-
-+---+
-$ hdfs dfs -cat /user/me/samples/student_marks
-alice 50
-bruce 70
-charlie 80
-dan 75
-
-$ c2='cut -f2'; hadoop jar hadoop-streaming-${project.version}.jar \
- -D mapreduce.job.name='Experiment' \
- -input /user/me/samples/student_marks \
- -output /user/me/samples/student_out \
- -mapper "$c2" -reducer 'cat'
-
-$ hdfs dfs -cat /user/me/samples/student_out/part-00000
-50
-70
-75
-80
-+---+
-
-** Can I use UNIX pipes?
-
- For example, will -mapper "cut -f1 | sed s/foo/bar/g" work?
-
- Currently this does not work and gives an "java.io.IOException: Broken pipe"
- error. This is probably a bug that needs to be investigated.
-
-** What do I do if I get the "No space left on device" error?
-
- For example, when I run a streaming job by distributing large executables
- (for example, 3.6G) through the -file option, I get a "No space left on
- device" error.
-
- The jar packaging happens in a directory pointed to by the configuration
- variable stream.tmpdir. The default value of stream.tmpdir is /tmp. Set the
- value to a directory with more space:
-
-+---+
--D stream.tmpdir=/export/bigspace/...
-+---+
-
-** How do I specify multiple input directories?
-
- You can specify multiple input directories with multiple '-input' options:
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -input '/user/foo/dir1' -input '/user/foo/dir2' \
- (rest of the command)
-+---+
-
-** How do I generate output files with gzip format?
-
- Instead of plain text files, you can generate gzip files as your generated
- output. Pass '-D mapreduce.output.fileoutputformat.compress=true -D
- mapreduce.output.fileoutputformat.compress.codec=org.apache.hadoop.io.compress.GzipCodec'
- as option to your streaming job.
-
-** How do I provide my own input/output format with streaming?
-
- You can specify your own custom class by packing them and putting the custom
- jar to \$\{HADOOP_CLASSPATH\}.
-
-** How do I parse XML documents using streaming?
-
- You can use the record reader StreamXmlRecordReader to process XML documents.
-
-+---+
-hadoop jar hadoop-streaming-${project.version}.jar \
- -inputreader "StreamXmlRecord,begin=BEGIN_STRING,end=END_STRING" \
- (rest of the command)
-+---+
-
- Anything found between BEGIN_STRING and END_STRING would be treated as one
- record for map tasks.
-
-** How do I update counters in streaming applications?
-
- A streaming process can use the stderr to emit counter information.
- <<<reporter:counter:\<group\>,\<counter\>,\<amount\>>>> should be sent to
- stderr to update the counter.
-
-** How do I update status in streaming applications?
-
- A streaming process can use the stderr to emit status information. To set a
- status, <<<reporter:status:\<message\>>>> should be sent to stderr.
-
-** How do I get the Job variables in a streaming job's mapper/reducer?
-
- See {{{./MapReduceTutorial.html#Configured_Parameters}
- Configured Parameters}}. During the execution of a streaming job, the names
- of the "mapred" parameters are transformed. The dots ( . ) become underscores
- ( _ ). For example, mapreduce.job.id becomes mapreduce_job_id and
- mapreduce.job.jar becomes mapreduce_job_jar. In your code, use the parameter
- names with the underscores.
[37/50] [abbrv] hadoop git commit: HADOOP-11295. RPC Server Reader
thread can't shutdown if RPCCallQueue is full. Contributed by Ming Ma.
Posted by zj...@apache.org.
HADOOP-11295. RPC Server Reader thread can't shutdown if RPCCallQueue is full. Contributed by Ming Ma.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/685af8a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/685af8a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/685af8a3
Branch: refs/heads/YARN-2928
Commit: 685af8a3d0504724fe588daf3722519fedc45b01
Parents: 6dc8812
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Feb 17 17:14:58 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Feb 17 17:14:58 2015 -0600
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../main/java/org/apache/hadoop/ipc/Server.java | 1 +
.../java/org/apache/hadoop/ipc/TestRPC.java | 68 ++++++++++++++++++++
3 files changed, 72 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/685af8a3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c1caf5f..d8a85f7 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -953,6 +953,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11570. S3AInputStream.close() downloads the remaining bytes of
the object from S3. (Dan Hecht via stevel).
+ HADOOP-11295. RPC Server Reader thread can't shutdown if RPCCallQueue is
+ full. (Ming Ma via kihwal)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/685af8a3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 475fb11..893e0eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -666,6 +666,7 @@ public abstract class Server {
assert !running;
readSelector.wakeup();
try {
+ super.interrupt();
super.join();
} catch (InterruptedException ie) {
Thread.currentThread().interrupt();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/685af8a3/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
index c1b1bfb..2db8522 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPC.java
@@ -38,9 +38,16 @@ import java.lang.reflect.Proxy;
import java.net.ConnectException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
+import java.util.ArrayList;
import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.Callable;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicReference;
@@ -1013,6 +1020,67 @@ public class TestRPC {
}
}
+ /**
+ * Verify the RPC server can shutdown properly when callQueue is full.
+ */
+ @Test (timeout=30000)
+ public void testRPCServerShutdown() throws Exception {
+ final int numClients = 3;
+ final List<Future<Void>> res = new ArrayList<Future<Void>>();
+ final ExecutorService executorService =
+ Executors.newFixedThreadPool(numClients);
+ final Configuration conf = new Configuration();
+ conf.setInt(CommonConfigurationKeys.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY, 0);
+ final Server server = new RPC.Builder(conf)
+ .setProtocol(TestProtocol.class).setInstance(new TestImpl())
+ .setBindAddress(ADDRESS).setPort(0)
+ .setQueueSizePerHandler(1).setNumHandlers(1).setVerbose(true)
+ .build();
+ server.start();
+
+ final TestProtocol proxy =
+ RPC.getProxy(TestProtocol.class, TestProtocol.versionID,
+ NetUtils.getConnectAddress(server), conf);
+ try {
+ // start a sleep RPC call to consume the only handler thread.
+ // Start another sleep RPC call to make callQueue full.
+ // Start another sleep RPC call to make reader thread block on CallQueue.
+ for (int i = 0; i < numClients; i++) {
+ res.add(executorService.submit(
+ new Callable<Void>() {
+ @Override
+ public Void call() throws IOException, InterruptedException {
+ proxy.sleep(100000);
+ return null;
+ }
+ }));
+ }
+ while (server.getCallQueueLen() != 1
+ && countThreads(CallQueueManager.class.getName()) != 1
+ && countThreads(TestProtocol.class.getName()) != 1) {
+ Thread.sleep(100);
+ }
+ } finally {
+ try {
+ server.stop();
+ assertEquals("Not enough clients", numClients, res.size());
+ for (Future<Void> f : res) {
+ try {
+ f.get();
+ fail("Future get should not return");
+ } catch (ExecutionException e) {
+ assertTrue("Unexpected exception: " + e,
+ e.getCause() instanceof IOException);
+ LOG.info("Expected exception", e.getCause());
+ }
+ }
+ } finally {
+ RPC.stopProxy(proxy);
+ executorService.shutdown();
+ }
+ }
+ }
+
public static void main(String[] args) throws IOException {
new TestRPC().testCallsInternal(conf);
[28/50] [abbrv] hadoop git commit: HADOOP-11575. Daemon log
documentation is misleading (Naganarasimha G R via aw)
Posted by zj...@apache.org.
HADOOP-11575. Daemon log documentation is misleading (Naganarasimha G R via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72389c78
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72389c78
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72389c78
Branch: refs/heads/YARN-2928
Commit: 72389c78dba1e2f7727245838180fb51e5241075
Parents: 8b787e2
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 17 07:00:00 2015 -1000
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 17 07:00:00 2015 -1000
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/log/LogLevel.java | 10 +++++-----
.../src/site/markdown/CommandsManual.md | 10 ++++++----
.../src/site/apt/YarnCommands.apt.vm | 21 +++++++++++---------
4 files changed, 26 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72389c78/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0d1ef36..1256ae5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -175,6 +175,9 @@ Trunk (Unreleased)
HADOOP-11495. Convert site documentation from apt to markdown
(Masatake Iwasaki via aw)
+ HADOOP-11575. Daemon log documentation is misleading
+ (Naganarasimha G R via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72389c78/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
index 4749ce1..baf71b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/log/LogLevel.java
@@ -38,8 +38,8 @@ import org.apache.hadoop.util.ServletUtil;
@InterfaceStability.Evolving
public class LogLevel {
public static final String USAGES = "\nUsage: General options are:\n"
- + "\t[-getlevel <host:httpPort> <name>]\n"
- + "\t[-setlevel <host:httpPort> <name> <level>]\n";
+ + "\t[-getlevel <host:httpPort> <classname>]\n"
+ + "\t[-setlevel <host:httpPort> <classname> <level>]\n";
/**
* A command line implementation
@@ -106,7 +106,7 @@ public class LogLevel {
if (logName != null) {
out.println("<br /><hr /><h3>Results</h3>");
out.println(MARKER
- + "Submitted Log Name: <b>" + logName + "</b><br />");
+ + "Submitted Class Name: <b>" + logName + "</b><br />");
Log log = LogFactory.getLog(logName);
out.println(MARKER
@@ -131,10 +131,10 @@ public class LogLevel {
}
static final String FORMS = "\n<br /><hr /><h3>Get / Set</h3>"
- + "\n<form>Log: <input type='text' size='50' name='log' /> "
+ + "\n<form>Class Name: <input type='text' size='50' name='log' /> "
+ "<input type='submit' value='Get Log Level' />"
+ "</form>"
- + "\n<form>Log: <input type='text' size='50' name='log' /> "
+ + "\n<form>Class Name: <input type='text' size='50' name='log' /> "
+ "Level: <input type='text' name='level' /> "
+ "<input type='submit' value='Set Log Level' />"
+ "</form>";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72389c78/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index f0c0367..62235c5 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -212,14 +212,16 @@ Commands useful for administrators of a hadoop cluster.
### `daemonlog`
-Usage: `hadoop daemonlog -getlevel <host:port> <name> ` Usage: `hadoop daemonlog -setlevel <host:port> <name> <level> `
+Usage: `hadoop daemonlog -getlevel <host:httpport> <classname> ` Usage: `hadoop daemonlog -setlevel <host:httpport> <classname> <level> `
| COMMAND\_OPTION | Description |
|:---- |:---- |
-| `-getlevel` *host:port* *name* | Prints the log level of the daemon running at *host:port*. This command internally connects to http://host:port/logLevel?log=name |
-| `-setlevel` *host:port* *name* *level* | Sets the log level of the daemon running at *host:port*. This command internally connects to http://host:port/logLevel?log=name |
+| `-getlevel` *host:httpport* *classname* | Prints the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>` |
+| `-setlevel` *host:httpport* *classname* *level* | Sets the log level of the log identified by a qualified *classname*, in the daemon running at *host:httpport*. This command internally connects to `http://<host:httpport>/logLevel?log=<classname>&level=<level>` |
-Get/Set the log level for each daemon.
+Get/Set the log level for a Log identified by a qualified class name in the daemon.
+
+ Example: $ bin/hadoop daemonlog -setlevel 127.0.0.1:50070 org.apache.hadoop.hdfs.server.namenode.NameNode DEBUG
Files
-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/72389c78/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
index 6333d70..67f8a58 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
@@ -194,24 +194,27 @@ YARN Commands
Usage:
---------------------------------
- yarn daemonlog -getlevel <host:port> <name>
- yarn daemonlog -setlevel <host:port> <name> <level>
+ yarn daemonlog -getlevel <host:httpport> <classname>
+ yarn daemonlog -setlevel <host:httpport> <classname> <level>
---------------------------------
*---------------+--------------+
|| COMMAND_OPTIONS || Description |
*---------------+--------------+
-| -getlevel \<host:port\> \<name\> | Prints the log level of the daemon running
-| | at \<host:port\>. This command internally connects to
-| | http://\<host:port\>/logLevel?log=\<name\>
+| -getlevel \<host:httpport\> \<classname\> | Prints the log level of the log identified
+| | by a qualified \<classname\>, in the daemon running at \<host:httpport\>. This
+| | command internally connects to http://\<host:httpport\>/logLevel?log=\<classname\>
*---------------+--------------+
-| -setlevel \<host:port\> \<name\> \<level\> | Sets the log level of the daemon
-| | running at \<host:port\>. This command internally connects to
-| | http://\<host:port\>/logLevel?log=\<name\>
+| -setlevel \<host:httpport\> \<classname\> \<level\> | Sets the log level of the log
+| | identified by a qualified \<classname\> in the daemon running at \<host:httpport\>.
+| | This command internally connects to http://\<host:httpport\>/logLevel?log=\<classname\>&level=\<level\>
*---------------+--------------+
- Get/Set the log level for each daemon.
+ Get/Set the log level for a Log identified by a qualified class name in the daemon.
+----
+ Example: $ bin/yarn daemonlog -setlevel 127.0.0.1:8088 org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppImpl DEBUG
+----
** <<<nodemanager>>>
[11/50] [abbrv] hadoop git commit: HDFS-7604. Track and display
failed DataNode storage locations in NameNode. Contributed by Chris Nauroth.
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
index d07bb45..a8f7990 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestDataNodeVolumeFailureReporting.java
@@ -19,7 +19,9 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assume.assumeTrue;
@@ -30,6 +32,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.ReconfigurationException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -39,6 +42,10 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
+import org.apache.hadoop.io.IOUtils;
import org.apache.log4j.Level;
import org.junit.After;
import org.junit.Before;
@@ -58,6 +65,7 @@ public class TestDataNodeVolumeFailureReporting {
private MiniDFSCluster cluster;
private Configuration conf;
private String dataDir;
+ private long volumeCapacity;
// Sleep at least 3 seconds (a 1s heartbeat plus padding) to allow
// for heartbeats to propagate from the datanodes to the namenode.
@@ -69,29 +77,29 @@ public class TestDataNodeVolumeFailureReporting {
@Before
public void setUp() throws Exception {
- conf = new HdfsConfiguration();
- conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512L);
- /*
- * Lower the DN heartbeat, DF rate, and recheck interval to one second
- * so state about failures and datanode death propagates faster.
- */
- conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
- conf.setInt(DFSConfigKeys.DFS_DF_INTERVAL_KEY, 1000);
- conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
+ // These tests simulate volume failures by denying execute permission on the
+ // volume's path. On Windows, the owner of an object is always allowed
+ // access, so we can't run these tests on Windows.
+ assumeTrue(!Path.WINDOWS);
// Allow a single volume failure (there are two volumes)
- conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY, 1);
- cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
- cluster.waitActive();
- fs = cluster.getFileSystem();
- dataDir = cluster.getDataDirectory();
+ initCluster(1, 2, 1);
}
@After
public void tearDown() throws Exception {
- for (int i = 0; i < 3; i++) {
- FileUtil.setExecutable(new File(dataDir, "data"+(2*i+1)), true);
- FileUtil.setExecutable(new File(dataDir, "data"+(2*i+2)), true);
+ // Restore executable permission on all directories where a failure may have
+ // been simulated by denying execute access. This is based on the maximum
+ // number of datanodes and the maximum number of storages per data node used
+ // throughout the tests in this suite.
+ int maxDataNodes = 3;
+ int maxStoragesPerDataNode = 4;
+ for (int i = 0; i < maxDataNodes; i++) {
+ for (int j = 1; j <= maxStoragesPerDataNode; j++) {
+ String subDir = "data" + ((i * maxStoragesPerDataNode) + j);
+ FileUtil.setExecutable(new File(dataDir, subDir), true);
+ }
}
+ IOUtils.cleanup(LOG, fs);
cluster.shutdown();
}
@@ -102,8 +110,6 @@ public class TestDataNodeVolumeFailureReporting {
*/
@Test
public void testSuccessiveVolumeFailures() throws Exception {
- assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
-
// Bring up two more datanodes
cluster.startDataNodes(conf, 2, true, null, null);
cluster.waitActive();
@@ -151,12 +157,9 @@ public class TestDataNodeVolumeFailureReporting {
/*
* The metrics should confirm the volume failures.
*/
- assertCounter("VolumeFailures", 1L,
- getMetrics(dns.get(0).getMetrics().name()));
- assertCounter("VolumeFailures", 1L,
- getMetrics(dns.get(1).getMetrics().name()));
- assertCounter("VolumeFailures", 0L,
- getMetrics(dns.get(2).getMetrics().name()));
+ checkFailuresAtDataNode(dns.get(0), 1, true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(1), 1, true, dn2Vol1.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(2), 0, true);
// Ensure we wait a sufficient amount of time
assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
@@ -164,6 +167,10 @@ public class TestDataNodeVolumeFailureReporting {
// Eventually the NN should report two volume failures
DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2,
origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(2), true);
/*
* Now fail a volume on the third datanode. We should be able to get
@@ -174,17 +181,10 @@ public class TestDataNodeVolumeFailureReporting {
DFSTestUtil.createFile(fs, file2, 1024, (short)3, 1L);
DFSTestUtil.waitReplication(fs, file2, (short)3);
assertTrue("DN3 should still be up", dns.get(2).isDatanodeUp());
- assertCounter("VolumeFailures", 1L,
- getMetrics(dns.get(2).getMetrics().name()));
-
- ArrayList<DatanodeDescriptor> live = new ArrayList<DatanodeDescriptor>();
- ArrayList<DatanodeDescriptor> dead = new ArrayList<DatanodeDescriptor>();
- dm.fetchDatanodes(live, dead, false);
- live.clear();
- dead.clear();
- dm.fetchDatanodes(live, dead, false);
- assertEquals("DN3 should have 1 failed volume",
- 1, live.get(2).getVolumeFailures());
+ checkFailuresAtDataNode(dns.get(2), 1, true, dn3Vol1.getAbsolutePath());
+
+ DataNodeTestUtils.triggerHeartbeat(dns.get(2));
+ checkFailuresAtNameNode(dm, dns.get(2), true, dn3Vol1.getAbsolutePath());
/*
* Once the datanodes have a chance to heartbeat their new capacity the
@@ -194,6 +194,10 @@ public class TestDataNodeVolumeFailureReporting {
dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0);
DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 3,
origCapacity - (3*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 3);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(2), true, dn3Vol1.getAbsolutePath());
/*
* Now fail the 2nd volume on the 3rd datanode. All its volumes
@@ -210,12 +214,15 @@ public class TestDataNodeVolumeFailureReporting {
DFSTestUtil.waitForDatanodeDeath(dns.get(2));
// And report two failed volumes
- assertCounter("VolumeFailures", 2L,
- getMetrics(dns.get(2).getMetrics().name()));
+ checkFailuresAtDataNode(dns.get(2), 2, true, dn3Vol1.getAbsolutePath(),
+ dn3Vol2.getAbsolutePath());
// The NN considers the DN dead
DFSTestUtil.waitForDatanodeStatus(dm, 2, 1, 2,
origCapacity - (4*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath());
/*
* The datanode never tries to restore the failed volume, even if
@@ -240,6 +247,11 @@ public class TestDataNodeVolumeFailureReporting {
*/
DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 0, origCapacity,
WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 0);
+ dns = cluster.getDataNodes();
+ checkFailuresAtNameNode(dm, dns.get(0), true);
+ checkFailuresAtNameNode(dm, dns.get(1), true);
+ checkFailuresAtNameNode(dm, dns.get(2), true);
}
/**
@@ -247,8 +259,6 @@ public class TestDataNodeVolumeFailureReporting {
*/
@Test
public void testVolFailureStatsPreservedOnNNRestart() throws Exception {
- assumeTrue(!System.getProperty("os.name").startsWith("Windows"));
-
// Bring up two more datanodes that can tolerate 1 failure
cluster.startDataNodes(conf, 2, true, null, null);
cluster.waitActive();
@@ -268,15 +278,346 @@ public class TestDataNodeVolumeFailureReporting {
Path file1 = new Path("/test1");
DFSTestUtil.createFile(fs, file1, 1024, (short)2, 1L);
DFSTestUtil.waitReplication(fs, file1, (short)2);
+ ArrayList<DataNode> dns = cluster.getDataNodes();
// The NN reports two volumes failures
DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2,
origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath());
// After restarting the NN it still see the two failures
cluster.restartNameNode(0);
cluster.waitActive();
DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2,
origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath());
+ }
+
+ @Test
+ public void testMultipleVolFailuresOnNode() throws Exception {
+ // Reinitialize the cluster, configured with 4 storage locations per DataNode
+ // and tolerating up to 2 failures.
+ tearDown();
+ initCluster(3, 4, 2);
+
+ // Calculate the total capacity of all the datanodes. Sleep for three seconds
+ // to be sure the datanodes have had a chance to heartbeat their capacities.
+ Thread.sleep(WAIT_FOR_HEARTBEATS);
+ DatanodeManager dm = cluster.getNamesystem().getBlockManager()
+ .getDatanodeManager();
+
+ long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm);
+ long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0);
+
+ File dn1Vol1 = new File(dataDir, "data"+(4*0+1));
+ File dn1Vol2 = new File(dataDir, "data"+(4*0+2));
+ File dn2Vol1 = new File(dataDir, "data"+(4*1+1));
+ File dn2Vol2 = new File(dataDir, "data"+(4*1+2));
+
+ // Make the first two volume directories on the first two datanodes
+ // non-accessible.
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1,
+ false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol2,
+ false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1,
+ false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol2,
+ false));
+
+ // Create file1 and wait for 3 replicas (ie all DNs can still store a block).
+ // Then assert that all DNs are up, despite the volume failures.
+ Path file1 = new Path("/test1");
+ DFSTestUtil.createFile(fs, file1, 1024, (short)3, 1L);
+ DFSTestUtil.waitReplication(fs, file1, (short)3);
+
+ ArrayList<DataNode> dns = cluster.getDataNodes();
+ assertTrue("DN1 should be up", dns.get(0).isDatanodeUp());
+ assertTrue("DN2 should be up", dns.get(1).isDatanodeUp());
+ assertTrue("DN3 should be up", dns.get(2).isDatanodeUp());
+
+ checkFailuresAtDataNode(dns.get(0), 1, true, dn1Vol1.getAbsolutePath(),
+ dn1Vol2.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(1), 1, true, dn2Vol1.getAbsolutePath(),
+ dn2Vol2.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(2), 0, true);
+
+ // Ensure we wait a sufficient amount of time
+ assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
+
+ // Eventually the NN should report four volume failures
+ DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 4,
+ origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 4);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath(),
+ dn1Vol2.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath(),
+ dn2Vol2.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(2), true);
+ }
+
+ @Test
+ public void testDataNodeReconfigureWithVolumeFailures() throws Exception {
+ // Bring up two more datanodes
+ cluster.startDataNodes(conf, 2, true, null, null);
+ cluster.waitActive();
+
+ final DatanodeManager dm = cluster.getNamesystem().getBlockManager(
+ ).getDatanodeManager();
+ long origCapacity = DFSTestUtil.getLiveDatanodeCapacity(dm);
+ long dnCapacity = DFSTestUtil.getDatanodeCapacity(dm, 0);
+
+ // Fail the first volume on both datanodes (we have to keep the
+ // third healthy so one node in the pipeline will not fail).
+ File dn1Vol1 = new File(dataDir, "data"+(2*0+1));
+ File dn1Vol2 = new File(dataDir, "data"+(2*0+2));
+ File dn2Vol1 = new File(dataDir, "data"+(2*1+1));
+ File dn2Vol2 = new File(dataDir, "data"+(2*1+2));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn1Vol1, false));
+ assertTrue("Couldn't chmod local vol", FileUtil.setExecutable(dn2Vol1, false));
+
+ Path file1 = new Path("/test1");
+ DFSTestUtil.createFile(fs, file1, 1024, (short)2, 1L);
+ DFSTestUtil.waitReplication(fs, file1, (short)2);
+
+ ArrayList<DataNode> dns = cluster.getDataNodes();
+ assertTrue("DN1 should be up", dns.get(0).isDatanodeUp());
+ assertTrue("DN2 should be up", dns.get(1).isDatanodeUp());
+ assertTrue("DN3 should be up", dns.get(2).isDatanodeUp());
+
+ checkFailuresAtDataNode(dns.get(0), 1, true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(1), 1, true, dn2Vol1.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(2), 0, true);
+
+ // Ensure we wait a sufficient amount of time
+ assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
+
+ // The NN reports two volume failures
+ DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2,
+ origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), true, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), true, dn2Vol1.getAbsolutePath());
+
+ // Reconfigure each DataNode to remove its failed volumes.
+ reconfigureDataNode(dns.get(0), dn1Vol2);
+ reconfigureDataNode(dns.get(1), dn2Vol2);
+
+ DataNodeTestUtils.triggerHeartbeat(dns.get(0));
+ DataNodeTestUtils.triggerHeartbeat(dns.get(1));
+
+ checkFailuresAtDataNode(dns.get(0), 1, true);
+ checkFailuresAtDataNode(dns.get(1), 1, true);
+
+ // NN sees reduced capacity, but no volume failures.
+ DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 0,
+ origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(true, 0);
+ checkFailuresAtNameNode(dm, dns.get(0), true);
+ checkFailuresAtNameNode(dm, dns.get(1), true);
+
+ // Reconfigure again to try to add back the failed volumes.
+ reconfigureDataNode(dns.get(0), dn1Vol1, dn1Vol2);
+ reconfigureDataNode(dns.get(1), dn2Vol1, dn2Vol2);
+
+ DataNodeTestUtils.triggerHeartbeat(dns.get(0));
+ DataNodeTestUtils.triggerHeartbeat(dns.get(1));
+
+ checkFailuresAtDataNode(dns.get(0), 1, false, dn1Vol1.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(1), 1, false, dn2Vol1.getAbsolutePath());
+
+ // Ensure we wait a sufficient amount of time.
+ assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
+
+ // The NN reports two volume failures again.
+ DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2,
+ origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(false, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), false, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), false, dn2Vol1.getAbsolutePath());
+
+ // Reconfigure a third time with the failed volumes. Afterwards, we expect
+ // the same volume failures to be reported. (No double-counting.)
+ reconfigureDataNode(dns.get(0), dn1Vol1, dn1Vol2);
+ reconfigureDataNode(dns.get(1), dn2Vol1, dn2Vol2);
+
+ DataNodeTestUtils.triggerHeartbeat(dns.get(0));
+ DataNodeTestUtils.triggerHeartbeat(dns.get(1));
+
+ checkFailuresAtDataNode(dns.get(0), 1, false, dn1Vol1.getAbsolutePath());
+ checkFailuresAtDataNode(dns.get(1), 1, false, dn2Vol1.getAbsolutePath());
+
+ // Ensure we wait a sufficient amount of time.
+ assert (WAIT_FOR_HEARTBEATS * 10) > WAIT_FOR_DEATH;
+
+ // The NN reports two volume failures again.
+ DFSTestUtil.waitForDatanodeStatus(dm, 3, 0, 2,
+ origCapacity - (1*dnCapacity), WAIT_FOR_HEARTBEATS);
+ checkAggregateFailuresAtNameNode(false, 2);
+ checkFailuresAtNameNode(dm, dns.get(0), false, dn1Vol1.getAbsolutePath());
+ checkFailuresAtNameNode(dm, dns.get(1), false, dn2Vol1.getAbsolutePath());
+ }
+
+ /**
+ * Checks the NameNode for correct values of aggregate counters tracking failed
+ * volumes across all DataNodes.
+ *
+ * @param expectCapacityKnown if true, then expect that the capacities of the
+ * volumes were known before the failures, and therefore the lost capacity
+ * can be reported
+ * @param expectedVolumeFailuresTotal expected number of failed volumes
+ */
+ private void checkAggregateFailuresAtNameNode(boolean expectCapacityKnown,
+ int expectedVolumeFailuresTotal) {
+ FSNamesystem ns = cluster.getNamesystem();
+ assertEquals(expectedVolumeFailuresTotal, ns.getVolumeFailuresTotal());
+ long expectedCapacityLost = getExpectedCapacityLost(expectCapacityKnown,
+ expectedVolumeFailuresTotal);
+ assertEquals(expectedCapacityLost, ns.getEstimatedCapacityLostTotal());
+ }
+
+ /**
+ * Checks a DataNode for correct reporting of failed volumes.
+ *
+ * @param dn DataNode to check
+ * @param expectedVolumeFailuresCounter metric counter value for
+ * VolumeFailures. The current implementation actually counts the number
+ * of failed disk checker cycles, which may be different from the length of
+ * expectedFailedVolumes if multiple disks fail in the same disk checker
+ * cycle
+ * @param expectCapacityKnown if true, then expect that the capacities of the
+ * volumes were known before the failures, and therefore the lost capacity
+ * can be reported
+ * @param expectedFailedVolumes expected locations of failed volumes
+ * @throws Exception if there is any failure
+ */
+ private void checkFailuresAtDataNode(DataNode dn,
+ long expectedVolumeFailuresCounter, boolean expectCapacityKnown,
+ String... expectedFailedVolumes) throws Exception {
+ assertCounter("VolumeFailures", expectedVolumeFailuresCounter,
+ getMetrics(dn.getMetrics().name()));
+ FsDatasetSpi<?> fsd = dn.getFSDataset();
+ assertEquals(expectedFailedVolumes.length, fsd.getNumFailedVolumes());
+ assertArrayEquals(expectedFailedVolumes, fsd.getFailedStorageLocations());
+ if (expectedFailedVolumes.length > 0) {
+ assertTrue(fsd.getLastVolumeFailureDate() > 0);
+ long expectedCapacityLost = getExpectedCapacityLost(expectCapacityKnown,
+ expectedFailedVolumes.length);
+ assertEquals(expectedCapacityLost, fsd.getEstimatedCapacityLostTotal());
+ } else {
+ assertEquals(0, fsd.getLastVolumeFailureDate());
+ assertEquals(0, fsd.getEstimatedCapacityLostTotal());
+ }
+ }
+
+ /**
+ * Checks NameNode tracking of a particular DataNode for correct reporting of
+ * failed volumes.
+ *
+ * @param dm DatanodeManager to check
+ * @param dn DataNode to check
+ * @param expectCapacityKnown if true, then expect that the capacities of the
+ * volumes were known before the failures, and therefore the lost capacity
+ * can be reported
+ * @param expectedFailedVolumes expected locations of failed volumes
+ * @throws Exception if there is any failure
+ */
+ private void checkFailuresAtNameNode(DatanodeManager dm, DataNode dn,
+ boolean expectCapacityKnown, String... expectedFailedVolumes)
+ throws Exception {
+ DatanodeDescriptor dd = cluster.getNamesystem().getBlockManager()
+ .getDatanodeManager().getDatanode(dn.getDatanodeId());
+ assertEquals(expectedFailedVolumes.length, dd.getVolumeFailures());
+ VolumeFailureSummary volumeFailureSummary = dd.getVolumeFailureSummary();
+ if (expectedFailedVolumes.length > 0) {
+ assertArrayEquals(expectedFailedVolumes, volumeFailureSummary
+ .getFailedStorageLocations());
+ assertTrue(volumeFailureSummary.getLastVolumeFailureDate() > 0);
+ long expectedCapacityLost = getExpectedCapacityLost(expectCapacityKnown,
+ expectedFailedVolumes.length);
+ assertEquals(expectedCapacityLost,
+ volumeFailureSummary.getEstimatedCapacityLostTotal());
+ } else {
+ assertNull(volumeFailureSummary);
+ }
+ }
+
+ /**
+ * Returns expected capacity lost for use in assertions. The return value is
+ * dependent on whether or not it is expected that the volume capacities were
+ * known prior to the failures.
+ *
+ * @param expectCapacityKnown if true, then expect that the capacities of the
+ * volumes were known before the failures, and therefore the lost capacity
+ * can be reported
+ * @param expectedVolumeFailuresTotal expected number of failed volumes
+ * @return estimated capacity lost in bytes
+ */
+ private long getExpectedCapacityLost(boolean expectCapacityKnown,
+ int expectedVolumeFailuresTotal) {
+ return expectCapacityKnown ? expectedVolumeFailuresTotal * volumeCapacity :
+ 0;
+ }
+
+ /**
+ * Initializes the cluster.
+ *
+ * @param numDataNodes number of datanodes
+ * @param storagesPerDatanode number of storage locations on each datanode
+ * @param failedVolumesTolerated number of acceptable volume failures
+ * @throws Exception if there is any failure
+ */
+ private void initCluster(int numDataNodes, int storagesPerDatanode,
+ int failedVolumesTolerated) throws Exception {
+ conf = new HdfsConfiguration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, 512L);
+ /*
+ * Lower the DN heartbeat, DF rate, and recheck interval to one second
+ * so state about failures and datanode death propagates faster.
+ */
+ conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
+ conf.setInt(DFSConfigKeys.DFS_DF_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 1000);
+ conf.setInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
+ failedVolumesTolerated);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(numDataNodes)
+ .storagesPerDatanode(storagesPerDatanode).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ dataDir = cluster.getDataDirectory();
+ long dnCapacity = DFSTestUtil.getDatanodeCapacity(
+ cluster.getNamesystem().getBlockManager().getDatanodeManager(), 0);
+ volumeCapacity = dnCapacity / cluster.getStoragesPerDatanode();
+ }
+
+ /**
+ * Reconfigure a DataNode by setting a new list of volumes.
+ *
+ * @param dn DataNode to reconfigure
+ * @param newVols new volumes to configure
+ * @throws Exception if there is any failure
+ */
+ private static void reconfigureDataNode(DataNode dn, File... newVols)
+ throws Exception {
+ StringBuilder dnNewDataDirs = new StringBuilder();
+ for (File newVol: newVols) {
+ if (dnNewDataDirs.length() > 0) {
+ dnNewDataDirs.append(',');
+ }
+ dnNewDataDirs.append(newVol.getAbsolutePath());
+ }
+ try {
+ dn.reconfigurePropertyImpl(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY,
+ dnNewDataDirs.toString());
+ } catch (ReconfigurationException e) {
+ // This can be thrown if reconfiguration tries to use a failed volume.
+ // We need to swallow the exception, because some of our tests want to
+ // cover this case.
+ LOG.warn("Could not reconfigure DataNode.", e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
index c049d81..7a09630 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestFsDatasetCache.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.HeartbeatResponse;
import org.apache.hadoop.hdfs.server.protocol.NNHAStatusHeartbeat;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.nativeio.NativeIO;
import org.apache.hadoop.io.nativeio.NativeIO.POSIX.CacheManipulator;
@@ -162,7 +163,7 @@ public class TestFsDatasetCache {
doReturn(response).when(spyNN).sendHeartbeat(
(DatanodeRegistration) any(),
(StorageReport[]) any(), anyLong(), anyLong(),
- anyInt(), anyInt(), anyInt());
+ anyInt(), anyInt(), anyInt(), (VolumeFailureSummary) any());
}
private static DatanodeCommand[] cacheBlock(HdfsBlockLocation loc) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
index b6b3fe6..ec39892 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestStorageReport.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -101,7 +102,8 @@ public class TestStorageReport {
Mockito.verify(nnSpy).sendHeartbeat(
any(DatanodeRegistration.class),
captor.capture(),
- anyLong(), anyLong(), anyInt(), anyInt(), anyInt());
+ anyLong(), anyLong(), anyInt(), anyInt(), anyInt(),
+ Mockito.any(VolumeFailureSummary.class));
StorageReport[] reports = captor.getValue();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index c377bdb..fc11886 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.ReplicaRecoveryInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.server.protocol.VolumeFailureSummary;
import org.apache.hadoop.util.DiskChecker;
import org.apache.hadoop.util.DiskChecker.DiskErrorException;
@@ -376,6 +377,26 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
}
@Override
+ public String[] getFailedStorageLocations() {
+ return null;
+ }
+
+ @Override
+ public long getLastVolumeFailureDate() {
+ return 0;
+ }
+
+ @Override
+ public long getEstimatedCapacityLostTotal() {
+ return 0;
+ }
+
+ @Override
+ public VolumeFailureSummary getVolumeFailureSummary() {
+ return null;
+ }
+
+ @Override
public long getCacheUsed() {
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 8f87f57..c3b871c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -47,6 +47,7 @@ import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
@@ -102,6 +103,7 @@ public class TestFsDatasetImpl {
String dataDir = StringUtils.join(",", dirStrings);
conf.set(DFSConfigKeys.DFS_DATANODE_DATA_DIR_KEY, dataDir);
+ when(storage.dirIterator()).thenReturn(dirs.iterator());
when(storage.getNumStorageDirs()).thenReturn(numDirs);
}
@@ -240,8 +242,8 @@ public class TestFsDatasetImpl {
RoundRobinVolumeChoosingPolicy<FsVolumeImpl> blockChooser =
new RoundRobinVolumeChoosingPolicy<>();
final BlockScanner blockScanner = new BlockScanner(datanode, conf);
- final FsVolumeList volumeList =
- new FsVolumeList(0, blockScanner, blockChooser);
+ final FsVolumeList volumeList = new FsVolumeList(
+ Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser);
final List<FsVolumeImpl> oldVolumes = new ArrayList<>();
// Initialize FsVolumeList with 5 mock volumes.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index d477e5b..f87c404 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -31,6 +31,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import static org.junit.Assert.assertNotEquals;
@@ -57,7 +58,8 @@ public class TestFsVolumeList {
@Test
public void testGetNextVolumeWithClosedVolume() throws IOException {
- FsVolumeList volumeList = new FsVolumeList(0, blockScanner, blockChooser);
+ FsVolumeList volumeList = new FsVolumeList(
+ Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser);
List<FsVolumeImpl> volumes = new ArrayList<>();
for (int i = 0; i < 3; i++) {
File curDir = new File(baseDir, "nextvolume-" + i);
@@ -82,7 +84,8 @@ public class TestFsVolumeList {
@Test
public void testCheckDirsWithClosedVolume() throws IOException {
- FsVolumeList volumeList = new FsVolumeList(0, blockScanner, blockChooser);
+ FsVolumeList volumeList = new FsVolumeList(
+ Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser);
List<FsVolumeImpl> volumes = new ArrayList<>();
for (int i = 0; i < 3; i++) {
File curDir = new File(baseDir, "volume-" + i);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index c8def37..c11abfc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -951,7 +951,7 @@ public class NNThroughputBenchmark implements Tool {
StorageReport[] rep = { new StorageReport(storage, false,
DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration, rep,
- 0L, 0L, 0, 0, 0).getCommands();
+ 0L, 0L, 0, 0, 0, null).getCommands();
if(cmds != null) {
for (DatanodeCommand cmd : cmds ) {
if(LOG.isDebugEnabled()) {
@@ -998,7 +998,7 @@ public class NNThroughputBenchmark implements Tool {
StorageReport[] rep = { new StorageReport(storage,
false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
DatanodeCommand[] cmds = nameNodeProto.sendHeartbeat(dnRegistration,
- rep, 0L, 0L, 0, 0, 0).getCommands();
+ rep, 0L, 0L, 0, 0, 0, null).getCommands();
if (cmds != null) {
for (DatanodeCommand cmd : cmds) {
if (cmd.getAction() == DatanodeProtocol.DNA_TRANSFER) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
index 7aad378..fa23fbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
@@ -117,7 +117,7 @@ public class NameNodeAdapter {
DatanodeDescriptor dd, FSNamesystem namesystem) throws IOException {
return namesystem.handleHeartbeat(nodeReg,
BlockManagerTestUtil.getStorageReportsForDatanode(dd),
- dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0);
+ dd.getCacheCapacity(), dd.getCacheRemaining(), 0, 0, 0, null);
}
public static boolean setReplication(final FSNamesystem ns,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9729b244/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 71bf124..fb1418a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -117,7 +117,7 @@ public class TestDeadDatanode {
StorageReport[] rep = { new StorageReport(
new DatanodeStorage(reg.getDatanodeUuid()),
false, 0, 0, 0, 0) };
- DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0)
+ DatanodeCommand[] cmd = dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null)
.getCommands();
assertEquals(1, cmd.length);
assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER
[26/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduceTutorial.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduceTutorial.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduceTutorial.apt.vm
deleted file mode 100644
index 9fb1056..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduceTutorial.apt.vm
+++ /dev/null
@@ -1,1605 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- MapReduce Tutorial
- ---
- ---
- ${maven.build.timestamp}
-
-MapReduce Tutorial
-
-%{toc|section=1|fromDepth=0|toDepth=4}
-
-* Purpose
-
- This document comprehensively describes all user-facing facets of
- the Hadoop MapReduce framework and serves as a tutorial.
-
-* Prerequisites
-
- Ensure that Hadoop is installed, configured and is running. More details:
-
- * {{{../../hadoop-project-dist/hadoop-common/SingleCluster.html}
- Single Node Setup}} for first-time users.
-
- * {{{../../hadoop-project-dist/hadoop-common/ClusterSetup.html}
- Cluster Setup}} for large, distributed clusters.
-
-* Overview
-
- Hadoop MapReduce is a software framework for easily writing applications
- which process vast amounts of data (multi-terabyte data-sets) in-parallel
- on large clusters (thousands of nodes) of commodity hardware in a reliable,
- fault-tolerant manner.
-
- A MapReduce <job> usually splits the input data-set into independent chunks
- which are processed by the <map tasks> in a completely parallel manner. The
- framework sorts the outputs of the maps, which are then input to the <reduce
- tasks>. Typically both the input and the output of the job are stored in
- a file-system. The framework takes care of scheduling tasks, monitoring them
- and re-executes the failed tasks.
-
- Typically the compute nodes and the storage nodes are the same, that is,
- the MapReduce framework and the Hadoop Distributed File System
- (see {{{../../hadoop-project-dist/hadoop-hdfs/HdfsDesign.html}
- HDFS Architecture Guide}}) are running on the same set of nodes. This
- configuration allows the framework to effectively schedule tasks on the nodes
- where data is already present, resulting in very high aggregate bandwidth
- across the cluster.
-
- The MapReduce framework consists of a single master <<<ResourceManager>>>,
- one slave <<<NodeManager>>> per cluster-node, and <<<MRAppMaster>>> per
- application (see {{{../../hadoop-yarn/hadoop-yarn-site/YARN.html}
- YARN Architecture Guide}}).
-
- Minimally, applications specify the input/output locations and supply <map>
- and <reduce> functions via implementations of appropriate interfaces and/or
- abstract-classes. These, and other job parameters, comprise the <job
- configuration>.
-
- The Hadoop <job client> then submits the job (jar/executable etc.) and
- configuration to the <<<ResourceManager>>> which then assumes the
- responsibility of distributing the software/configuration to the slaves,
- scheduling tasks and monitoring them, providing status and diagnostic
- information to the job-client.
-
- Although the Hadoop framework is implemented in Java\u2122, MapReduce
- applications need not be written in Java.
-
- * {{{../../api/org/apache/hadoop/streaming/package-summary.html}
- Hadoop Streaming}} is a utility which allows users to create and run jobs
- with any executables (e.g. shell utilities) as the mapper and/or the
- reducer.
-
- * {{{../../api/org/apache/hadoop/mapred/pipes/package-summary.html}
- Hadoop Pipes}} is a {{{http://www.swig.org/}SWIG}}-compatible C++ API to
- implement MapReduce applications (non JNI\u2122 based).
-
-* Inputs and Outputs
-
- The MapReduce framework operates exclusively on <<<\<key, value\>>>> pairs,
- that is, the framework views the input to the job as a set of <<<\<key,
- value\>>>> pairs and produces a set of <<<\<key, value\>>>> pairs as the
- output of the job, conceivably of different types.
-
- The <<<key>>> and <<<value>>> classes have to be serializable by the
- framework and hence need to implement the
- {{{../../api/org/apache/hadoop/io/Writable.html}Writable}} interface.
- Additionally, the key classes have to implement the
- {{{../../api/org/apache/hadoop/io/WritableComparable.html}
- WritableComparable}} interface to facilitate sorting by the framework.
-
- Input and Output types of a MapReduce job:
-
- (input) <<<\<k1, v1\> -\>>>> <<map>> <<<-\> \<k2, v2\> -\>>>> <<combine>>
- <<<-\> \<k2, v2\> -\>>>> <<reduce>> <<<-\> \<k3, v3\>>>> (output)
-
-* Example: WordCount v1.0
-
- Before we jump into the details, lets walk through an example MapReduce
- application to get a flavour for how they work.
-
- <<<WordCount>>> is a simple application that counts the number of
- occurrences of each word in a given input set.
-
- This works with a local-standalone, pseudo-distributed or fully-distributed
- Hadoop installation
- ({{{../../hadoop-project-dist/hadoop-common/SingleCluster.html}
- Single Node Setup}}).
-
-** Source Code
-
-+---+
-import java.io.IOException;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-
-public class WordCount {
-
- public static class TokenizerMapper
- extends Mapper<Object, Text, Text, IntWritable>{
-
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
-
- public void map(Object key, Text value, Context context
- ) throws IOException, InterruptedException {
- StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
- }
-
- public static class IntSumReducer
- extends Reducer<Text,IntWritable,Text,IntWritable> {
- private IntWritable result = new IntWritable();
-
- public void reduce(Text key, Iterable<IntWritable> values,
- Context context
- ) throws IOException, InterruptedException {
- int sum = 0;
- for (IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(key, result);
- }
- }
-
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
- Job job = Job.getInstance(conf, "word count");
- job.setJarByClass(WordCount.class);
- job.setMapperClass(TokenizerMapper.class);
- job.setCombinerClass(IntSumReducer.class);
- job.setReducerClass(IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
- FileInputFormat.addInputPath(job, new Path(args[0]));
- FileOutputFormat.setOutputPath(job, new Path(args[1]));
- System.exit(job.waitForCompletion(true) ? 0 : 1);
- }
-}
-+---+
-
-** Usage
-
- Assuming environment variables are set as follows:
-
-+---+
-export JAVA_HOME=/usr/java/default
-export PATH=$JAVA_HOME/bin:$PATH
-export HADOOP_CLASSPATH=$JAVA_HOME/lib/tools.jar
-+---+
-
- Compile <<<WordCount.java>>> and create a jar:
-
- <<<$ bin/hadoop com.sun.tools.javac.Main WordCount.java>>> \
- <<<$ jar cf wc.jar WordCount\*.class>>>
-
- Assuming that:
-
- * <<</user/joe/wordcount/input>>> - input directory in HDFS
-
- * <<</user/joe/wordcount/output>>> - output directory in HDFS
-
- Sample text-files as input:
-
- <<<$ bin/hdfs dfs -ls /user/joe/wordcount/input/>>> \
- <<</user/joe/wordcount/input/file01>>> \
- <<</user/joe/wordcount/input/file02>>>
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/input/file01>>> \
- <<<Hello World Bye World>>>
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/input/file02>>> \
- <<<Hello Hadoop Goodbye Hadoop>>>
-
- Run the application:
-
- <<<$ bin/hadoop jar wc.jar WordCount /user/joe/wordcount/input
- /user/joe/wordcount/output>>>
-
- Output:
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/output/part-r-00000>>>
-
- <<<Bye 1>>> \
- <<<Goodbye 1>>> \
- <<<Hadoop 2>>> \
- <<<Hello 2>>> \
- <<<World 2>>>
-
- Applications can specify a comma separated list of paths which would be
- present in the current working directory of the task using the option
- <<<-files>>>. The <<<-libjars>>> option allows applications to add jars to
- the classpaths of the maps and reduces. The option <<<-archives>>> allows
- them to pass comma separated list of archives as arguments. These archives
- are unarchived and a link with name of the archive is created in the current
- working directory of tasks. More details about the command line options are
- available at {{{../../hadoop-project-dist/hadoop-common/CommandsManual.html}
- Commands Guide}}.
-
- Running <<<wordcount>>> example with <<<-libjars>>>, <<<-files>>> and
- <<<-archives>>>: \
- <<<bin/hadoop jar hadoop-mapreduce-examples-<ver>.jar wordcount -files
- cachefile.txt -libjars mylib.jar -archives myarchive.zip input output>>>
- Here, myarchive.zip will be placed and unzipped into a directory by the name
- "myarchive.zip".
-
- Users can specify a different symbolic name for files and archives passed
- through <<<-files>>> and <<<-archives>>> option, using #.
-
- For example, <<<bin/hadoop jar hadoop-mapreduce-examples-<ver>.jar wordcount
- -files dir1/dict.txt#dict1,dir2/dict.txt#dict2 -archives mytar.tgz#tgzdir
- input output>>> Here, the files dir1/dict.txt and dir2/dict.txt can be
- accessed by tasks using the symbolic names dict1 and dict2 respectively.
- The archive mytar.tgz will be placed and unarchived into a directory by the
- name "tgzdir".
-
-** Walk-through
-
- The <<<WordCount>>> application is quite straight-forward.
-
-+---+
- public void map(Object key, Text value, Context context
- ) throws IOException, InterruptedException {
- StringTokenizer itr = new StringTokenizer(value.toString());
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
- }
- }
-+---+
-
- The <<<Mapper>>> implementation, via the <<<map>>> method, processes one
- line at a time, as provided by the specified <<<TextInputFormat>>>. It then
- splits the line into tokens separated by whitespaces, via the
- <<<StringTokenizer>>>, and emits a key-value pair of <<<\< \<word\>, 1\>>>>.
-
- For the given sample input the first map emits: \
- <<<\< Hello, 1\>>>> \
- <<<\< World, 1\>>>> \
- <<<\< Bye, 1\>>>> \
- <<<\< World, 1\>>>>
-
- The second map emits: \
- <<<\< Hello, 1\>>>> \
- <<<\< Hadoop, 1\>>>> \
- <<<\< Goodbye, 1\>>>> \
- <<<\< Hadoop, 1\>>>>
-
- We'll learn more about the number of maps spawned for a given job, and how to
- control them in a fine-grained manner, a bit later in the tutorial.
-
-+---+
- job.setCombinerClass(IntSumReducer.class);
-+---+
-
- <<<WordCount>>> also specifies a <<<combiner>>>. Hence, the output of each
- map is passed through the local combiner (which is same as the <<<Reducer>>>
- as per the job configuration) for local aggregation, after being sorted on
- the <key>s.
-
- The output of the first map: \
- <<<\< Bye, 1\>>>> \
- <<<\< Hello, 1\>>>> \
- <<<\< World, 2\>>>>
-
- The output of the second map: \
- <<<\< Goodbye, 1\>>>> \
- <<<\< Hadoop, 2\>>>> \
- <<<\< Hello, 1\>>>>
-
-+---+
- public void reduce(Text key, Iterable<IntWritable> values,
- Context context
- ) throws IOException, InterruptedException {
- int sum = 0;
- for (IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(key, result);
- }
-+---+
-
- The <<<Reducer>>> implementation, via the <<<reduce>>> method just sums up
- the values, which are the occurence counts for each key (i.e. words in this
- example).
-
- Thus the output of the job is: \
- <<<\< Bye, 1\>>>> \
- <<<\< Goodbye, 1\>>>> \
- <<<\< Hadoop, 2\>>>> \
- <<<\< Hello, 2\>>>> \
- <<<\< World, 2\>>>>
-
- The <<<main>>> method specifies various facets of the job, such as the
- input/output paths (passed via the command line), key/value types,
- input/output formats etc., in the <<<Job>>>. It then calls the
- <<<job.waitForCompletion>>> to submit the job and monitor its progress.
-
- We'll learn more about <<<Job>>>, <<<InputFormat>>>, <<<OutputFormat>>> and
- other interfaces and classes a bit later in the tutorial.
-
-* MapReduce - User Interfaces
-
- This section provides a reasonable amount of detail on every user-facing
- aspect of the MapReduce framework. This should help users implement,
- configure and tune their jobs in a fine-grained manner. However, please note
- that the javadoc for each class/interface remains the most comprehensive
- documentation available; this is only meant to be a tutorial.
-
- Let us first take the <<<Mapper>>> and <<<Reducer>>> interfaces. Applications
- typically implement them to provide the <<<map>>> and <<<reduce>>> methods.
-
- We will then discuss other core interfaces including <<<Job>>>,
- <<<Partitioner>>>, <<<InputFormat>>>, <<<OutputFormat>>>, and others.
-
- Finally, we will wrap up by discussing some useful features of the framework
- such as the <<<DistributedCache>>>, <<<IsolationRunner>>> etc.
-
-** Payload
-
- Applications typically implement the <<<Mapper>>> and <<<Reducer>>>
- interfaces to provide the <<<map>>> and <<<reduce>>> methods. These form
- the core of the job.
-
-*** Mapper
-
- {{{../../api/org/apache/hadoop/mapreduce/Mapper.html}Mapper}} maps input
- key/value pairs to a set of intermediate key/value pairs.
-
- Maps are the individual tasks that transform input records into intermediate
- records. The transformed intermediate records do not need to be of the same
- type as the input records. A given input pair may map to zero or many output
- pairs.
-
- The Hadoop MapReduce framework spawns one map task for each <<<InputSplit>>>
- generated by the <<<InputFormat>>> for the job.
-
- Overall, <<<Mapper>>> implementations are passed the <<<Job>>> for the job
- via the {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setMapperClass(Class)}} method. The framework then calls
- {{{../../api/org/apache/hadoop/mapreduce/Mapper.html}
- map(WritableComparable, Writable, Context)}} for each key/value pair in the
- <<<InputSplit>>> for that task. Applications can then override the
- <<<cleanup(Context)>>> method to perform any required cleanup.
-
- Output pairs do not need to be of the same types as input pairs. A given
- input pair may map to zero or many output pairs. Output pairs are collected
- with calls to context.write(WritableComparable, Writable).
-
- Applications can use the <<<Counter>>> to report its statistics.
-
- All intermediate values associated with a given output key are subsequently
- grouped by the framework, and passed to the <<<Reducer>>>(s) to determine the
- final output. Users can control the grouping by specifying a <<<Comparator>>>
- via {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setGroupingComparatorClass(Class)}}.
-
- The <<<Mapper>>> outputs are sorted and then partitioned per <<<Reducer>>>.
- The total number of partitions is the same as the number of reduce tasks for
- the job. Users can control which keys (and hence records) go to which
- <<<Reducer>>> by implementing a custom <<<Partitioner>>>.
-
- Users can optionally specify a <<<combiner>>>, via
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setCombinerClass(Class)}}, to perform local aggregation of the
- intermediate outputs, which helps to cut down the amount of data transferred
- from the <<<Mapper>>> to the <<<Reducer>>>.
-
- The intermediate, sorted outputs are always stored in a simple (key-len, key,
- value-len, value) format. Applications can control if, and how, the
- intermediate outputs are to be compressed and the
- {{{../../api/org/apache/hadoop/io/compress/CompressionCodec.html}
- CompressionCodec}} to be used via the <<<Configuration>>>.
-
-**** How Many Maps?
-
- The number of maps is usually driven by the total size of the inputs, that
- is, the total number of blocks of the input files.
-
- The right level of parallelism for maps seems to be around 10-100 maps
- per-node, although it has been set up to 300 maps for very cpu-light map
- tasks. Task setup takes a while, so it is best if the maps take at least a
- minute to execute.
-
- Thus, if you expect 10TB of input data and have a blocksize of <<<128MB>>>,
- you'll end up with 82,000 maps, unless
- Configuration.set(<<<MRJobConfig.NUM_MAPS>>>, int) (which only provides a
- hint to the framework) is used to set it even higher.
-
-*** Reducer
-
- {{{../../api/org/apache/hadoop/mapreduce/Reducer.html}Reducer}} reduces a
- set of intermediate values which share a key to a smaller set of values.
-
- The number of reduces for the job is set by the user via
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setNumReduceTasks(int)}}.
-
- Overall, <<<Reducer>>> implementations are passed the <<<Job>>> for the
- job via the {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setReducerClass(Class)}} method and can override it to initialize
- themselves. The framework then calls
- {{{../../api/org/apache/hadoop/mapreduce/Reducer.html}
- reduce(WritableComparable, Iterable\<Writable\>, Context)}} method for each
- <<<\<key, (list of values)\>>>> pair in the grouped inputs. Applications can
- then override the <<<cleanup(Context)>>> method to perform any required
- cleanup.
-
- <<<Reducer>>> has 3 primary phases: shuffle, sort and reduce.
-
-**** Shuffle
-
- Input to the <<<Reducer>>> is the sorted output of the mappers. In this phase
- the framework fetches the relevant partition of the output of all the
- mappers, via HTTP.
-
-**** Sort
-
- The framework groups <<<Reducer>>> inputs by keys (since different mappers
- may have output the same key) in this stage.
-
- The shuffle and sort phases occur simultaneously; while map-outputs are being
- fetched they are merged.
-
-**** Secondary Sort
-
- If equivalence rules for grouping the intermediate keys are required to be
- different from those for grouping keys before reduction, then one may specify
- a <<<Comparator>>> via
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setSortComparatorClass(Class)}}. Since
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setGroupingComparatorClass(Class)}} can be used to control how
- intermediate keys are grouped, these can be used in conjunction to simulate
- <secondary sort on values>.
-
-**** Reduce
-
- In this phase the reduce(WritableComparable, Iterable\<Writable\>, Context)
- method is called for each <<<\<key, (list of values)\>>>> pair in the grouped
- inputs.
-
- The output of the reduce task is typically written to the
- {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}} via
- Context.write(WritableComparable, Writable).
-
- Applications can use the <<<Counter>>> to report its statistics.
-
- The output of the <<<Reducer>>> is <not sorted>.
-
-**** How Many Reduces?
-
- The right number of reduces seems to be <<<0.95>>> or <<<1.75>>> multiplied
- by (\<<no. of nodes>\> * \<<no. of maximum containers per node>\>).
-
- With <<<0.95>>> all of the reduces can launch immediately and start
- transferring map outputs as the maps finish. With <<<1.75>>> the faster nodes
- will finish their first round of reduces and launch a second wave of reduces
- doing a much better job of load balancing.
-
- Increasing the number of reduces increases the framework overhead, but
- increases load balancing and lowers the cost of failures.
-
- The scaling factors above are slightly less than whole numbers to reserve a
- few reduce slots in the framework for speculative-tasks and failed tasks.
-
-**** Reducer NONE
-
- It is legal to set the number of reduce-tasks to <zero> if no reduction is
- desired.
-
- In this case the outputs of the map-tasks go directly to the
- <<<FileSystem>>>, into the output path set by
- {{{../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html}
- FileOutputFormat.setOutputPath(Job, Path)}}. The framework does not sort the
- map-outputs before writing them out to the <<<FileSystem>>>.
-
-*** Partitioner
-
- {{{../../api/org/apache/hadoop/mapreduce/Partitioner.html}Partitioner}}
- partitions the key space.
-
- Partitioner controls the partitioning of the keys of the intermediate
- map-outputs. The key (or a subset of the key) is used to derive the
- partition, typically by a <hash function>. The total number of partitions is
- the same as the number of reduce tasks for the job. Hence this controls which
- of the <<<m>>> reduce tasks the intermediate key (and hence the record) is
- sent to for reduction.
-
- {{{../../api/org/apache/hadoop/mapreduce/lib/partition/HashPartitioner.html}
- HashPartitioner}} is the default <<<Partitioner>>>.
-
-*** Counter
-
- {{{../../api/org/apache/hadoop/mapreduce/Counter.html}Counter}} is a facility
- for MapReduce applications to report its statistics.
-
- <<<Mapper>>> and <<<Reducer>>> implementations can use the <<<Counter>>> to
- report statistics.
-
- Hadoop MapReduce comes bundled with a
- {{{../../api/org/apache/hadoop/mapreduce/package-summary.html}library}}
- of generally useful mappers, reducers, and partitioners.
-
-** Job Configuration
-
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}Job}} represents a
- MapReduce job configuration.
-
- <<<Job>>> is the primary interface for a user to describe a MapReduce job to
- the Hadoop framework for execution. The framework tries to faithfully execute
- the job as described by <<<Job>>>, however:
-
- * Some configuration parameters may have been marked as final by
- administrators
- (see {{{../../api/org/apache/hadoop/conf/Configuration.html#FinalParams}
- Final Parameters}}) and hence cannot be altered.
-
- * While some job parameters are straight-forward to set (e.g.
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setNumReduceTasks(int)}}), other parameters interact subtly with the
- rest of the framework and/or job configuration and are more complex to set
- (e.g. {{{../../api/org/apache/hadoop/conf/Configuration.html}
- Configuration.set(<<<JobContext.NUM_MAPS>>>, int)}}).
-
- <<<Job>>> is typically used to specify the <<<Mapper>>>, combiner (if any),
- <<<Partitioner>>>, <<<Reducer>>>, <<<InputFormat>>>, <<<OutputFormat>>>
- implementations.
- {{{../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html}
- FileInputFormat}} indicates the set of input files
- ({{{../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html}
- FileInputFormat.setInputPaths(Job, Path...)}}/
- {{{../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html}
- FileInputFormat.addInputPath(Job, Path)}}) and
- ({{{../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html}
- FileInputFormat.setInputPaths(Job, String...)}}/
- {{{../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html}
- FileInputFormat.addInputPaths(Job, String))}} and where the output files
- should be written
- ({{{../../api/org/apache/hadoop/mapreduce/lib/input/FileOutputFormat.html}
- FileOutputFormat.setOutputPath(Path)}}).
-
- Optionally, <<<Job>>> is used to specify other advanced facets of the job
- such as the <<<Comparator>>> to be used, files to be put in the
- <<<DistributedCache>>>, whether intermediate and/or job outputs are to be
- compressed (and how), whether job tasks can be executed in a <speculative>
- manner ({{{../../api/org/apache/hadoop/mapreduce/Job.html}
- setMapSpeculativeExecution(boolean)}})/
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- setReduceSpeculativeExecution(boolean)}}),
- maximum number of attempts per task
- ({{{../../api/org/apache/hadoop/mapreduce/Job.html}setMaxMapAttempts(int)}}/
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- setMaxReduceAttempts(int)}}) etc.
-
- Of course, users can use
- {{{../../api/org/apache/hadoop/conf/Configuration.html}
- Configuration.set(String, String)}}/
- {{{../../api/org/apache/hadoop/conf/Configuration.html}
- Configuration.get(String)}} to set/get arbitrary parameters needed by
- applications. However, use the <<<DistributedCache>>> for large amounts of
- (read-only) data.
-
-** Task Execution & Environment
-
- The <<<MRAppMaster>>> executes the <<<Mapper>>>/<<<Reducer>>> <task> as a
- child process in a separate jvm.
-
- The child-task inherits the environment of the parent <<<MRAppMaster>>>. The
- user can specify additional options to the child-jvm via the
- <<<mapreduce.\{map|reduce\}.java.opts>>> and configuration parameter in the
- <<<Job>>> such as non-standard paths for the run-time linker to search
- shared libraries via <<<-Djava.library.path=\<\>>>> etc. If the
- <<<mapreduce.\{map|reduce\}.java.opts>>> parameters contains the symbol
- <@taskid@> it is interpolated with value of <<<taskid>>> of the MapReduce
- task.
-
- Here is an example with multiple arguments and substitutions, showing jvm GC
- logging, and start of a passwordless JVM JMX agent so that it can connect
- with jconsole and the likes to watch child memory, threads and get thread
- dumps. It also sets the maximum heap-size of the map and reduce child jvm to
- 512MB & 1024MB respectively. It also adds an additional path to the
- <<<java.library.path>>> of the child-jvm.
-
-+---+
-<property>
- <name>mapreduce.map.java.opts</name>
- <value>
- -Xmx512M -Djava.library.path=/home/mycompany/lib -verbose:gc -Xloggc:/tmp/@taskid@.gc
- -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false
- </value>
-</property>
-
-<property>
- <name>mapreduce.reduce.java.opts</name>
- <value>
- -Xmx1024M -Djava.library.path=/home/mycompany/lib -verbose:gc -Xloggc:/tmp/@taskid@.gc
- -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false
- </value>
-</property>
-+---+
-
-*** Memory Management
-
- Users/admins can also specify the maximum virtual memory of the launched
- child-task, and any sub-process it launches recursively, using
- <<<mapreduce.\{map|reduce\}.memory.mb>>>. Note that the value set here is a
- per process limit. The value for <<<mapreduce.\{map|reduce\}.memory.mb>>>
- should be specified in mega bytes (MB). And also the value must be greater
- than or equal to the -Xmx passed to JavaVM, else the VM might not start.
-
- Note: <<<mapreduce.\{map|reduce\}.java.opts>>> are used only for configuring
- the launched child tasks from MRAppMaster. Configuring the memory options for
- daemons is documented in
- {{{../../hadoop-project-dist/hadoop-common/ClusterSetup.html#Configuring_Environment_of_Hadoop_Daemons}
- Configuring the Environment of the Hadoop Daemons}}.
-
- The memory available to some parts of the framework is also configurable.
- In map and reduce tasks, performance may be influenced by adjusting
- parameters influencing the concurrency of operations and the frequency with
- which data will hit disk. Monitoring the filesystem counters for a job-
- particularly relative to byte counts from the map and into the reduce- is
- invaluable to the tuning of these parameters.
-
-*** Map Parameters
-
- A record emitted from a map will be serialized into a buffer and metadata
- will be stored into accounting buffers. As described in the following
- options, when either the serialization buffer or the metadata exceed a
- threshold, the contents of the buffers will be sorted and written to disk in
- the background while the map continues to output records. If either buffer
- fills completely while the spill is in progress, the map thread will block.
- When the map is finished, any remaining records are written to disk and all
- on-disk segments are merged into a single file. Minimizing the number of
- spills to disk can decrease map time, but a larger buffer also decreases the
- memory available to the mapper.
-
-*-------------*-------*-------------------------------------------------------*
-|| Name || Type || Description |
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.io.sort.mb | int | The cumulative size of the serialization
-| | | and accounting buffers storing records emitted from the
-| | | map, in megabytes.
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.map.sort.spill.percent | float | The soft limit in the
-| | | serialization buffer. Once reached, a thread will begin
-| | | to spill the contents to disk in the background.
-*-------------+-------+-------------------------------------------------------+
-
- Other notes
-
- * If either spill threshold is exceeded while a spill is in progress,
- collection will continue until the spill is finished. For example, if
- <<<mapreduce.map.sort.spill.percent>>> is set to 0.33, and the remainder
- of the buffer is filled while the spill runs, the next spill will include
- all the collected records, or 0.66 of the buffer, and will not generate
- additional spills. In other words, the thresholds are defining triggers,
- not blocking.
-
- * A record larger than the serialization buffer will first trigger a spill,
- then be spilled to a separate file. It is undefined whether or not this
- record will first pass through the combiner.
-
-*** Shuffle/Reduce Parameters
-
- As described previously, each reduce fetches the output assigned to it by the
- Partitioner via HTTP into memory and periodically merges these outputs to
- disk. If intermediate compression of map outputs is turned on, each output is
- decompressed into memory. The following options affect the frequency of these
- merges to disk prior to the reduce and the memory allocated to map output
- during the reduce.
-
-*-------------*-------*-------------------------------------------------------*
-|| Name || Type || Description |
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.io.soft.factor | int | Specifies the number of segments on
-| | | disk to be merged at the same time. It limits the
-| | | number of open files and compression codecs during
-| | | merge. If the number of files exceeds this limit, the
-| | | merge will proceed in several passes. Though this limit
-| | | also applies to the map, most jobs should be configured
-| | | so that hitting this limit is unlikely there.
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.reduce.merge.inmem.thresholds | int | The number of sorted map
-| | | outputs fetched into memory before being merged to
-| | | disk. Like the spill thresholds in the preceding note,
-| | | this is not defining a unit of partition, but a
-| | | trigger. In practice, this is usually set very high
-| | | (1000) or disabled (0), since merging in-memory
-| | | segments is often less expensive than merging from disk
-| | | (see notes following this table). This threshold
-| | | influences only the frequency of in-memory merges
-| | | during the shuffle.
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.reduce.shuffle.merge.percent | float | The memory threshold for
-| | | fetched map outputs before an in-memory merge is started,
-| | | expressed as a percentage of memory allocated to
-| | | storing map outputs in memory. Since map outputs that
-| | | can't fit in memory can be stalled, setting this high
-| | | may decrease parallelism between the fetch and merge.
-| | | Conversely, values as high as 1.0 have been effective
-| | | for reduces whose input can fit entirely in memory.
-| | | This parameter influences only the frequency of
-| | | in-memory merges during the shuffle.
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.reduce.shuffle.input.buffer.percent | float | The percentage of
-| | | memory- relative to the maximum heapsize as typically
-| | | specified in <<<mapreduce.reduce.java.opts>>>- that can
-| | | be allocated to storing map outputs during the shuffle.
-| | | Though some memory should be set aside for the
-| | | framework, in general it is advantageous to set this
-| | | high enough to store large and numerous map outputs.
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.reduce.input.buffer.percent | float | The percentage of memory
-| | | relative to the maximum heapsize in which map outputs
-| | | may be retained during the reduce. When the reduce
-| | | begins, map outputs will be merged to disk until those
-| | | that remain are under the resource limit this defines.
-| | | By default, all map outputs are merged to disk before
-| | | the reduce begins to maximize the memory available to
-| | | the reduce. For less memory-intensive reduces, this
-| | | should be increased to avoid trips to disk.
-*-------------+-------+-------------------------------------------------------+
-
- Other notes
-
- * If a map output is larger than 25 percent of the memory allocated to
- copying map outputs, it will be written directly to disk without first
- staging through memory.
-
- * When running with a combiner, the reasoning about high merge thresholds
- and large buffers may not hold. For merges started before all map outputs
- have been fetched, the combiner is run while spilling to disk. In some
- cases, one can obtain better reduce times by spending resources combining
- map outputs- making disk spills small and parallelizing spilling and
- fetching- rather than aggressively increasing buffer sizes.
-
- * When merging in-memory map outputs to disk to begin the reduce, if an
- intermediate merge is necessary because there are segments to spill and at
- least <<<mapreduce.task.io.sort.factor>>> segments already on disk, the
- in-memory map outputs will be part of the intermediate merge.
-
-*** Configured Parameters
-
- The following properties are localized in the job configuration for each
- task's execution:
-
-*-------------*-------*-------------------------------------------------------*
-|| Name || Type || Description |
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.job.id | String | The job id
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.job.jar | String | job.jar location in job directory
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.job.local.dir | String | The job specific shared scratch space
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.id | String | The task id
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.attempt.id | String | The task attempt id
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.is.map | boolean | Is this a map task
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.partition | int | The id of the task within the job
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.map.input.file | String | The filename that the map is reading from
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.map.input.start | long | The offset of the start of the map input
-| | | split
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.map.input.length | long | The number of bytes in the map input
-| | | split
-*-------------+-------+-------------------------------------------------------+
-| mapreduce.task.output.dir | String | The task's temporary output directory
-*-------------+-------+-------------------------------------------------------+
-
- <<Note:>> During the execution of a streaming job, the names of the
- "mapreduce" parameters are transformed. The dots ( . ) become underscores
- ( _ ). For example, mapreduce.job.id becomes mapreduce_job_id and
- mapreduce.job.jar becomes mapreduce_job_jar. To get the values in a streaming
- job's mapper/reducer use the parameter names with the underscores.
-
-*** Task Logs
-
- The standard output (stdout) and error (stderr) streams and the syslog of the
- task are read by the NodeManager and logged to
- <<<$\{HADOOP_LOG_DIR\}/userlogs>>>.
-
-*** Distributing Libraries
-
- The {{DistributedCache}} can also be used to distribute both jars and native
- libraries for use in the map and/or reduce tasks. The child-jvm always has
- its <current working directory> added to the <<<java.library.path>>> and
- <<<LD_LIBRARY_PATH>>>. And hence the cached libraries can be loaded via
- {{{http://docs.oracle.com/javase/7/docs/api/java/lang/System.html}
- System.loadLibrary}} or
- {{{http://docs.oracle.com/javase/7/docs/api/java/lang/System.html}
- System.load}}. More details on how to load shared libraries through
- distributed cache are documented at
- {{{../../hadoop-project-dist/hadoop-common/NativeLibraries.html#Native_Shared_Libraries}
- Native Libraries}}.
-
-** Job Submission and Monitoring
-
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}Job}} is the primary
- interface by which user-job interacts with the <<<ResourceManager>>>.
-
- <<<Job>>> provides facilities to submit jobs, track their progress, access
- component-tasks' reports and logs, get the MapReduce cluster's status
- information and so on.
-
- The job submission process involves:
-
- [[1]] Checking the input and output specifications of the job.
-
- [[2]] Computing the <<<InputSplit>>> values for the job.
-
- [[3]] Setting up the requisite accounting information for the
- <<<DistributedCache>>> of the job, if necessary.
-
- [[4]] Copying the job's jar and configuration to the MapReduce system
- directory on the <<<FileSystem>>>.
-
- [[5]] Submitting the job to the <<<ResourceManager>>> and optionally
- monitoring it's status.
-
- Job history files are also logged to user specified directory
- <<<mapreduce.jobhistory.intermediate-done-dir>>> and
- <<<mapreduce.jobhistory.done-dir>>>, which defaults to job output directory.
-
- User can view the history logs summary in specified directory using the
- following command \
- <<<$ mapred job -history output.jhist>>> \
- This command will print job details, failed and killed tip details. \
- More details about the job such as successful tasks and task attempts made
- for each task can be viewed using the following command \
- <<<$ mapred job -history all output.jhist>>>
-
- Normally the user uses <<<Job>>> to create the application, describe various
- facets of the job, submit the job, and monitor its progress.
-
-*** Job Control
-
- Users may need to chain MapReduce jobs to accomplish complex tasks which
- cannot be done via a single MapReduce job. This is fairly easy since the
- output of the job typically goes to distributed file-system, and the output,
- in turn, can be used as the input for the next job.
-
- However, this also means that the onus on ensuring jobs are complete
- (success/failure) lies squarely on the clients. In such cases, the various
- job-control options are:
-
- * {{{../../api/org/apache/hadoop/mapreduce/Job.html}Job.submit()}} :
- Submit the job to the cluster and return immediately.
-
- * {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.waitForCompletion(boolean)}} :
- Submit the job to the cluster and wait for it to finish.
-
-** Job Input
-
- {{{../../api/org/apache/hadoop/mapreduce/InputFormat.html}InputFormat}}
- describes the input-specification for a MapReduce job.
-
- The MapReduce framework relies on the <<<InputFormat>>> of the job to:
-
- [[1]] Validate the input-specification of the job.
-
- [[2]] Split-up the input file(s) into logical <<<InputSplit>>> instances,
- each of which is then assigned to an individual <<<Mapper>>>.
-
- [[3]] Provide the <<<RecordReader>>> implementation used to glean input
- records from the logical <<<InputSplit>>> for processing by the
- <<<Mapper>>>.
-
- The default behavior of file-based <<<InputFormat>>> implementations,
- typically sub-classes of
- {{{../../api/org/apache/hadoop/mapreduce/lib/input/FileInputFormat.html}
- FileInputFormat}}, is to split the input into <logical> <<<InputSplit>>>
- instances based on the total size, in bytes, of the input files. However, the
- <<<FileSystem>>> blocksize of the input files is treated as an upper bound
- for input splits. A lower bound on the split size can be set via
- <<<mapreduce.input.fileinputformat.split.minsize>>>.
-
- Clearly, logical splits based on input-size is insufficient for many
- applications since record boundaries must be respected. In such cases, the
- application should implement a <<<RecordReader>>>, who is responsible for
- respecting record-boundaries and presents a record-oriented view of the
- logical <<<InputSplit>>> to the individual task.
-
- {{{../../api/org/apache/hadoop/mapreduce/lib/input/TextInputFormat.html}
- TextInputFormat}} is the default <<<InputFormat>>>.
-
- If <<<TextInputFormat>>> is the <<<InputFormat>>> for a given job, the
- framework detects input-files with the <.gz> extensions and automatically
- decompresses them using the appropriate <<<CompressionCodec>>>. However, it
- must be noted that compressed files with the above extensions cannot be
- <split> and each compressed file is processed in its entirety by a single
- mapper.
-
-*** InputSplit
-
- {{{../../api/org/apache/hadoop/mapreduce/InputSplit.html}InputSplit}}
- represents the data to be processed by an individual <<<Mapper>>>.
-
- Typically <<<InputSplit>>> presents a byte-oriented view of the input, and it
- is the responsibility of <<<RecordReader>>> to process and present a
- record-oriented view.
-
- {{{../../api/org/apache/hadoop/mapreduce/lib/input/FileSplit.html}FileSplit}}
- is the default <<<InputSplit>>>. It sets <<<mapreduce.map.input.file>>> to
- the path of the input file for the logical split.
-
-*** RecordReader
-
- {{{../../api/org/apache/hadoop/mapreduce/RecordReader.html}RecordReader}}
- reads <<<\<key, value\>>>> pairs from an <<<InputSplit>>>.
-
- Typically the <<<RecordReader>>> converts the byte-oriented view of the
- input, provided by the <<<InputSplit>>>, and presents a record-oriented to
- the <<<Mapper>>> implementations for processing. <<<RecordReader>>> thus
- assumes the responsibility of processing record boundaries and presents the
- tasks with keys and values.
-
-** Job Output
-
- {{{../../api/org/apache/hadoop/mapreduce/OutputFormat.html}OutputFormat}}
- describes the output-specification for a MapReduce job.
-
- The MapReduce framework relies on the <<<OutputFormat>>> of the job to:
-
- [[1]] Validate the output-specification of the job; for example, check that
- the output directory doesn't already exist.
-
- [[2]] Provide the <<<RecordWriter>>> implementation used to write the output
- files of the job. Output files are stored in a <<<FileSystem>>>.
-
- <<<TextOutputFormat>>> is the default <<<OutputFormat>>>.
-
-*** OutputCommitter
-
- {{{../../api/org/apache/hadoop/mapreduce/OutputCommitter.html}
- OutputCommitter}} describes the commit of task output for a MapReduce job.
-
- The MapReduce framework relies on the <<<OutputCommitter>>> of the job to:
-
- [[1]] Setup the job during initialization. For example, create the temporary
- output directory for the job during the initialization of the job. Job
- setup is done by a separate task when the job is in PREP state and
- after initializing tasks. Once the setup task completes, the job will
- be moved to RUNNING state.
-
- [[2]] Cleanup the job after the job completion. For example, remove the
- temporary output directory after the job completion. Job cleanup is
- done by a separate task at the end of the job. Job is declared
- SUCCEDED/FAILED/KILLED after the cleanup task completes.
-
- [[3]] Setup the task temporary output. Task setup is done as part of the
- same task, during task initialization.
-
- [[4]] Check whether a task needs a commit. This is to avoid the commit
- procedure if a task does not need commit.
-
- [[5]] Commit of the task output. Once task is done, the task will commit
- it's output if required.
-
- [[6]] Discard the task commit. If the task has been failed/killed, the
- output will be cleaned-up. If task could not cleanup (in exception
- block), a separate task will be launched with same attempt-id to do
- the cleanup.
-
- <<<FileOutputCommitter>>> is the default <<<OutputCommitter>>>. Job
- setup/cleanup tasks occupy map or reduce containers, whichever is available
- on the NodeManager. And JobCleanup task, TaskCleanup tasks and JobSetup task
- have the highest priority, and in that order.
-
-*** Task Side-Effect Files
-
- In some applications, component tasks need to create and/or write to
- side-files, which differ from the actual job-output files.
-
- In such cases there could be issues with two instances of the same
- <<<Mapper>>> or <<<Reducer>>> running simultaneously (for example,
- speculative tasks) trying to open and/or write to the same file (path) on the
- <<<FileSystem>>>. Hence the application-writer will have to pick unique names
- per task-attempt (using the attemptid, say
- <<<attempt_200709221812_0001_m_000000_0>>>), not just per task.
-
- To avoid these issues the MapReduce framework, when the <<<OutputCommitter>>>
- is <<<FileOutputCommitter>>>, maintains a special
- <<<$\{mapreduce.output.fileoutputformat.outputdir\}/_temporary/_$\{taskid\}>>>
- sub-directory accessible via <<<$\{mapreduce.task.output.dir\}>>> for each
- task-attempt on the <<<FileSystem>>> where the output of the task-attempt is
- stored. On successful completion of the task-attempt, the files in the
- <<<$\{mapreduce.output.fileoutputformat.outputdir\}/_temporary/_$\{taskid\}>>>
- (only) are <promoted> to
- <<<$\{mapreduce.output.fileoutputformat.outputdir\}>>>. Of course, the
- framework discards the sub-directory of unsuccessful task-attempts. This
- process is completely transparent to the application.
-
- The application-writer can take advantage of this feature by creating any
- side-files required in <<<$\{mapreduce.task.output.dir\}>>> during execution
- of a task via
- {{{../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html}
- FileOutputFormat.getWorkOutputPath(Conext)}}, and the framework will promote
- them similarly for succesful task-attempts, thus eliminating the need to pick
- unique paths per task-attempt.
-
- Note: The value of <<<$\{mapreduce.task.output.dir\}>>> during execution of a
- particular task-attempt is actually
- <<<$\{mapreduce.output.fileoutputformat.outputdir\}/_temporary/_\{$taskid\}>>>,
- and this value is set by the MapReduce framework. So, just create any
- side-files in the path returned by
- {{{../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html}
- FileOutputFormat.getWorkOutputPath(Conext)}} from MapReduce task to take
- advantage of this feature.
-
- The entire discussion holds true for maps of jobs with reducer=NONE
- (i.e. 0 reduces) since output of the map, in that case, goes directly to
- HDFS.
-
-*** RecordWriter
-
- {{{../../api/org/apache/hadoop/mapreduce/RecordWriter.html}RecordWriter}}
- writes the output <<<\<key, value\>>>> pairs to an output file.
-
- RecordWriter implementations write the job outputs to the <<<FileSystem>>>.
-
-** Other Useful Features
-
-*** Submitting Jobs to Queues
-
- Users submit jobs to Queues. Queues, as collection of jobs, allow the system
- to provide specific functionality. For example, queues use ACLs to control
- which users who can submit jobs to them. Queues are expected to be primarily
- used by Hadoop Schedulers.
-
- Hadoop comes configured with a single mandatory queue, called 'default'.
- Queue names are defined in the <<<mapreduce.job.queuename>>>> property of the
- Hadoop site configuration. Some job schedulers, such as the
- {{{../../hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html}
- Capacity Scheduler}}, support multiple queues.
-
- A job defines the queue it needs to be submitted to through the
- <<<mapreduce.job.queuename>>> property, or through the
- Configuration.set(<<<MRJobConfig.QUEUE_NAME>>>, String) API. Setting the
- queue name is optional. If a job is submitted without an associated queue
- name, it is submitted to the 'default' queue.
-
-*** Counters
-
- <<<Counters>>> represent global counters, defined either by the MapReduce
- framework or applications. Each <<<Counter>>> can be of any <<<Enum>>> type.
- Counters of a particular <<<Enum>>> are bunched into groups of type
- <<<Counters.Group>>>.
-
- Applications can define arbitrary <<<Counters>>> (of type <<<Enum>>>) and
- update them via
- {{{../../api/org/apache/hadoop/mapred/Counters.html}
- Counters.incrCounter(Enum, long)}} or Counters.incrCounter(String, String,
- long) in the <<<map>>> and/or <<<reduce>>> methods. These counters are then
- globally aggregated by the framework.
-
-*** DistributedCache
-
- <<<DistributedCache>>> distributes application-specific, large, read-only
- files efficiently.
-
- <<<DistributedCache>>> is a facility provided by the MapReduce framework to
- cache files (text, archives, jars and so on) needed by applications.
-
- Applications specify the files to be cached via urls (hdfs://) in the
- <<<Job>>>. The <<<DistributedCache>>> assumes that the files specified via
- hdfs:// urls are already present on the <<<FileSystem>>>.
-
- The framework will copy the necessary files to the slave node before any
- tasks for the job are executed on that node. Its efficiency stems from the
- fact that the files are only copied once per job and the ability to cache
- archives which are un-archived on the slaves.
-
- <<<DistributedCache>>> tracks the modification timestamps of the cached
- files. Clearly the cache files should not be modified by the application or
- externally while the job is executing.
-
- <<<DistributedCache>>> can be used to distribute simple, read-only data/text
- files and more complex types such as archives and jars. Archives (zip, tar,
- tgz and tar.gz files) are <un-archived> at the slave nodes. Files have
- <execution permissions> set.
-
- The files/archives can be distributed by setting the property
- <<<mapreduce.job.cache.\{files|archives\}>>>. If more than one file/archive
- has to be distributed, they can be added as comma separated paths. The
- properties can also be set by APIs
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}Job.addCacheFile(URI)}}/
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}Job.addCacheArchive(URI)}}
- and
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setCacheFiles(URI\[\])}}/
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setCacheArchives(URI\[\])}} where URI is of the form
- <<<hdfs://host:port/absolute-path\#link-name>>>. In Streaming, the files can
- be distributed through command line option <<<-cacheFile/-cacheArchive>>>.
-
- The <<<DistributedCache>>> can also be used as a rudimentary software
- distribution mechanism for use in the map and/or reduce tasks. It can be used
- to distribute both jars and native libraries. The
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.addArchiveToClassPath(Path)}} or
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.addFileToClassPath(Path)}} api can be used to cache files/jars and also
- add them to the <classpath> of child-jvm. The same can be done by setting the
- configuration properties <<<mapreduce.job.classpath.\{files|archives\}>>>.
- Similarly the cached files that are symlinked into the working directory of
- the task can be used to distribute native libraries and load them.
-
-**** Private and Public DistributedCache Files
-
- DistributedCache files can be private or public, that determines how they can
- be shared on the slave nodes.
-
- * "Private" DistributedCache files are cached in a localdirectory private to
- the user whose jobs need these files. These files are shared by all tasks
- and jobs of the specific user only and cannot be accessed by jobs of
- other users on the slaves. A DistributedCache file becomes private by
- virtue of its permissions on the file system where the files are
- uploaded, typically HDFS. If the file has no world readable access, or if
- the directory path leading to the file has no world executable access for
- lookup, then the file becomes private.
-
- * "Public" DistributedCache files are cached in a global directory and the
- file access is setup such that they are publicly visible to all users.
- These files can be shared by tasks and jobs of all users on the slaves. A
- DistributedCache file becomes public by virtue of its permissions on the
- file system where the files are uploaded, typically HDFS. If the file has
- world readable access, AND if the directory path leading to the file has
- world executable access for lookup, then the file becomes public. In other
- words, if the user intends to make a file publicly available to all users,
- the file permissions must be set to be world readable, and the directory
- permissions on the path leading to the file must be world executable.
-
-*** Profiling
-
- Profiling is a utility to get a representative (2 or 3) sample of built-in
- java profiler for a sample of maps and reduces.
-
- User can specify whether the system should collect profiler information for
- some of the tasks in the job by setting the configuration property
- <<<mapreduce.task.profile>>>. The value can be set using the api
- Configuration.set(<<<MRJobConfig.TASK_PROFILE>>>, boolean). If the value is
- set <<<true>>>, the task profiling is enabled. The profiler information is
- stored in the user log directory. By default, profiling is not enabled for
- the job.
-
- Once user configures that profiling is needed, she/he can use the
- configuration property <<<mapreduce.task.profile.\{maps|reduces\}>>>
- to set the ranges of MapReduce tasks to profile. The value can be set using
- the api Configuration.set(<<<MRJobConfig.NUM_\{MAP|REDUCE\}_PROFILES>>>,
- String). By default, the specified range is <<<0-2>>>.
-
- User can also specify the profiler configuration arguments by setting the
- configuration property <<<mapreduce.task.profile.params>>>. The value can be
- specified using the api
- Configuration.set(<<<MRJobConfig.TASK_PROFILE_PARAMS>>>, String). If the
- string contains a <<<%s>>>, it will be replaced with the name of the
- profiling output file when the task runs. These parameters are passed to the
- task child JVM on the command line. The default value for the profiling
- parameters is
- <<<-agentlib:hprof=cpu=samples,heap=sites,force=n,thread=y,verbose=n,file=%s>>>.
-
-*** Debugging
-
- The MapReduce framework provides a facility to run user-provided scripts for
- debugging. When a MapReduce task fails, a user can run a debug script, to
- process task logs for example. The script is given access to the task's
- stdout and stderr outputs, syslog and jobconf. The output from the debug
- script's stdout and stderr is displayed on the console diagnostics and also
- as part of the job UI.
-
- In the following sections we discuss how to submit a debug script with a job.
- The script file needs to be distributed and submitted to the framework.
-
-**** How to distribute the script file:
-
- The user needs to use {{DistributedCache}} to <distribute> and <symlink> the
- script file.
-
-**** How to submit the script:
-
- A quick way to submit the debug script is to set values for the properties
- <<<mapreduce.map.debug.script>>> and <<<mapreduce.reduce.debug.script>>>, for
- debugging map and reduce tasks respectively. These properties can also be set
- by using APIs
- {{{../../api/org/apache/hadoop/conf/Configuration.html}
- Configuration.set(<<<MRJobConfig.MAP_DEBUG_SCRIPT>>>, String)}} and
- {{{../../api/org/apache/hadoop/conf/Configuration.html}
- Configuration.set(<<<MRJobConfig.REDUCE_DEBUG_SCRIPT>>>, String)}}. In
- streaming mode, a debug script can be submitted with the command-line options
- <<<-mapdebug>>> and <<<-reducedebug>>>, for debugging map and reduce tasks
- respectively.
-
- The arguments to the script are the task's stdout, stderr, syslog and jobconf
- files. The debug command, run on the node where the MapReduce task failed,
- is: \
- <<<$script $stdout $stderr $syslog $jobconf>>>
-
- Pipes programs have the c++ program name as a fifth argument for the command.
- Thus for the pipes programs the command is \
- <<<$script $stdout $stderr $syslog $jobconf $program>>>
-
-**** Default Behavior:
-
- For pipes, a default script is run to process core dumps under gdb, prints
- stack trace and gives info about running threads.
-
-*** Data Compression
-
- Hadoop MapReduce provides facilities for the application-writer to specify
- compression for both intermediate map-outputs and the job-outputs i.e. output
- of the reduces. It also comes bundled with
- {{{../../api/org/apache/hadoop/io/compress/CompressionCodec.html}
- CompressionCodec}} implementation for the {{{http://www.zlib.net}zlib}}
- compression algorithm. The {{{http://www.gzip.org}gzip}},
- {{{http://www.bzip.org}bzip2}}, {{{http://code.google.com/p/snappy/}snappy}},
- and {{{http://code.google.com/p/lz4/}lz4}} file format are also supported.
-
- Hadoop also provides native implementations of the above compression codecs
- for reasons of both performance (zlib) and non-availability of Java
- libraries. More details on their usage and availability are available
- {{{../../hadoop-project-dist/hadoop-common/NativeLibraries.html}here}}.
-
-**** Intermediate Outputs
-
- Applications can control compression of intermediate map-outputs via the
- Configuration.set(<<<MRJobConfig.MAP_OUTPUT_COMPRESS>>>, boolean) api and the
- <<<CompressionCodec>>> to be used via the
- Configuration.set(<<<MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC>>>, Class) api.
-
-**** Job Outputs
-
- Applications can control compression of job-outputs via the
- {{{../../api/org/apache/hadoop/mapreduce/lib/output/FileOutputFormat.html}
- FileOutputFormat.setCompressOutput(Job, boolean)}} api and the
- <<<CompressionCodec>>> to be used can be specified via the
- FileOutputFormat.setOutputCompressorClass(Job, Class) api.
-
- If the job outputs are to be stored in the
- {{{../../api/org/apache/hadoop/mapreduce/lib/output/SequenceFileOutputFormat.html}
- SequenceFileOutputFormat}}, the required <<<SequenceFile.CompressionType>>>
- (i.e. <<<RECORD>>> / <<<BLOCK>>> - defaults to <<<RECORD>>>) can be specified
- via the SequenceFileOutputFormat.setOutputCompressionType(Job,
- SequenceFile.CompressionType) api.
-
-*** Skipping Bad Records
-
- Hadoop provides an option where a certain set of bad input records can be
- skipped when processing map inputs. Applications can control this feature
- through the {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords}} class.
-
- This feature can be used when map tasks crash deterministically on certain
- input. This usually happens due to bugs in the map function. Usually, the
- user would have to fix these bugs. This is, however, not possible sometimes.
- The bug may be in third party libraries, for example, for which the source
- code is not available. In such cases, the task never completes successfully
- even after multiple attempts, and the job fails. With this feature, only a
- small portion of data surrounding the bad records is lost, which may be
- acceptable for some applications (those performing statistical analysis on
- very large data, for example).
-
- By default this feature is disabled. For enabling it, refer to
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.setMapperMaxSkipRecords(Configuration, long)}} and
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.setReducerMaxSkipGroups(Configuration, long)}}.
-
- With this feature enabled, the framework gets into 'skipping mode' after a
- certain number of map failures. For more details, see
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.setAttemptsToStartSkipping(Configuration, int)}}. In 'skipping
- mode', map tasks maintain the range of records being processed. To do this,
- the framework relies on the processed record counter. See
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.COUNTER_MAP_PROCESSED_RECORDS}} and
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.COUNTER_REDUCE_PROCESSED_GROUPS}}. This counter enables the
- framework to know how many records have been processed successfully, and
- hence, what record range caused a task to crash. On further attempts,
- this range of records is skipped.
-
- The number of records skipped depends on how frequently the processed record
- counter is incremented by the application. It is recommended that this
- counter be incremented after every record is processed. This may not be
- possible in some applications that typically batch their processing. In such
- cases, the framework may skip additional records surrounding the bad record.
- Users can control the number of skipped records through
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.setMapperMaxSkipRecords(Configuration, long)}} and
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.setReducerMaxSkipGroups(Configuration, long)}}. The framework
- tries to narrow the range of skipped records using a binary search-like
- approach. The skipped range is divided into two halves and only one half gets
- executed. On subsequent failures, the framework figures out which half
- contains bad records. A task will be re-executed till the acceptable skipped
- value is met or all task attempts are exhausted. To increase the number of
- task attempts, use
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setMaxMapAttempts(int)}} and
- {{{../../api/org/apache/hadoop/mapreduce/Job.html}
- Job.setMaxReduceAttempts(int)}}
-
- Skipped records are written to HDFS in the sequence file format, for later
- analysis. The location can be changed through
- {{{../../api/org/apache/hadoop/mapred/SkipBadRecords.html}
- SkipBadRecords.setSkipOutputPath(JobConf, Path)}}.
-
-** Example: WordCount v2.0
-
- Here is a more complete <<<WordCount>>> which uses many of the features
- provided by the MapReduce framework we discussed so far.
-
- This needs the HDFS to be up and running, especially for the
- <<<DistributedCache>>>-related features. Hence it only works with a
- {{{../../hadoop-project-dist/hadoop-common/SingleCluster.html}
- pseudo-distributed}} or
- {{{../../hadoop-project-dist/hadoop-common/ClusterSetup.html}
- fully-distributed}} Hadoop installation.
-
-*** Source Code
-
-+---+
-import java.io.BufferedReader;
-import java.io.FileReader;
-import java.io.IOException;
-import java.net.URI;
-import java.util.ArrayList;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-import java.util.StringTokenizer;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.IntWritable;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.Mapper;
-import org.apache.hadoop.mapreduce.Reducer;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.util.GenericOptionsParser;
-import org.apache.hadoop.util.StringUtils;
-
-public class WordCount2 {
-
- public static class TokenizerMapper
- extends Mapper<Object, Text, Text, IntWritable>{
-
- static enum CountersEnum { INPUT_WORDS }
-
- private final static IntWritable one = new IntWritable(1);
- private Text word = new Text();
-
- private boolean caseSensitive;
- private Set<String> patternsToSkip = new HashSet<String>();
-
- private Configuration conf;
- private BufferedReader fis;
-
- @Override
- public void setup(Context context) throws IOException,
- InterruptedException {
- conf = context.getConfiguration();
- caseSensitive = conf.getBoolean("wordcount.case.sensitive", true);
- if (conf.getBoolean("wordcount.skip.patterns", true)) {
- URI[] patternsURIs = Job.getInstance(conf).getCacheFiles();
- for (URI patternsURI : patternsURIs) {
- Path patternsPath = new Path(patternsURI.getPath());
- String patternsFileName = patternsPath.getName().toString();
- parseSkipFile(patternsFileName);
- }
- }
- }
-
- private void parseSkipFile(String fileName) {
- try {
- fis = new BufferedReader(new FileReader(fileName));
- String pattern = null;
- while ((pattern = fis.readLine()) != null) {
- patternsToSkip.add(pattern);
- }
- } catch (IOException ioe) {
- System.err.println("Caught exception while parsing the cached file '"
- + StringUtils.stringifyException(ioe));
- }
- }
-
- @Override
- public void map(Object key, Text value, Context context
- ) throws IOException, InterruptedException {
- String line = (caseSensitive) ?
- value.toString() : value.toString().toLowerCase();
- for (String pattern : patternsToSkip) {
- line = line.replaceAll(pattern, "");
- }
- StringTokenizer itr = new StringTokenizer(line);
- while (itr.hasMoreTokens()) {
- word.set(itr.nextToken());
- context.write(word, one);
- Counter counter = context.getCounter(CountersEnum.class.getName(),
- CountersEnum.INPUT_WORDS.toString());
- counter.increment(1);
- }
- }
- }
-
- public static class IntSumReducer
- extends Reducer<Text,IntWritable,Text,IntWritable> {
- private IntWritable result = new IntWritable();
-
- public void reduce(Text key, Iterable<IntWritable> values,
- Context context
- ) throws IOException, InterruptedException {
- int sum = 0;
- for (IntWritable val : values) {
- sum += val.get();
- }
- result.set(sum);
- context.write(key, result);
- }
- }
-
- public static void main(String[] args) throws Exception {
- Configuration conf = new Configuration();
- GenericOptionsParser optionParser = new GenericOptionsParser(conf, args);
- String[] remainingArgs = optionParser.getRemainingArgs();
- if (!(remainingArgs.length != 2 || remainingArgs.length != 4)) {
- System.err.println("Usage: wordcount <in> <out> [-skip skipPatternFile]");
- System.exit(2);
- }
- Job job = Job.getInstance(conf, "word count");
- job.setJarByClass(WordCount2.class);
- job.setMapperClass(TokenizerMapper.class);
- job.setCombinerClass(IntSumReducer.class);
- job.setReducerClass(IntSumReducer.class);
- job.setOutputKeyClass(Text.class);
- job.setOutputValueClass(IntWritable.class);
-
- List<String> otherArgs = new ArrayList<String>();
- for (int i=0; i < remainingArgs.length; ++i) {
- if ("-skip".equals(remainingArgs[i])) {
- job.addCacheFile(new Path(remainingArgs[++i]).toUri());
- job.getConfiguration().setBoolean("wordcount.skip.patterns", true);
- } else {
- otherArgs.add(remainingArgs[i]);
- }
- }
- FileInputFormat.addInputPath(job, new Path(otherArgs.get(0)));
- FileOutputFormat.setOutputPath(job, new Path(otherArgs.get(1)));
-
- System.exit(job.waitForCompletion(true) ? 0 : 1);
- }
-}
-+---+
-
-*** Sample Runs
-
- Sample text-files as input:
-
- <<<$ bin/hdfs dfs -ls /user/joe/wordcount/input/>>> \
- <<</user/joe/wordcount/input/file01>>> \
- <<</user/joe/wordcount/input/file02>>> \
- \
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/input/file01>>> \
- <<<Hello World, Bye World!>>> \
- \
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/input/file02>>> \
- <<<Hello Hadoop, Goodbye to hadoop.>>>
-
- Run the application:
-
- <<<$ bin/hadoop jar wc.jar WordCount2 /user/joe/wordcount/input
- /user/joe/wordcount/output>>>
-
- Output:
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/output/part-r-00000>>> \
- <<<Bye 1>>> \
- <<<Goodbye 1>>> \
- <<<Hadoop, 1>>> \
- <<<Hello 2>>> \
- <<<World! 1>>> \
- <<<World, 1>>> \
- <<<hadoop. 1>>> \
- <<<to 1>>>
-
- Notice that the inputs differ from the first version we looked at, and how
- they affect the outputs.
-
- Now, lets plug-in a pattern-file which lists the word-patterns to be ignored,
- via the <<<DistributedCache>>>.
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/patterns.txt>>> \
- <<<\\.>>> \
- <<<\\,>>> \
- <<<\\!>>> \
- <<<to>>>
-
- Run it again, this time with more options:
-
- <<<$ bin/hadoop jar wc.jar WordCount2
- -Dwordcount.case.sensitive=true /user/joe/wordcount/input
- /user/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt>>>
-
- As expected, the output:
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/output/part-r-00000>>> \
- <<<Bye 1>>> \
- <<<Goodbye 1>>> \
- <<<Hadoop 1>>> \
- <<<Hello 2>>> \
- <<<World 2>>> \
- <<<hadoop 1>>>
-
- Run it once more, this time switch-off case-sensitivity:
-
- <<<$ bin/hadoop jar wc.jar WordCount2
- -Dwordcount.case.sensitive=false /user/joe/wordcount/input
- /user/joe/wordcount/output -skip /user/joe/wordcount/patterns.txt>>>
-
- Sure enough, the output:
-
- <<<$ bin/hdfs dfs -cat /user/joe/wordcount/output/part-r-00000>>> \
- <<<bye 1>>> \
- <<<goodbye 1>>> \
- <<<hadoop 2>>> \
- <<<hello 2>>> \
- <<<horld 2>>>
-
-*** Highlights
-
- The second version of <<<WordCount>>> improves upon the previous one by using
- some features offered by the MapReduce framework:
-
- * Demonstrates how applications can access configuration parameters in the
- <<<setup>>> method of the <<<Mapper>>> (and <<<Reducer>>>)
- implementations.
-
- * Demonstrates how the <<<DistributedCache>>> can be used to distribute
- read-only data needed by the jobs. Here it allows the user to specify
- word-patterns to skip while counting.
-
- * Demonstrates the utility of the <<<GenericOptionsParser>>> to handle
- generic Hadoop command-line options.
-
- * Demonstrates how applications can use <<<Counters>>> and how they can set
- application-specific status information passed to the <<<map>>> (and
- <<<reduce>>>) method.
-
- <Java and JNI are trademarks or registered trademarks of Oracle America,
- Inc. in the United States and other countries.>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm
deleted file mode 100644
index e0fce63..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/apt/MapReduce_Compatibility_Hadoop1_Hadoop2.apt.vm
+++ /dev/null
@@ -1,114 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Map Reduce Next Generation-${project.version} - Backward Compatibility
- ---
- ---
- ${maven.build.timestamp}
-
-Apache Hadoop MapReduce - Migrating from Apache Hadoop 1.x to Apache Hadoop 2.x
-
-* {Introduction}
-
- This document provides information for users to migrate their Apache Hadoop
- MapReduce applications from Apache Hadoop 1.x to Apache Hadoop 2.x.
-
- In Apache Hadoop 2.x we have spun off resource management capabilities
- into Apache Hadoop YARN, a general purpose, distributed application management
- framework while Apache Hadoop MapReduce (aka MRv2) remains as a pure
- distributed computation framework.
-
- In general, the previous MapReduce runtime (aka MRv1) has been reused and
- no major surgery has been conducted on it. Therefore, MRv2 is able to ensure
- satisfactory compatibility with MRv1 applications. However, due to some
- improvements and code refactorings, a few APIs have been rendered
- backward-incompatible.
-
- The remainder of this page will discuss the scope and the level of backward
- compatibility that we support in Apache Hadoop MapReduce 2.x (MRv2).
-
-* {Binary Compatibility}
-
- First, we ensure binary compatibility to the applications that use old
- <<mapred>> APIs. This means that applications which were built against MRv1
- <<mapred>> APIs can run directly on YARN without recompilation, merely by
- pointing them to an Apache Hadoop 2.x cluster via configuration.
-
-* {Source Compatibility}
-
- We cannot ensure complete binary compatibility with the applications that use
- <<mapreduce>> APIs, as these APIs have evolved a lot since MRv1. However, we
- ensure source compatibility for <<mapreduce>> APIs that break binary
- compatibility. In other words, users should recompile their applications that
- use <<mapreduce>> APIs against MRv2 jars. One notable binary incompatibility
- break is Counter and CounterGroup.
-
-* {Not Supported}
-
- MRAdmin has been removed in MRv2 because because <<<mradmin>>> commands
- no longer exist. They have been replaced by the commands in <<<rmadmin>>>. We
- neither support binary compatibility nor source compatibility for the
- applications that use this class directly.
-
-* {Tradeoffs between MRv1 Users and Early MRv2 Adopters}
-
- Unfortunately, maintaining binary compatibility for MRv1 applications may lead
- to binary incompatibility issues for early MRv2 adopters, in particular Hadoop
- 0.23 users. For <<mapred>> APIs, we have chosen to be compatible with MRv1
- applications, which have a larger user base. For <<mapreduce>> APIs, if they
- don't significantly break Hadoop 0.23 applications, we still change them to be
- compatible with MRv1 applications. Below is the list of MapReduce APIs which
- are incompatible with Hadoop 0.23.
-
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<Problematic Function>> | <<Incompatibility Issue>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.util.ProgramDriver#drive>>> | Return type changes from <<<void>>> to <<<int>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapred.jobcontrol.Job#getMapredJobID>>> | Return type changes from <<<String>>> to <<<JobID>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapred.TaskReport#getTaskId>>> | Return type changes from <<<String>>> to <<<TaskID>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapred.ClusterStatus#UNINITIALIZED_MEMORY_VALUE>>> | Data type changes from <<<long>>> to <<<int>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapreduce.filecache.DistributedCache#getArchiveTimestamps>>> | Return type changes from <<<long[]>>> to <<<String[]>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapreduce.filecache.DistributedCache#getFileTimestamps>>> | Return type changes from <<<long[]>>> to <<<String[]>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapreduce.Job#failTask>>> | Return type changes from <<<void>>> to <<<boolean>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapreduce.Job#killTask>>> | Return type changes from <<<void>>> to <<<boolean>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-| <<<org.apache.hadoop.mapreduce.Job#getTaskCompletionEvents>>> | Return type changes from <<<o.a.h.mapred.TaskCompletionEvent[]>>> to <<<o.a.h.mapreduce.TaskCompletionEvent[]>>> |
-*-----------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------+
-
-* {Malicious}
-
- For the users who are going to try <<<hadoop-examples-1.x.x.jar>>> on YARN,
- please note that <<<hadoop -jar hadoop-examples-1.x.x.jar>>> will still use
- <<<hadoop-mapreduce-examples-2.x.x.jar>>>, which is installed together with
- other MRv2 jars. By default Hadoop framework jars appear before the users'
- jars in the classpath, such that the classes from the 2.x.x jar will still be
- picked. Users should remove <<<hadoop-mapreduce-examples-2.x.x.jar>>>
- from the classpath of all the nodes in a cluster. Otherwise, users need to
- set <<<HADOOP_USER_CLASSPATH_FIRST=true>>> and
- <<<HADOOP_CLASSPATH=...:hadoop-examples-1.x.x.jar>>> to run their target
- examples jar, and add the following configuration in <<<mapred-site.xml>>> to
- make the processes in YARN containers pick this jar as well.
-
-+---+
- <property>
- <name>mapreduce.job.user.classpath.first</name>
- <value>true</value>
- </property>
-+---+
[40/50] [abbrv] hadoop git commit: HADOOP-11593. Convert site
documentation from apt to markdown (stragglers) (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
new file mode 100644
index 0000000..0b64586
--- /dev/null
+++ b/hadoop-tools/hadoop-streaming/src/site/markdown/HadoopStreaming.md.vm
@@ -0,0 +1,559 @@
+%<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+#set ( $H3 = '###' )
+#set ( $H4 = '####' )
+#set ( $H5 = '#####' )
+
+Hadoop Streaming
+================
+
+* [Hadoop Streaming](#Hadoop_Streaming)
+ * [Hadoop Streaming](#Hadoop_Streaming)
+ * [How Streaming Works](#How_Streaming_Works)
+ * [Streaming Command Options](#Streaming_Command_Options)
+ * [Specifying a Java Class as the Mapper/Reducer](#Specifying_a_Java_Class_as_the_MapperReducer)
+ * [Packaging Files With Job Submissions](#Packaging_Files_With_Job_Submissions)
+ * [Specifying Other Plugins for Jobs](#Specifying_Other_Plugins_for_Jobs)
+ * [Setting Environment Variables](#Setting_Environment_Variables)
+ * [Generic Command Options](#Generic_Command_Options)
+ * [Specifying Configuration Variables with the -D Option](#Specifying_Configuration_Variables_with_the_-D_Option)
+ * [Specifying Directories](#Specifying_Directories)
+ * [Specifying Map-Only Jobs](#Specifying_Map-Only_Jobs)
+ * [Specifying the Number of Reducers](#Specifying_the_Number_of_Reducers)
+ * [Customizing How Lines are Split into Key/Value Pairs](#Customizing_How_Lines_are_Split_into_KeyValue_Pairs)
+ * [Working with Large Files and Archives](#Working_with_Large_Files_and_Archives)
+ * [Making Files Available to Tasks](#Making_Files_Available_to_Tasks)
+ * [Making Archives Available to Tasks](#Making_Archives_Available_to_Tasks)
+ * [More Usage Examples](#More_Usage_Examples)
+ * [Hadoop Partitioner Class](#Hadoop_Partitioner_Class)
+ * [Hadoop Comparator Class](#Hadoop_Comparator_Class)
+ * [Hadoop Aggregate Package](#Hadoop_Aggregate_Package)
+ * [Hadoop Field Selection Class](#Hadoop_Field_Selection_Class)
+ * [Frequently Asked Questions](#Frequently_Asked_Questions)
+ * [How do I use Hadoop Streaming to run an arbitrary set of (semi) independent tasks?](#How_do_I_use_Hadoop_Streaming_to_run_an_arbitrary_set_of_semi_independent_tasks)
+ * [How do I process files, one per map?](#How_do_I_process_files_one_per_map)
+ * [How many reducers should I use?](#How_many_reducers_should_I_use)
+ * [If I set up an alias in my shell script, will that work after -mapper?](#If_I_set_up_an_alias_in_my_shell_script_will_that_work_after_-mapper)
+ * [Can I use UNIX pipes?](#Can_I_use_UNIX_pipes)
+ * [What do I do if I get the "No space left on device" error?](#What_do_I_do_if_I_get_the_No_space_left_on_device_error)
+ * [How do I specify multiple input directories?](#How_do_I_specify_multiple_input_directories)
+ * [How do I generate output files with gzip format?](#How_do_I_generate_output_files_with_gzip_format)
+ * [How do I provide my own input/output format with streaming?](#How_do_I_provide_my_own_inputoutput_format_with_streaming)
+ * [How do I parse XML documents using streaming?](#How_do_I_parse_XML_documents_using_streaming)
+ * [How do I update counters in streaming applications?](#How_do_I_update_counters_in_streaming_applications)
+ * [How do I update status in streaming applications?](#How_do_I_update_status_in_streaming_applications)
+ * [How do I get the Job variables in a streaming job's mapper/reducer?](#How_do_I_get_the_Job_variables_in_a_streaming_jobs_mapperreducer)
+
+Hadoop Streaming
+----------------
+
+Hadoop streaming is a utility that comes with the Hadoop distribution. The utility allows you to create and run Map/Reduce jobs with any executable or script as the mapper and/or the reducer. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper /bin/cat \
+ -reducer /usr/bin/wc
+
+How Streaming Works
+-------------------
+
+In the above example, both the mapper and the reducer are executables that read the input from stdin (line by line) and emit the output to stdout. The utility will create a Map/Reduce job, submit the job to an appropriate cluster, and monitor the progress of the job until it completes.
+
+When an executable is specified for mappers, each mapper task will launch the executable as a separate process when the mapper is initialized. As the mapper task runs, it converts its inputs into lines and feed the lines to the stdin of the process. In the meantime, the mapper collects the line oriented outputs from the stdout of the process and converts each line into a key/value pair, which is collected as the output of the mapper. By default, the *prefix of a line up to the first tab character* is the `key` and the rest of the line (excluding the tab character) will be the `value`. If there is no tab character in the line, then entire line is considered as key and the value is null. However, this can be customized by setting `-inputformat` command option, as discussed later.
+
+When an executable is specified for reducers, each reducer task will launch the executable as a separate process then the reducer is initialized. As the reducer task runs, it converts its input key/values pairs into lines and feeds the lines to the stdin of the process. In the meantime, the reducer collects the line oriented outputs from the stdout of the process, converts each line into a key/value pair, which is collected as the output of the reducer. By default, the prefix of a line up to the first tab character is the key and the rest of the line (excluding the tab character) is the value. However, this can be customized by setting `-outputformat` command option, as discussed later.
+
+This is the basis for the communication protocol between the Map/Reduce framework and the streaming mapper/reducer.
+
+User can specify `stream.non.zero.exit.is.failure` as `true` or `false` to make a streaming task that exits with a non-zero status to be `Failure` or `Success` respectively. By default, streaming tasks exiting with non-zero status are considered to be failed tasks.
+
+Streaming Command Options
+-------------------------
+
+Streaming supports streaming command options as well as [generic command options](#Generic_Command_Options). The general command line syntax is shown below.
+
+**Note:** Be sure to place the generic options before the streaming options, otherwise the command will fail. For an example, see [Making Archives Available to Tasks](#Making_Archives_Available_to_Tasks).
+
+ hadoop command [genericOptions] [streamingOptions]
+
+The Hadoop streaming command options are listed here:
+
+| Parameter | Optional/Required | Description |
+|:---- |:---- |:---- |
+| -input directoryname or filename | Required | Input location for mapper |
+| -output directoryname | Required | Output location for reducer |
+| -mapper executable or JavaClassName | Required | Mapper executable |
+| -reducer executable or JavaClassName | Required | Reducer executable |
+| -file filename | Optional | Make the mapper, reducer, or combiner executable available locally on the compute nodes |
+| -inputformat JavaClassName | Optional | Class you supply should return key/value pairs of Text class. If not specified, TextInputFormat is used as the default |
+| -outputformat JavaClassName | Optional | Class you supply should take key/value pairs of Text class. If not specified, TextOutputformat is used as the default |
+| -partitioner JavaClassName | Optional | Class that determines which reduce a key is sent to |
+| -combiner streamingCommand or JavaClassName | Optional | Combiner executable for map output |
+| -cmdenv name=value | Optional | Pass environment variable to streaming commands |
+| -inputreader | Optional | For backwards-compatibility: specifies a record reader class (instead of an input format class) |
+| -verbose | Optional | Verbose output |
+| -lazyOutput | Optional | Create output lazily. For example, if the output format is based on FileOutputFormat, the output file is created only on the first call to Context.write |
+| -numReduceTasks | Optional | Specify the number of reducers |
+| -mapdebug | Optional | Script to call when map task fails |
+| -reducedebug | Optional | Script to call when reduce task fails |
+
+$H3 Specifying a Java Class as the Mapper/Reducer
+
+You can supply a Java class as the mapper and/or the reducer.
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -input myInputDirs \
+ -output myOutputDir \
+ -inputformat org.apache.hadoop.mapred.KeyValueTextInputFormat \
+ -mapper org.apache.hadoop.mapred.lib.IdentityMapper \
+ -reducer /usr/bin/wc
+
+You can specify `stream.non.zero.exit.is.failure` as `true` or `false` to make a streaming task that exits with a non-zero status to be `Failure` or `Success` respectively. By default, streaming tasks exiting with non-zero status are considered to be failed tasks.
+
+$H3 Packaging Files With Job Submissions
+
+You can specify any executable as the mapper and/or the reducer. The executables do not need to pre-exist on the machines in the cluster; however, if they don't, you will need to use "-file" option to tell the framework to pack your executable files as a part of job submission. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper myPythonScript.py \
+ -reducer /usr/bin/wc \
+ -file myPythonScript.py
+
+The above example specifies a user defined Python executable as the mapper. The option "-file myPythonScript.py" causes the python executable shipped to the cluster machines as a part of job submission.
+
+In addition to executable files, you can also package other auxiliary files (such as dictionaries, configuration files, etc) that may be used by the mapper and/or the reducer. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper myPythonScript.py \
+ -reducer /usr/bin/wc \
+ -file myPythonScript.py \
+ -file myDictionary.txt
+
+$H3 Specifying Other Plugins for Jobs
+
+Just as with a normal Map/Reduce job, you can specify other plugins for a streaming job:
+
+ -inputformat JavaClassName
+ -outputformat JavaClassName
+ -partitioner JavaClassName
+ -combiner streamingCommand or JavaClassName
+
+The class you supply for the input format should return key/value pairs of Text class. If you do not specify an input format class, the TextInputFormat is used as the default. Since the TextInputFormat returns keys of LongWritable class, which are actually not part of the input data, the keys will be discarded; only the values will be piped to the streaming mapper.
+
+The class you supply for the output format is expected to take key/value pairs of Text class. If you do not specify an output format class, the TextOutputFormat is used as the default.
+
+$H3 Setting Environment Variables
+
+To set an environment variable in a streaming command use:
+
+ -cmdenv EXAMPLE_DIR=/home/example/dictionaries/
+
+Generic Command Options
+-----------------------
+
+Streaming supports [streaming command options](#Streaming_Command_Options) as well as generic command options. The general command line syntax is shown below.
+
+**Note:** Be sure to place the generic options before the streaming options, otherwise the command will fail. For an example, see [Making Archives Available to Tasks](#Making_Archives_Available_to_Tasks).
+
+ hadoop command [genericOptions] [streamingOptions]
+
+The Hadoop generic command options you can use with streaming are listed here:
+
+| Parameter | Optional/Required | Description |
+|:---- |:---- |:---- |
+| -conf configuration\_file | Optional | Specify an application configuration file |
+| -D property=value | Optional | Use value for given property |
+| -fs host:port or local | Optional | Specify a namenode |
+| -files | Optional | Specify comma-separated files to be copied to the Map/Reduce cluster |
+| -libjars | Optional | Specify comma-separated jar files to include in the classpath |
+| -archives | Optional | Specify comma-separated archives to be unarchived on the compute machines |
+
+$H3 Specifying Configuration Variables with the -D Option
+
+You can specify additional configuration variables by using "-D \<property\>=\<value\>".
+
+$H4 Specifying Directories
+
+To change the local temp directory use:
+
+ -D dfs.data.dir=/tmp
+
+To specify additional local temp directories use:
+
+ -D mapred.local.dir=/tmp/local
+ -D mapred.system.dir=/tmp/system
+ -D mapred.temp.dir=/tmp/temp
+
+**Note:** For more details on job configuration parameters see: [mapred-default.xml](./mapred-default.xml)
+
+$H4 Specifying Map-Only Jobs
+
+Often, you may want to process input data using a map function only. To do this, simply set `mapreduce.job.reduces` to zero. The Map/Reduce framework will not create any reducer tasks. Rather, the outputs of the mapper tasks will be the final output of the job.
+
+ -D mapreduce.job.reduces=0
+
+To be backward compatible, Hadoop Streaming also supports the "-reducer NONE" option, which is equivalent to "-D mapreduce.job.reduces=0".
+
+$H4 Specifying the Number of Reducers
+
+To specify the number of reducers, for example two, use:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -D mapreduce.job.reduces=2 \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper /bin/cat \
+ -reducer /usr/bin/wc
+
+$H4 Customizing How Lines are Split into Key/Value Pairs
+
+As noted earlier, when the Map/Reduce framework reads a line from the stdout of the mapper, it splits the line into a key/value pair. By default, the prefix of the line up to the first tab character is the key and the rest of the line (excluding the tab character) is the value.
+
+However, you can customize this default. You can specify a field separator other than the tab character (the default), and you can specify the nth (n \>= 1) character rather than the first character in a line (the default) as the separator between the key and value. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -D stream.map.output.field.separator=. \
+ -D stream.num.map.output.key.fields=4 \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper /bin/cat \
+ -reducer /bin/cat
+
+In the above example, "-D stream.map.output.field.separator=." specifies "." as the field separator for the map outputs, and the prefix up to the fourth "." in a line will be the key and the rest of the line (excluding the fourth ".") will be the value. If a line has less than four "."s, then the whole line will be the key and the value will be an empty Text object (like the one created by new Text("")).
+
+Similarly, you can use "-D stream.reduce.output.field.separator=SEP" and "-D stream.num.reduce.output.fields=NUM" to specify the nth field separator in a line of the reduce outputs as the separator between the key and the value.
+
+Similarly, you can specify "stream.map.input.field.separator" and "stream.reduce.input.field.separator" as the input separator for Map/Reduce inputs. By default the separator is the tab character.
+
+$H3 Working with Large Files and Archives
+
+The -files and -archives options allow you to make files and archives available to the tasks. The argument is a URI to the file or archive that you have already uploaded to HDFS. These files and archives are cached across jobs. You can retrieve the host and fs\_port values from the fs.default.name config variable.
+
+**Note:** The -files and -archives options are generic options. Be sure to place the generic options before the command options, otherwise the command will fail.
+
+$H4 Making Files Available to Tasks
+
+The -files option creates a symlink in the current working directory of the tasks that points to the local copy of the file.
+
+In this example, Hadoop automatically creates a symlink named testfile.txt in the current working directory of the tasks. This symlink points to the local copy of testfile.txt.
+
+ -files hdfs://host:fs_port/user/testfile.txt
+
+User can specify a different symlink name for -files using \#.
+
+ -files hdfs://host:fs_port/user/testfile.txt#testfile
+
+Multiple entries can be specified like this:
+
+ -files hdfs://host:fs_port/user/testfile1.txt,hdfs://host:fs_port/user/testfile2.txt
+
+$H4 Making Archives Available to Tasks
+
+The -archives option allows you to copy jars locally to the current working directory of tasks and automatically unjar the files.
+
+In this example, Hadoop automatically creates a symlink named testfile.jar in the current working directory of tasks. This symlink points to the directory that stores the unjarred contents of the uploaded jar file.
+
+ -archives hdfs://host:fs_port/user/testfile.jar
+
+User can specify a different symlink name for -archives using \#.
+
+ -archives hdfs://host:fs_port/user/testfile.tgz#tgzdir
+
+In this example, the input.txt file has two lines specifying the names of the two files: cachedir.jar/cache.txt and cachedir.jar/cache2.txt. "cachedir.jar" is a symlink to the archived directory, which has the files "cache.txt" and "cache2.txt".
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -archives 'hdfs://hadoop-nn1.example.com/user/me/samples/cachefile/cachedir.jar' \
+ -D mapreduce.job.maps=1 \
+ -D mapreduce.job.reduces=1 \
+ -D mapreduce.job.name="Experiment" \
+ -input "/user/me/samples/cachefile/input.txt" \
+ -output "/user/me/samples/cachefile/out" \
+ -mapper "xargs cat" \
+ -reducer "cat"
+
+ $ ls test_jar/
+ cache.txt cache2.txt
+
+ $ jar cvf cachedir.jar -C test_jar/ .
+ added manifest
+ adding: cache.txt(in = 30) (out= 29)(deflated 3%)
+ adding: cache2.txt(in = 37) (out= 35)(deflated 5%)
+
+ $ hdfs dfs -put cachedir.jar samples/cachefile
+
+ $ hdfs dfs -cat /user/me/samples/cachefile/input.txt
+ cachedir.jar/cache.txt
+ cachedir.jar/cache2.txt
+
+ $ cat test_jar/cache.txt
+ This is just the cache string
+
+ $ cat test_jar/cache2.txt
+ This is just the second cache string
+
+ $ hdfs dfs -ls /user/me/samples/cachefile/out
+ Found 2 items
+ -rw-r--r-* 1 me supergroup 0 2013-11-14 17:00 /user/me/samples/cachefile/out/_SUCCESS
+ -rw-r--r-* 1 me supergroup 69 2013-11-14 17:00 /user/me/samples/cachefile/out/part-00000
+
+ $ hdfs dfs -cat /user/me/samples/cachefile/out/part-00000
+ This is just the cache string
+ This is just the second cache string
+
+More Usage Examples
+-------------------
+
+$H3 Hadoop Partitioner Class
+
+Hadoop has a library class, [KeyFieldBasedPartitioner](../../api/org/apache/hadoop/mapred/lib/KeyFieldBasedPartitioner.html), that is useful for many applications. This class allows the Map/Reduce framework to partition the map outputs based on certain key fields, not the whole keys. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -D stream.map.output.field.separator=. \
+ -D stream.num.map.output.key.fields=4 \
+ -D map.output.key.field.separator=. \
+ -D mapreduce.partition.keypartitioner.options=-k1,2 \
+ -D mapreduce.job.reduces=12 \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper /bin/cat \
+ -reducer /bin/cat \
+ -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner
+
+Here, *-D stream.map.output.field.separator=.* and *-D stream.num.map.output.key.fields=4* are as explained in previous example. The two variables are used by streaming to identify the key/value pair of mapper.
+
+The map output keys of the above Map/Reduce job normally have four fields separated by ".". However, the Map/Reduce framework will partition the map outputs by the first two fields of the keys using the *-D mapred.text.key.partitioner.options=-k1,2* option. Here, *-D map.output.key.field.separator=.* specifies the separator for the partition. This guarantees that all the key/value pairs with the same first two fields in the keys will be partitioned into the same reducer.
+
+*This is effectively equivalent to specifying the first two fields as the primary key and the next two fields as the secondary. The primary key is used for partitioning, and the combination of the primary and secondary keys is used for sorting.* A simple illustration is shown here:
+
+Output of map (the keys)
+
+ 11.12.1.2
+ 11.14.2.3
+ 11.11.4.1
+ 11.12.1.1
+ 11.14.2.2
+
+Partition into 3 reducers (the first 2 fields are used as keys for partition)
+
+ 11.11.4.1
+ -----------
+ 11.12.1.2
+ 11.12.1.1
+ -----------
+ 11.14.2.3
+ 11.14.2.2
+
+Sorting within each partition for the reducer(all 4 fields used for sorting)
+
+ 11.11.4.1
+ -----------
+ 11.12.1.1
+ 11.12.1.2
+ -----------
+ 11.14.2.2
+ 11.14.2.3
+
+$H3 Hadoop Comparator Class
+
+Hadoop has a library class, [KeyFieldBasedComparator](../../api/org/apache/hadoop/mapreduce/lib/partition/KeyFieldBasedComparator.html), that is useful for many applications. This class provides a subset of features provided by the Unix/GNU Sort. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -D mapreduce.job.output.key.comparator.class=org.apache.hadoop.mapreduce.lib.partition.KeyFieldBasedComparator \
+ -D stream.map.output.field.separator=. \
+ -D stream.num.map.output.key.fields=4 \
+ -D mapreduce.map.output.key.field.separator=. \
+ -D mapreduce.partition.keycomparator.options=-k2,2nr \
+ -D mapreduce.job.reduces=1 \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper /bin/cat \
+ -reducer /bin/cat
+
+The map output keys of the above Map/Reduce job normally have four fields separated by ".". However, the Map/Reduce framework will sort the outputs by the second field of the keys using the *-D mapreduce.partition.keycomparator.options=-k2,2nr* option. Here, *-n* specifies that the sorting is numerical sorting and *-r* specifies that the result should be reversed. A simple illustration is shown below:
+
+Output of map (the keys)
+
+ 11.12.1.2
+ 11.14.2.3
+ 11.11.4.1
+ 11.12.1.1
+ 11.14.2.2
+
+Sorting output for the reducer (where second field used for sorting)
+
+ 11.14.2.3
+ 11.14.2.2
+ 11.12.1.2
+ 11.12.1.1
+ 11.11.4.1
+
+$H3 Hadoop Aggregate Package
+
+Hadoop has a library package called [Aggregate](../../org/apache/hadoop/mapred/lib/aggregate/package-summary.html). Aggregate provides a special reducer class and a special combiner class, and a list of simple aggregators that perform aggregations such as "sum", "max", "min" and so on over a sequence of values. Aggregate allows you to define a mapper plugin class that is expected to generate "aggregatable items" for each input key/value pair of the mappers. The combiner/reducer will aggregate those aggregatable items by invoking the appropriate aggregators.
+
+To use Aggregate, simply specify "-reducer aggregate":
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper myAggregatorForKeyCount.py \
+ -reducer aggregate \
+ -file myAggregatorForKeyCount.py \
+
+The python program myAggregatorForKeyCount.py looks like:
+
+ #!/usr/bin/python
+
+ import sys;
+
+ def generateLongCountToken(id):
+ return "LongValueSum:" + id + "\t" + "1"
+
+ def main(argv):
+ line = sys.stdin.readline();
+ try:
+ while line:
+ line = line[:-1];
+ fields = line.split("\t");
+ print generateLongCountToken(fields[0]);
+ line = sys.stdin.readline();
+ except "end of file":
+ return None
+ if __name__ == "__main__":
+ main(sys.argv)
+
+$H3 Hadoop Field Selection Class
+
+Hadoop has a library class, [FieldSelectionMapReduce](../../api/org/apache/hadoop/mapred/lib/FieldSelectionMapReduce.html), that effectively allows you to process text data like the unix "cut" utility. The map function defined in the class treats each input key/value pair as a list of fields. You can specify the field separator (the default is the tab character). You can select an arbitrary list of fields as the map output key, and an arbitrary list of fields as the map output value. Similarly, the reduce function defined in the class treats each input key/value pair as a list of fields. You can select an arbitrary list of fields as the reduce output key, and an arbitrary list of fields as the reduce output value. For example:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -D mapreduce.map.output.key.field.separator=. \
+ -D mapreduce.partition.keypartitioner.options=-k1,2 \
+ -D mapreduce.fieldsel.data.field.separator=. \
+ -D mapreduce.fieldsel.map.output.key.value.fields.spec=6,5,1-3:0- \
+ -D mapreduce.fieldsel.reduce.output.key.value.fields.spec=0-2:5- \
+ -D mapreduce.map.output.key.class=org.apache.hadoop.io.Text \
+ -D mapreduce.job.reduces=12 \
+ -input myInputDirs \
+ -output myOutputDir \
+ -mapper org.apache.hadoop.mapred.lib.FieldSelectionMapReduce \
+ -reducer org.apache.hadoop.mapred.lib.FieldSelectionMapReduce \
+ -partitioner org.apache.hadoop.mapred.lib.KeyFieldBasedPartitioner
+
+The option "-D mapreduce.fieldsel.map.output.key.value.fields.spec=6,5,1-3:0-" specifies key/value selection for the map outputs. Key selection spec and value selection spec are separated by ":". In this case, the map output key will consist of fields 6, 5, 1, 2, and 3. The map output value will consist of all fields (0- means field 0 and all the subsequent fields).
+
+The option "-D mapreduce.fieldsel.reduce.output.key.value.fields.spec=0-2:5-" specifies key/value selection for the reduce outputs. In this case, the reduce output key will consist of fields 0, 1, 2 (corresponding to the original fields 6, 5, 1). The reduce output value will consist of all fields starting from field 5 (corresponding to all the original fields).
+
+Frequently Asked Questions
+--------------------------
+
+$H3 How do I use Hadoop Streaming to run an arbitrary set of (semi) independent tasks?
+
+Often you do not need the full power of Map Reduce, but only need to run multiple instances of the same program - either on different parts of the data, or on the same data, but with different parameters. You can use Hadoop Streaming to do this.
+
+$H3 How do I process files, one per map?
+
+As an example, consider the problem of zipping (compressing) a set of files across the hadoop cluster. You can achieve this by using Hadoop Streaming and custom mapper script:
+
+* Generate a file containing the full HDFS path of the input files. Each map
+ task would get one file name as input.
+
+* Create a mapper script which, given a filename, will get the file to local
+ disk, gzip the file and put it back in the desired output directory.
+
+$H3 How many reducers should I use?
+
+See MapReduce Tutorial for details: [Reducer](./MapReduceTutorial.html#Reducer)
+
+$H3 If I set up an alias in my shell script, will that work after -mapper?
+
+For example, say I do: alias c1='cut -f1'. Will -mapper "c1" work?
+
+Using an alias will not work, but variable substitution is allowed as shown in this example:
+
+ $ hdfs dfs -cat /user/me/samples/student_marks
+ alice 50
+ bruce 70
+ charlie 80
+ dan 75
+
+ $ c2='cut -f2'; hadoop jar hadoop-streaming-${project.version}.jar \
+ -D mapreduce.job.name='Experiment' \
+ -input /user/me/samples/student_marks \
+ -output /user/me/samples/student_out \
+ -mapper "$c2" -reducer 'cat'
+
+ $ hdfs dfs -cat /user/me/samples/student_out/part-00000
+ 50
+ 70
+ 75
+ 80
+
+$H3 Can I use UNIX pipes?
+
+For example, will -mapper "cut -f1 | sed s/foo/bar/g" work?
+
+Currently this does not work and gives an "java.io.IOException: Broken pipe" error. This is probably a bug that needs to be investigated.
+
+$H3 What do I do if I get the "No space left on device" error?
+
+For example, when I run a streaming job by distributing large executables (for example, 3.6G) through the -file option, I get a "No space left on device" error.
+
+The jar packaging happens in a directory pointed to by the configuration variable stream.tmpdir. The default value of stream.tmpdir is /tmp. Set the value to a directory with more space:
+
+ -D stream.tmpdir=/export/bigspace/...
+
+$H3 How do I specify multiple input directories?
+
+You can specify multiple input directories with multiple '-input' options:
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -input '/user/foo/dir1' -input '/user/foo/dir2' \
+ (rest of the command)
+
+$H3 How do I generate output files with gzip format?
+
+Instead of plain text files, you can generate gzip files as your generated output. Pass '-D mapreduce.output.fileoutputformat.compress=true -D mapreduce.output.fileoutputformat.compress.codec=org.apache.hadoop.io.compress.GzipCodec' as option to your streaming job.
+
+$H3 How do I provide my own input/output format with streaming?
+
+You can specify your own custom class by packing them and putting the custom jar to `$HADOOP_CLASSPATH`.
+
+$H3 How do I parse XML documents using streaming?
+
+You can use the record reader StreamXmlRecordReader to process XML documents.
+
+ hadoop jar hadoop-streaming-${project.version}.jar \
+ -inputreader "StreamXmlRecord,begin=BEGIN_STRING,end=END_STRING" \
+ (rest of the command)
+
+Anything found between BEGIN\_STRING and END\_STRING would be treated as one record for map tasks.
+
+$H3 How do I update counters in streaming applications?
+
+A streaming process can use the stderr to emit counter information. `reporter:counter:<group>,<counter>,<amount>` should be sent to stderr to update the counter.
+
+$H3 How do I update status in streaming applications?
+
+A streaming process can use the stderr to emit status information. To set a status, `reporter:status:<message>` should be sent to stderr.
+
+$H3 How do I get the Job variables in a streaming job's mapper/reducer?
+
+See [Configured Parameters](./MapReduceTutorial.html#Configured_Parameters). During the execution of a streaming job, the names of the "mapred" parameters are transformed. The dots ( . ) become underscores ( \_ ). For example, mapreduce.job.id becomes mapreduce\_job\_id and mapreduce.job.jar becomes mapreduce\_job\_jar. In your code, use the parameter names with the underscores.
[39/50] [abbrv] hadoop git commit: MAPREDUCE-4286.
TestClientProtocolProviderImpls passes on failure conditions. Contributed by
Devaraj K.
Posted by zj...@apache.org.
MAPREDUCE-4286. TestClientProtocolProviderImpls passes on failure conditions. Contributed by Devaraj K.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c782047
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c782047
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c782047
Branch: refs/heads/YARN-2928
Commit: 7c782047c609b29178945bd566a0d162e64dbfdb
Parents: 57db50c
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Feb 18 15:45:52 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Feb 18 15:45:52 2015 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../TestClientProtocolProviderImpls.java | 71 ++++++--------------
2 files changed, 25 insertions(+), 49 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c782047/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 50e067c..e944d82 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -376,6 +376,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6221. Stringifier is left unclosed in Chain#getChainElementConf().
(Ted Yu via ozawa)
+ MAPREDUCE-4286. TestClientProtocolProviderImpls passes on failure
+ conditions. (Devaraj K via ozawa)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c782047/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
index e71c038..6ad76e9 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/TestClientProtocolProviderImpls.java
@@ -18,30 +18,22 @@
package org.apache.hadoop.mapreduce;
-import java.io.IOException;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
-import junit.framework.TestCase;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.mapred.LocalJobRunner;
+import org.apache.hadoop.mapred.YARNRunner;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.junit.Test;
-public class TestClientProtocolProviderImpls extends TestCase {
+public class TestClientProtocolProviderImpls {
@Test
public void testClusterWithLocalClientProvider() throws Exception {
-
Configuration conf = new Configuration();
-
- try {
- conf.set(MRConfig.FRAMEWORK_NAME, "incorrect");
- new Cluster(conf);
- fail("Cluster should not be initialized with incorrect framework name");
- } catch (IOException e) {
-
- }
-
conf.set(MRConfig.FRAMEWORK_NAME, "local");
Cluster cluster = new Cluster(conf);
assertTrue(cluster.getClient() instanceof LocalJobRunner);
@@ -50,57 +42,38 @@ public class TestClientProtocolProviderImpls extends TestCase {
@Test
public void testClusterWithJTClientProvider() throws Exception {
-
Configuration conf = new Configuration();
try {
- conf.set(MRConfig.FRAMEWORK_NAME, "incorrect");
- new Cluster(conf);
- fail("Cluster should not be initialized with incorrect framework name");
-
- } catch (IOException e) {
-
- }
-
- try {
conf.set(MRConfig.FRAMEWORK_NAME, "classic");
conf.set(JTConfig.JT_IPC_ADDRESS, "local");
new Cluster(conf);
- fail("Cluster with classic Framework name shouldnot use local JT address");
-
+ fail("Cluster with classic Framework name should not use "
+ + "local JT address");
} catch (IOException e) {
-
+ assertTrue(e.getMessage().contains(
+ "Cannot initialize Cluster. Please check"));
}
+ }
- try {
- conf = new Configuration();
- conf.set(MRConfig.FRAMEWORK_NAME, "classic");
- conf.set(JTConfig.JT_IPC_ADDRESS, "127.0.0.1:0");
- Cluster cluster = new Cluster(conf);
- cluster.close();
- } catch (IOException e) {
-
- }
+ @Test
+ public void testClusterWithYarnClientProvider() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(MRConfig.FRAMEWORK_NAME, "yarn");
+ Cluster cluster = new Cluster(conf);
+ assertTrue(cluster.getClient() instanceof YARNRunner);
+ cluster.close();
}
@Test
public void testClusterException() {
-
Configuration conf = new Configuration();
- conf.set(MRConfig.FRAMEWORK_NAME, MRConfig.CLASSIC_FRAMEWORK_NAME);
- conf.set(JTConfig.JT_IPC_ADDRESS, "local");
-
- // initializing a cluster with this conf should throw an error.
- // However the exception thrown should not be specific to either
- // the job tracker client provider or the local provider
- boolean errorThrown = false;
try {
- Cluster cluster = new Cluster(conf);
- cluster.close();
- fail("Not expected - cluster init should have failed");
+ conf.set(MRConfig.FRAMEWORK_NAME, "incorrect");
+ new Cluster(conf);
+ fail("Cluster should not be initialized with incorrect framework name");
} catch (IOException e) {
- errorThrown = true;
- assert(e.getMessage().contains("Cannot initialize Cluster. Please check"));
+ assertTrue(e.getMessage().contains(
+ "Cannot initialize Cluster. Please check"));
}
- assert(errorThrown);
}
}
[45/50] [abbrv] hadoop git commit: HDFS-7804. correct the haadmin
command usage in #HDFSHighAvailabilityWithQJM.html (Brahma Reddy Battula via
umamahesh)
Posted by zj...@apache.org.
HDFS-7804. correct the haadmin command usage in #HDFSHighAvailabilityWithQJM.html (Brahma Reddy Battula via umamahesh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ecea5ab
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ecea5ab
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ecea5ab
Branch: refs/heads/YARN-2928
Commit: 2ecea5ab741f62e8fd0449251f2ea4a5759f4e77
Parents: 3f56a4c
Author: Uma Maheswara Rao G <um...@apache.org>
Authored: Wed Feb 18 19:24:45 2015 +0530
Committer: Uma Maheswara Rao G <um...@apache.org>
Committed: Wed Feb 18 19:24:45 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecea5ab/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 308b61f..ec1c837 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -650,6 +650,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7780. Update use of Iterator to Iterable in DataXceiverServer and
SnapshotDiffInfo. (Ray Chiang via aajisaka)
+ HDFS-7804. correct the haadmin command usage in #HDFSHighAvailabilityWithQJM.html
+ (Brahma Reddy Battula via umamahesh)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ecea5ab/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
index a285fde..63813d9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSHighAvailabilityWithQJM.md
@@ -382,7 +382,7 @@ You can visit each of the NameNodes' web pages separately by browsing to their c
Now that your HA NameNodes are configured and started, you will have access to some additional commands to administer your HA HDFS cluster. Specifically, you should familiarize yourself with all of the subcommands of the "*hdfs haadmin*" command. Running this command without any additional arguments will display the following usage information:
- Usage: DFSHAAdmin [-ns <nameserviceId>]
+ Usage: haadmin
[-transitionToActive <serviceId>]
[-transitionToStandby <serviceId>]
[-failover [--forcefence] [--forceactive] <serviceId> <serviceId>]
[35/50] [abbrv] hadoop git commit: MAPREDUCE-6234. TestHighRamJob
fails due to the change in MAPREDUCE-5785. (Masatake Iwasaki via kasha)
Posted by zj...@apache.org.
MAPREDUCE-6234. TestHighRamJob fails due to the change in MAPREDUCE-5785. (Masatake Iwasaki via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/409113d8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/409113d8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/409113d8
Branch: refs/heads/YARN-2928
Commit: 409113d8f97fcfdb96cb028dbb6a20c9a1df81b0
Parents: 043e44b
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Feb 17 14:38:00 2015 -0800
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Tue Feb 17 14:38:00 2015 -0800
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/mapred/gridmix/TestHighRamJob.java | 11 +++++------
2 files changed, 8 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/409113d8/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index aebc71e..50e067c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -175,6 +175,9 @@ Trunk (Unreleased)
MAPREDUCE-4413. MR lib dir contains jdiff (which is gpl) (Nemon Lou via aw)
+ MAPREDUCE-6234. TestHighRamJob fails due to the change in MAPREDUCE-5785.
+ (Masatake Iwasaki via kasha)
+
BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by
http://git-wip-us.apache.org/repos/asf/hadoop/blob/409113d8/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
index 9cc84ea..179c941 100644
--- a/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
+++ b/hadoop-tools/hadoop-gridmix/src/test/java/org/apache/hadoop/mapred/gridmix/TestHighRamJob.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.mapred.gridmix.DebugJobProducer.MockJob;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.MRConfig;
import org.apache.hadoop.mapreduce.MRJobConfig;
+import org.apache.hadoop.mapreduce.TaskType;
import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.tools.rumen.JobStory;
@@ -92,15 +93,13 @@ public class TestHighRamJob {
GridmixJob job = new DummyGridmixJob(simulatedJobConf, story);
Job simulatedJob = job.getJob();
- Configuration simulatedConf = simulatedJob.getConfiguration();
+ JobConf simulatedConf = (JobConf)simulatedJob.getConfiguration();
// check if the high ram properties are not set
assertEquals(expectedMapMB,
- simulatedConf.getLong(MRJobConfig.MAP_MEMORY_MB,
- MRJobConfig.DEFAULT_MAP_MEMORY_MB));
+ simulatedConf.getMemoryRequired(TaskType.MAP));
assertEquals(expectedReduceMB,
- simulatedConf.getLong(MRJobConfig.REDUCE_MEMORY_MB,
- MRJobConfig.DEFAULT_MAP_MEMORY_MB));
+ simulatedConf.getMemoryRequired(TaskType.REDUCE));
}
/**
@@ -192,4 +191,4 @@ public class TestHighRamJob {
assertNotNull(failed);
assertTrue("Exception expected for exceeding reduce memory limit!", failed);
}
-}
\ No newline at end of file
+}
[34/50] [abbrv] hadoop git commit: HDFS-6662. WebHDFS cannot open a
file if its path contains "%". Contributed by Gerson Carlos.
Posted by zj...@apache.org.
HDFS-6662. WebHDFS cannot open a file if its path contains "%". Contributed by Gerson Carlos.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/043e44bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/043e44bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/043e44bc
Branch: refs/heads/YARN-2928
Commit: 043e44bc36fc7f7c59406d3722b0a93607b6fa49
Parents: 00b8095
Author: Haohui Mai <wh...@apache.org>
Authored: Tue Feb 17 13:04:38 2015 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Tue Feb 17 13:04:38 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../server/datanode/web/webhdfs/ParameterParser.java | 2 +-
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 7 +++++++
.../datanode/web/webhdfs/TestParameterParser.java | 13 +++++++++++++
4 files changed, 24 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/043e44bc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 48eb61c..391005c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -970,6 +970,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7798. Checkpointing failure caused by shared KerberosAuthenticator.
(Chengbing Liu via yliu)
+ HDFS-6662. WebHDFS cannot open a file if its path contains "%".
+ (Gerson Carlos via wheat9)
+
BREAKDOWN OF HDFS-7584 SUBTASKS AND RELATED JIRAS
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode
http://git-wip-us.apache.org/repos/asf/hadoop/blob/043e44bc/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index e1930b0..5749504 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -50,7 +50,7 @@ class ParameterParser {
private final Map<String, List<String>> params;
ParameterParser(QueryStringDecoder decoder, Configuration conf) {
- this.path = decoder.path().substring(WEBHDFS_PREFIX_LENGTH);
+ this.path = QueryStringDecoder.decodeComponent(decoder.path().substring(WEBHDFS_PREFIX_LENGTH));
this.params = decoder.parameters();
this.conf = conf;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/043e44bc/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index ca73506..87d47fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -102,6 +102,13 @@
menus.change();
}
+ function encode_path(abs_path) {
+ abs_path = encodeURIComponent(abs_path);
+ var re = /%2F/g;
+ return abs_path.replace(re, '/');
+ }
+
+ abs_path = encode_path(abs_path);
var url = '/webhdfs/v1' + abs_path + '?op=GET_BLOCK_LOCATIONS';
$.get(url).done(function(data) {
var d = get_response(data, "LocatedBlocks");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/043e44bc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
index 8b4235b..6a6c5d0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/TestParameterParser.java
@@ -52,4 +52,17 @@ public class TestParameterParser {
final Token<DelegationTokenIdentifier> tok2 = testParser.delegationToken();
Assert.assertTrue(HAUtil.isTokenForLogicalUri(tok2));
}
+
+ @Test
+ public void testDecodePath() {
+ final String SCAPED_PATH = "hdfs-6662/test%25251%26%3Dtest?op=OPEN";
+ final String EXPECTED_PATH = "/hdfs-6662/test%251&=test";
+
+ Configuration conf = DFSTestUtil.newHAConfiguration(LOGICAL_NAME);
+ QueryStringDecoder decoder = new QueryStringDecoder(
+ WebHdfsHandler.WEBHDFS_PREFIX + "/"
+ + SCAPED_PATH);
+ ParameterParser testParser = new ParameterParser(decoder, conf);
+ Assert.assertEquals(EXPECTED_PATH, testParser.path());
+ }
}
[30/50] [abbrv] hadoop git commit: HDFS-7795. Show warning if not all
favored nodes were chosen by namenode. Contributed by Kihwal Lee.
Posted by zj...@apache.org.
HDFS-7795. Show warning if not all favored nodes were chosen by namenode. Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/db660622
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/db660622
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/db660622
Branch: refs/heads/YARN-2928
Commit: db6606223ca2e17aa7e1b2e2be13c1a19d8e7465
Parents: 78a7e8d
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Feb 17 13:05:43 2015 -0600
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Feb 17 13:05:43 2015 -0600
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 26 ++++++++++++++------
2 files changed, 21 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db660622/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5e54731..48eb61c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -644,6 +644,9 @@ Release 2.7.0 - UNRELEASED
HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
+ HDFS-7795. Show warning if not all favored nodes were chosen by namenode
+ (kihwal)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/db660622/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
index 7d29b3d..85d3410 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSOutputStream.java
@@ -35,6 +35,7 @@ import java.nio.channels.ClosedChannelException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
+import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.concurrent.TimeUnit;
@@ -1443,7 +1444,7 @@ public class DFSOutputStream extends FSOutputSummer
ExtendedBlock blockCopy = new ExtendedBlock(block);
blockCopy.setNumBytes(blockSize);
- boolean[] targetPinnings = getPinnings(nodes);
+ boolean[] targetPinnings = getPinnings(nodes, true);
// send the request
new Sender(out).writeBlock(blockCopy, nodeStorageTypes[0], accessToken,
dfsClient.clientName, nodes, nodeStorageTypes, null, bcs,
@@ -1537,20 +1538,29 @@ public class DFSOutputStream extends FSOutputSummer
}
}
- private boolean[] getPinnings(DatanodeInfo[] nodes) {
+ private boolean[] getPinnings(DatanodeInfo[] nodes, boolean shouldLog) {
if (favoredNodes == null) {
return null;
} else {
boolean[] pinnings = new boolean[nodes.length];
+ HashSet<String> favoredSet =
+ new HashSet<String>(Arrays.asList(favoredNodes));
for (int i = 0; i < nodes.length; i++) {
- pinnings[i] = false;
- for (int j = 0; j < favoredNodes.length; j++) {
- if (nodes[i].getXferAddrWithHostname().equals(favoredNodes[j])) {
- pinnings[i] = true;
- break;
- }
+ pinnings[i] = favoredSet.remove(nodes[i].getXferAddrWithHostname());
+ if (DFSClient.LOG.isDebugEnabled()) {
+ DFSClient.LOG.debug(nodes[i].getXferAddrWithHostname() +
+ " was chosen by name node (favored=" + pinnings[i] +
+ ").");
}
}
+ if (shouldLog && !favoredSet.isEmpty()) {
+ // There is one or more favored nodes that were not allocated.
+ DFSClient.LOG.warn(
+ "These favored nodes were specified but not chosen: " +
+ favoredSet +
+ " Specified favored nodes: " + Arrays.toString(favoredNodes));
+
+ }
return pinnings;
}
}
[21/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
new file mode 100644
index 0000000..3cfa39d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/PluggableShuffleAndPluggableSort.md
@@ -0,0 +1,73 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop: Pluggable Shuffle and Pluggable Sort
+============================================
+
+Introduction
+------------
+
+The pluggable shuffle and pluggable sort capabilities allow replacing the built in shuffle and sort logic with alternate implementations. Example use cases for this are: using a different application protocol other than HTTP such as RDMA for shuffling data from the Map nodes to the Reducer nodes; or replacing the sort logic with custom algorithms that enable Hash aggregation and Limit-N query.
+
+**IMPORTANT:** The pluggable shuffle and pluggable sort capabilities are experimental and unstable. This means the provided APIs may change and break compatibility in future versions of Hadoop.
+
+Implementing a Custom Shuffle and a Custom Sort
+-----------------------------------------------
+
+A custom shuffle implementation requires a
+`org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices.AuxiliaryService`
+implementation class running in the NodeManagers and a
+`org.apache.hadoop.mapred.ShuffleConsumerPlugin`
+implementation class running in the Reducer tasks.
+
+The default implementations provided by Hadoop can be used as references:
+
+* `org.apache.hadoop.mapred.ShuffleHandler`
+* `org.apache.hadoop.mapreduce.task.reduce.Shuffle`
+
+A custom sort implementation requires a `org.apache.hadoop.mapred.MapOutputCollector` implementation class running in the Mapper tasks and (optionally, depending on the sort implementation) a `org.apache.hadoop.mapred.ShuffleConsumerPlugin` implementation class running in the Reducer tasks.
+
+The default implementations provided by Hadoop can be used as references:
+
+* `org.apache.hadoop.mapred.MapTask$MapOutputBuffer`
+* `org.apache.hadoop.mapreduce.task.reduce.Shuffle`
+
+Configuration
+-------------
+
+Except for the auxiliary service running in the NodeManagers serving the shuffle (by default the `ShuffleHandler`), all the pluggable components run in the job tasks. This means, they can be configured on per job basis. The auxiliary service servicing the Shuffle must be configured in the NodeManagers configuration.
+
+### Job Configuration Properties (on per job basis):
+
+| **Property** | **Default Value** | **Explanation** |
+|:---- |:---- |:---- |
+| `mapreduce.job.reduce.shuffle.consumer.plugin.class` | `org.apache.hadoop.mapreduce.task.reduce.Shuffle` | The `ShuffleConsumerPlugin` implementation to use |
+| `mapreduce.job.map.output.collector.class` | `org.apache.hadoop.mapred.MapTask$MapOutputBuffer` | The `MapOutputCollector` implementation(s) to use |
+
+These properties can also be set in the `mapred-site.xml` to change the default values for all jobs.
+
+The collector class configuration may specify a comma-separated list of collector implementations. In this case, the map task will attempt to instantiate each in turn until one of the implementations successfully initializes. This can be useful if a given collector implementation is only compatible with certain types of keys or values, for example.
+
+### NodeManager Configuration properties, `yarn-site.xml` in all nodes:
+
+| **Property** | **Default Value** | **Explanation** |
+|:---- |:---- |:---- |
+| `yarn.nodemanager.aux-services` | `...,mapreduce_shuffle` | The auxiliary service name |
+| `yarn.nodemanager.aux-services.mapreduce_shuffle.class` | `org.apache.hadoop.mapred.ShuffleHandler` | The auxiliary service class to use |
+
+**IMPORTANT:** If setting an auxiliary service in addition the default
+`mapreduce_shuffle` service, then a new service key should be added to the
+`yarn.nodemanager.aux-services` property, for example `mapred.shufflex`.
+Then the property defining the corresponding class must be
+`yarn.nodemanager.aux-services.mapreduce_shufflex.class`.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/apt/HistoryServerRest.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/apt/HistoryServerRest.apt.vm b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/apt/HistoryServerRest.apt.vm
deleted file mode 100644
index 8dfbad7..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/site/apt/HistoryServerRest.apt.vm
+++ /dev/null
@@ -1,2672 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- MapReduce History Server REST API's.
- ---
- ---
- ${maven.build.timestamp}
-
-MapReduce History Server REST API's.
-
-%{toc|section=1|fromDepth=0|toDepth=3}
-
-* Overview
-
- The history server REST API's allow the user to get status on finished applications.
-
-* History Server Information API
-
- The history server information resource provides overall information about the history server.
-
-** URI
-
- Both of the following URI's give you the history server information, from an application id identified by the appid value.
-
-------
- * http://<history server http address:port>/ws/v1/history
- * http://<history server http address:port>/ws/v1/history/info
-------
-
-** HTTP Operations Supported
-
-------
- * GET
-------
-
-** Query Parameters Supported
-
-------
- None
-------
-
-** Elements of the <historyInfo> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| startedOn | long | The time the history server was started (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| hadoopVersion | string | Version of hadoop common |
-*---------------+--------------+-------------------------------+
-| hadoopBuildVersion | string | Hadoop common build string with build version, user, and checksum |
-*---------------+--------------+-------------------------------+
-| hadoopVersionBuiltOn | string | Timestamp when hadoop common was built |
-*---------------+--------------+-------------------------------+
-
-** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/info
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "historyInfo" : {
- "startedOn":1353512830963,
- "hadoopVersionBuiltOn" : "Wed Jan 11 21:18:36 UTC 2012",
- "hadoopBuildVersion" : "0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7",
- "hadoopVersion" : "0.23.1-SNAPSHOT"
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
------
- GET http://<history server http address:port>/ws/v1/history/info
- Accept: application/xml
------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 330
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<historyInfo>
- <startedOn>1353512830963</startedOn>
- <hadoopVersion>0.23.1-SNAPSHOT</hadoopVersion>
- <hadoopBuildVersion>0.23.1-SNAPSHOT from 1230253 by user1 source checksum bb6e554c6d50b0397d826081017437a7</hadoopBuildVersion>
- <hadoopVersionBuiltOn>Wed Jan 11 21:18:36 UTC 2012</hadoopVersionBuiltOn>
-</historyInfo>
-+---+
-
-* MapReduce API's
-
- The following list of resources apply to MapReduce.
-
-** Jobs API
-
- The jobs resource provides a list of the MapReduce jobs that have finished. It does not currently return a full list of parameters
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
- Multiple paramters can be specified. The started and finished times have a begin and end parameter to allow you to specify ranges. For example, one could request all jobs that started between 1:00am and 2:00pm on 12/19/2011 with startedTimeBegin=1324256400&startedTimeEnd=1324303200. If the Begin parameter is not specfied, it defaults to 0, and if the End parameter is not specified, it defaults to infinity.
-
-------
- * user - user name
- * state - the job state
- * queue - queue name
- * limit - total number of app objects to be returned
- * startedTimeBegin - jobs with start time beginning with this time, specified in ms since epoch
- * startedTimeEnd - jobs with start time ending with this time, specified in ms since epoch
- * finishedTimeBegin - jobs with finish time beginning with this time, specified in ms since epoch
- * finishedTimeEnd - jobs with finish time ending with this time, specified in ms since epoch
-------
-
-*** Elements of the <jobs> object
-
- When you make a request for the list of jobs, the information will be returned as an array of job objects.
- See also {{Job API}} for syntax of the job object. Except this is a subset of a full job. Only startTime,
- finishTime, id, name, queue, user, state, mapsTotal, mapsCompleted, reducesTotal, and reducesCompleted are
- returned.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| job | array of job objects(json)/zero or more job objects(XML) | The collection of job objects |
-*---------------+--------------+-------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobs" : {
- "job" : [
- {
- "submitTime" : 1326381344449,
- "state" : "SUCCEEDED",
- "user" : "user1",
- "reducesTotal" : 1,
- "mapsCompleted" : 1,
- "startTime" : 1326381344489,
- "id" : "job_1326381300833_1_1",
- "name" : "word count",
- "reducesCompleted" : 1,
- "mapsTotal" : 1,
- "queue" : "default",
- "finishTime" : 1326381356010
- },
- {
- "submitTime" : 1326381446500
- "state" : "SUCCEEDED",
- "user" : "user1",
- "reducesTotal" : 1,
- "mapsCompleted" : 1,
- "startTime" : 1326381446529,
- "id" : "job_1326381300833_2_2",
- "name" : "Sleep job",
- "reducesCompleted" : 1,
- "mapsTotal" : 1,
- "queue" : "default",
- "finishTime" : 1326381582106
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 1922
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobs>
- <job>
- <submitTime>1326381344449</submitTime>
- <startTime>1326381344489</startTime>
- <finishTime>1326381356010</finishTime>
- <id>job_1326381300833_1_1</id>
- <name>word count</name>
- <queue>default</queue>
- <user>user1</user>
- <state>SUCCEEDED</state>
- <mapsTotal>1</mapsTotal>
- <mapsCompleted>1</mapsCompleted>
- <reducesTotal>1</reducesTotal>
- <reducesCompleted>1</reducesCompleted>
- </job>
- <job>
- <submitTime>1326381446500</submitTime>
- <startTime>1326381446529</startTime>
- <finishTime>1326381582106</finishTime>
- <id>job_1326381300833_2_2</id>
- <name>Sleep job</name>
- <queue>default</queue>
- <user>user1</user>
- <state>SUCCEEDED</state>
- <mapsTotal>1</mapsTotal>
- <mapsCompleted>1</mapsCompleted>
- <reducesTotal>1</reducesTotal>
- <reducesCompleted>1</reducesCompleted>
- </job>
-</jobs>
-+---+
-
-** {Job API}
-
- A Job resource contains information about a particular job identified by {jobid}.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <job> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The job id|
-*---------------+--------------+-------------------------------+
-| name | string | The job name |
-*---------------+--------------+-------------------------------+
-| queue | string | The queue the job was submitted to|
-*---------------+--------------+-------------------------------+
-| user | string | The user name |
-*---------------+--------------+-------------------------------+
-| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED, ERROR|
-*---------------+--------------+-------------------------------+
-| diagnostics | string | A diagnostic message |
-*---------------+--------------+-------------------------------+
-| submitTime | long | The time the job submitted (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| startTime | long | The time the job started (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| finishTime | long | The time the job finished (in ms since epoch)|
-*---------------+--------------+-------------------------------+
-| mapsTotal | int | The total number of maps |
-*---------------+--------------+-------------------------------+
-| mapsCompleted | int | The number of completed maps |
-*---------------+--------------+-------------------------------+
-| reducesTotal | int | The total number of reduces |
-*---------------+--------------+-------------------------------+
-| reducesCompleted | int | The number of completed reduces|
-*---------------+--------------+-------------------------------+
-| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master|
-*---------------+--------------+-------------------------------+
-| avgMapTime | long | The average time of a map task (in ms)|
-*---------------+--------------+-------------------------------+
-| avgReduceTime | long | The average time of the reduce (in ms)|
-*---------------+--------------+-------------------------------+
-| avgShuffleTime | long | The average time of the shuffle (in ms)|
-*---------------+--------------+-------------------------------+
-| avgMergeTime | long | The average time of the merge (in ms)|
-*---------------+--------------+-------------------------------+
-| failedReduceAttempts | int | The number of failed reduce attempts |
-*---------------+--------------+-------------------------------+
-| killedReduceAttempts | int | The number of killed reduce attempts |
-*---------------+--------------+-------------------------------+
-| successfulReduceAttempts | int | The number of successful reduce attempts |
-*---------------+--------------+-------------------------------+
-| failedMapAttempts | int | The number of failed map attempts |
-*---------------+--------------+-------------------------------+
-| killedMapAttempts | int | The number of killed map attempts |
-*---------------+--------------+-------------------------------+
-| successfulMapAttempts | int | The number of successful map attempts |
-*---------------+--------------+-------------------------------+
-| acls | array of acls(json)/zero or more acls objects(xml)| A collection of acls objects |
-*---------------+--------------+-------------------------------+
-
-** Elements of the <acls> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| value | string | The acl value|
-*---------------+--------------+-------------------------------+
-| name | string | The acl name |
-*---------------+--------------+-------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Server: Jetty(6.1.26)
- Content-Length: 720
-+---+
-
- Response Body:
-
-+---+
-{
- "job" : {
- "submitTime": 1326381446500,
- "avgReduceTime" : 124961,
- "failedReduceAttempts" : 0,
- "state" : "SUCCEEDED",
- "successfulReduceAttempts" : 1,
- "acls" : [
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-modify-job"
- },
- {
- "value" : " ",
- "name" : "mapreduce.job.acl-view-job"
- }
- ],
- "user" : "user1",
- "reducesTotal" : 1,
- "mapsCompleted" : 1,
- "startTime" : 1326381446529,
- "id" : "job_1326381300833_2_2",
- "avgMapTime" : 2638,
- "successfulMapAttempts" : 1,
- "name" : "Sleep job",
- "avgShuffleTime" : 2540,
- "reducesCompleted" : 1,
- "diagnostics" : "",
- "failedMapAttempts" : 0,
- "avgMergeTime" : 2589,
- "killedReduceAttempts" : 0,
- "mapsTotal" : 1,
- "queue" : "default",
- "uberized" : false,
- "killedMapAttempts" : 0,
- "finishTime" : 1326381582106
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 983
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<job>
- <submitTime>1326381446500</submitTime>
- <startTime>1326381446529</startTime>
- <finishTime>1326381582106</finishTime>
- <id>job_1326381300833_2_2</id>
- <name>Sleep job</name>
- <queue>default</queue>
- <user>user1</user>
- <state>SUCCEEDED</state>
- <mapsTotal>1</mapsTotal>
- <mapsCompleted>1</mapsCompleted>
- <reducesTotal>1</reducesTotal>
- <reducesCompleted>1</reducesCompleted>
- <uberized>false</uberized>
- <diagnostics/>
- <avgMapTime>2638</avgMapTime>
- <avgReduceTime>124961</avgReduceTime>
- <avgShuffleTime>2540</avgShuffleTime>
- <avgMergeTime>2589</avgMergeTime>
- <failedReduceAttempts>0</failedReduceAttempts>
- <killedReduceAttempts>0</killedReduceAttempts>
- <successfulReduceAttempts>1</successfulReduceAttempts>
- <failedMapAttempts>0</failedMapAttempts>
- <killedMapAttempts>0</killedMapAttempts>
- <successfulMapAttempts>1</successfulMapAttempts>
- <acls>
- <name>mapreduce.job.acl-modify-job</name>
- <value> </value>
- </acls>
- <acls>
- <name>mapreduce.job.acl-view-job</name>
- <value> </value>
- </acls>
-</job>
-+---+
-
-** Job Attempts API
-
- With the job attempts API, you can obtain a collection of resources that represent a job attempt. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/jobattempts
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <jobAttempts> object
-
- When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects.
-
- jobAttempts:
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects |
-*---------------+--------------+--------------------------------+
-
-*** Elements of the <jobAttempt> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The job attempt id |
-*---------------+--------------+--------------------------------+
-| nodeId | string | The node id of the node the attempt ran on|
-*---------------+--------------+--------------------------------+
-| nodeHttpAddress | string | The node http address of the node the attempt ran on|
-*---------------+--------------+--------------------------------+
-| logsLink | string | The http link to the job attempt logs |
-*---------------+--------------+--------------------------------+
-| containerId | string | The id of the container for the job attempt |
-*---------------+--------------+--------------------------------+
-| startTime | long | The start time of the attempt (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattempts
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobAttempts" : {
- "jobAttempt" : [
- {
- "nodeId" : "host.domain.com:8041",
- "nodeHttpAddress" : "host.domain.com:8042",
- "startTime" : 1326381444693,
- "id" : 1,
- "logsLink" : "http://host.domain.com:19888/jobhistory/logs/host.domain.com:8041/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1",
- "containerId" : "container_1326381300833_0002_01_000001"
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/jobattmpts
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 575
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobAttempts>
- <jobAttempt>
- <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
- <nodeId>host.domain.com:8041</nodeId>
- <id>1</id>
- <startTime>1326381444693</startTime>
- <containerId>container_1326381300833_0002_01_000001</containerId>
- <logsLink>http://host.domain.com:19888/jobhistory/logs/host.domain.com:8041/container_1326381300833_0002_01_000001/job_1326381300833_2_2/user1</logsLink>
- </jobAttempt>
-</jobAttempts>
-+---+
-
-** Job Counters API
-
- With the job counters API, you can object a collection of resources that represent al the counters for that job.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/counters
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <jobCounters> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The job id |
-*---------------+--------------+-------------------------------+
-| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <counterGroup> objecs
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| counterGroupName | string | The name of the counter group |
-*---------------+--------------+-------------------------------+
-| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <counter> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the counter |
-*---------------+--------------+-------------------------------+
-| reduceCounterValue | long | The counter value of reduce tasks |
-*---------------+--------------+-------------------------------+
-| mapCounterValue | long | The counter value of map tasks |
-*---------------+--------------+-------------------------------+
-| totalCounterValue | long | The counter value of all tasks |
-*---------------+--------------+-------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobCounters" : {
- "id" : "job_1326381300833_2_2",
- "counterGroup" : [
- {
- "counterGroupName" : "Shuffle Errors",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "BAD_ID"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "CONNECTION"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "IO_ERROR"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "WRONG_LENGTH"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "WRONG_MAP"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "WRONG_REDUCE"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2483,
- "name" : "FILE_BYTES_READ"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 108525,
- "name" : "FILE_BYTES_WRITTEN"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FILE_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FILE_LARGE_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FILE_WRITE_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 48,
- "name" : "HDFS_BYTES_READ"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "HDFS_BYTES_WRITTEN"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "HDFS_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "HDFS_LARGE_READ_OPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "HDFS_WRITE_OPS"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "MAP_INPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1200,
- "name" : "MAP_OUTPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 4800,
- "name" : "MAP_OUTPUT_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2235,
- "name" : "MAP_OUTPUT_MATERIALIZED_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 48,
- "name" : "SPLIT_RAW_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "COMBINE_INPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "COMBINE_OUTPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1200,
- "name" : "REDUCE_INPUT_GROUPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2235,
- "name" : "REDUCE_SHUFFLE_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1200,
- "name" : "REDUCE_INPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "REDUCE_OUTPUT_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2400,
- "name" : "SPILLED_RECORDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "SHUFFLED_MAPS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "FAILED_SHUFFLE"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1,
- "name" : "MERGED_MAP_OUTPUTS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 113,
- "name" : "GC_TIME_MILLIS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 1830,
- "name" : "CPU_MILLISECONDS"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 478068736,
- "name" : "PHYSICAL_MEMORY_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 2159284224,
- "name" : "VIRTUAL_MEMORY_BYTES"
- },
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 378863616,
- "name" : "COMMITTED_HEAP_BYTES"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "BYTES_READ"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counter" : [
- {
- "reduceCounterValue" : 0,
- "mapCounterValue" : 0,
- "totalCounterValue" : 0,
- "name" : "BYTES_WRITTEN"
- }
- ]
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/counters
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 7030
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobCounters>
- <id>job_1326381300833_2_2</id>
- <counterGroup>
- <counterGroupName>Shuffle Errors</counterGroupName>
- <counter>
- <name>BAD_ID</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>CONNECTION</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>IO_ERROR</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>WRONG_LENGTH</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>WRONG_MAP</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>WRONG_REDUCE</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
- <counter>
- <name>FILE_BYTES_READ</name>
- <totalCounterValue>2483</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_BYTES_WRITTEN</name>
- <totalCounterValue>108525</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_READ_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_LARGE_READ_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FILE_WRITE_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_BYTES_READ</name>
- <totalCounterValue>48</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_BYTES_WRITTEN</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_READ_OPS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_LARGE_READ_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>HDFS_WRITE_OPS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
- <counter>
- <name>MAP_INPUT_RECORDS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MAP_OUTPUT_RECORDS</name>
- <totalCounterValue>1200</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MAP_OUTPUT_BYTES</name>
- <totalCounterValue>4800</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MAP_OUTPUT_MATERIALIZED_BYTES</name>
- <totalCounterValue>2235</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>SPLIT_RAW_BYTES</name>
- <totalCounterValue>48</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>COMBINE_INPUT_RECORDS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>COMBINE_OUTPUT_RECORDS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_INPUT_GROUPS</name>
- <totalCounterValue>1200</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_SHUFFLE_BYTES</name>
- <totalCounterValue>2235</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_INPUT_RECORDS</name>
- <totalCounterValue>1200</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>REDUCE_OUTPUT_RECORDS</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>SPILLED_RECORDS</name>
- <totalCounterValue>2400</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>SHUFFLED_MAPS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>FAILED_SHUFFLE</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>MERGED_MAP_OUTPUTS</name>
- <totalCounterValue>1</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>GC_TIME_MILLIS</name>
- <totalCounterValue>113</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>CPU_MILLISECONDS</name>
- <totalCounterValue>1830</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>PHYSICAL_MEMORY_BYTES</name>
- <totalCounterValue>478068736</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>VIRTUAL_MEMORY_BYTES</name>
- <totalCounterValue>2159284224</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- <counter>
- <name>COMMITTED_HEAP_BYTES</name>
- <totalCounterValue>378863616</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_READ</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
- <counterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_WRITTEN</name>
- <totalCounterValue>0</totalCounterValue>
- <mapCounterValue>0</mapCounterValue>
- <reduceCounterValue>0</reduceCounterValue>
- </counter>
- </counterGroup>
-</jobCounters>
-+---+
-
-
-** Job Conf API
-
- A job configuration resource contains information about the job configuration for this job.
-
-*** URI
-
- Use the following URI to obtain th job configuration information, from a job identified by the {jobid} value.
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/conf
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <conf> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| path | string | The path to the job configuration file|
-*---------------+--------------+-------------------------------+
-| property | array of the configuration properties(JSON)/zero or more configuration properties(XML) | Collection of configuration property objects|
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <property> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the configuration property |
-*---------------+--------------+-------------------------------+
-| value | string | The value of the configuration property |
-*---------------+--------------+-------------------------------+
-| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. |
-*---------------+--------------+-------------------------------+
-
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
- This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file.
-
-+---+
-{
- "conf" : {
- "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml",
- "property" : [
- {
- "value" : "/home/hadoop/hdfs/data",
- "name" : "dfs.datanode.data.dir"
- "source" : ["hdfs-site.xml", "job.xml"]
- },
- {
- "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
- "name" : "hadoop.http.filter.initializers"
- "source" : ["programmatically", "job.xml"]
- },
- {
- "value" : "/home/hadoop/tmp",
- "name" : "mapreduce.cluster.temp.dir"
- "source" : ["mapred-site.xml"]
- },
- ...
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/conf
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 552
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<conf>
- <path>hdfs://host.domain.com:9000/user/user1/.staging/job_1326381300833_0002/job.xml</path>
- <property>
- <name>dfs.datanode.data.dir</name>
- <value>/home/hadoop/hdfs/data</value>
- <source>hdfs-site.xml</source>
- <source>job.xml</source>
- </property>
- <property>
- <name>hadoop.http.filter.initializers</name>
- <value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
- <source>programmatically</source>
- <source>job.xml</source>
- </property>
- <property>
- <name>mapreduce.cluster.temp.dir</name>
- <value>/home/hadoop/tmp</value>
- <source>mapred-site.xml</source>
- </property>
- ...
-</conf>
-+---+
-
-** Tasks API
-
- With the tasks API, you can obtain a collection of resources that represent a task within a job. When you run a GET operation on this resource, you obtain a collection of Task Objects.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- * type - type of task, valid values are m or r. m for map task or r for reduce task.
-------
-
-*** Elements of the <tasks> object
-
- When you make a request for the list of tasks , the information will be returned as an array of task objects.
- See also {{Task API}} for syntax of the task object.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects. |
-*---------------+--------------+--------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "tasks" : {
- "task" : [
- {
- "progress" : 100,
- "elapsedTime" : 6777,
- "state" : "SUCCEEDED",
- "startTime" : 1326381446541,
- "id" : "task_1326381300833_2_2_m_0",
- "type" : "MAP",
- "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0",
- "finishTime" : 1326381453318
- },
- {
- "progress" : 100,
- "elapsedTime" : 135559,
- "state" : "SUCCEEDED",
- "startTime" : 1326381446544,
- "id" : "task_1326381300833_2_2_r_0",
- "type" : "REDUCE",
- "successfulAttempt" : "attempt_1326381300833_2_2_r_0_0",
- "finishTime" : 1326381582103
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 653
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<tasks>
- <task>
- <startTime>1326381446541</startTime>
- <finishTime>1326381453318</finishTime>
- <elapsedTime>6777</elapsedTime>
- <progress>100.0</progress>
- <id>task_1326381300833_2_2_m_0</id>
- <state>SUCCEEDED</state>
- <type>MAP</type>
- <successfulAttempt>attempt_1326381300833_2_2_m_0_0</successfulAttempt>
- </task>
- <task>
- <startTime>1326381446544</startTime>
- <finishTime>1326381582103</finishTime>
- <elapsedTime>135559</elapsedTime>
- <progress>100.0</progress>
- <id>task_1326381300833_2_2_r_0</id>
- <state>SUCCEEDED</state>
- <type>REDUCE</type>
- <successfulAttempt>attempt_1326381300833_2_2_r_0_0</successfulAttempt>
- </task>
-</tasks>
-+---+
-
-** {Task API}
-
- A Task resource contains information about a particular task within a job.
-
-*** URI
-
- Use the following URI to obtain an Task Object, from a task identified by the {taskid} value.
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <task> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task id |
-*---------------+--------------+--------------------------------+
-| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL_WAIT, KILLED
-*---------------+--------------+--------------------------------+
-| type | string | The task type - MAP or REDUCE|
-*---------------+--------------+--------------------------------+
-| successfulAttempt | string | The id of the last successful attempt |
-*---------------+--------------+--------------------------------+
-| progress | float | The progress of the task as a percent|
-*---------------+--------------+--------------------------------+
-| startTime | long | The time in which the task started (in ms since epoch) or -1 if it was never started |
-*---------------+--------------+--------------------------------+
-| finishTime | long | The time in which the task finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| elapsedTime | long | The elapsed time since the application started (in ms)|
-*---------------+--------------+--------------------------------+
-
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "task" : {
- "progress" : 100,
- "elapsedTime" : 6777,
- "state" : "SUCCEEDED",
- "startTime" : 1326381446541,
- "id" : "task_1326381300833_2_2_m_0",
- "type" : "MAP",
- "successfulAttempt" : "attempt_1326381300833_2_2_m_0_0",
- "finishTime" : 1326381453318
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 299
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<task>
- <startTime>1326381446541</startTime>
- <finishTime>1326381453318</finishTime>
- <elapsedTime>6777</elapsedTime>
- <progress>100.0</progress>
- <id>task_1326381300833_2_2_m_0</id>
- <state>SUCCEEDED</state>
- <type>MAP</type>
- <successfulAttempt>attempt_1326381300833_2_2_m_0_0</successfulAttempt>
-</task>
-+---+
-
-** Task Counters API
-
- With the task counters API, you can object a collection of resources that represent all the counters for that task.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <jobTaskCounters> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task id |
-*---------------+--------------+-------------------------------+
-| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <counterGroup> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| counterGroupName | string | The name of the counter group |
-*---------------+--------------+-------------------------------+
-| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <counter> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the counter |
-*---------------+--------------+-------------------------------+
-| value | long | The value of the counter |
-*---------------+--------------+-------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobTaskCounters" : {
- "id" : "task_1326381300833_2_2_m_0",
- "taskCounterGroup" : [
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
- "counter" : [
- {
- "value" : 2363,
- "name" : "FILE_BYTES_READ"
- },
- {
- "value" : 54372,
- "name" : "FILE_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "FILE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_WRITE_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_READ"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "HDFS_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_WRITE_OPS"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "COMBINE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "COMBINE_OUTPUT_RECORDS"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_GROUPS"
- },
- {
- "value" : 2235,
- "name" : "REDUCE_SHUFFLE_BYTES"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "REDUCE_OUTPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "SPILLED_RECORDS"
- },
- {
- "value" : 1,
- "name" : "SHUFFLED_MAPS"
- },
- {
- "value" : 0,
- "name" : "FAILED_SHUFFLE"
- },
- {
- "value" : 1,
- "name" : "MERGED_MAP_OUTPUTS"
- },
- {
- "value" : 26,
- "name" : "GC_TIME_MILLIS"
- },
- {
- "value" : 860,
- "name" : "CPU_MILLISECONDS"
- },
- {
- "value" : 107839488,
- "name" : "PHYSICAL_MEMORY_BYTES"
- },
- {
- "value" : 1123147776,
- "name" : "VIRTUAL_MEMORY_BYTES"
- },
- {
- "value" : 57475072,
- "name" : "COMMITTED_HEAP_BYTES"
- }
- ]
- },
- {
- "counterGroupName" : "Shuffle Errors",
- "counter" : [
- {
- "value" : 0,
- "name" : "BAD_ID"
- },
- {
- "value" : 0,
- "name" : "CONNECTION"
- },
- {
- "value" : 0,
- "name" : "IO_ERROR"
- },
- {
- "value" : 0,
- "name" : "WRONG_LENGTH"
- },
- {
- "value" : 0,
- "name" : "WRONG_MAP"
- },
- {
- "value" : 0,
- "name" : "WRONG_REDUCE"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "BYTES_WRITTEN"
- }
- ]
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/counters
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 2660
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobTaskCounters>
- <id>task_1326381300833_2_2_m_0</id>
- <taskCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
- <counter>
- <name>FILE_BYTES_READ</name>
- <value>2363</value>
- </counter>
- <counter>
- <name>FILE_BYTES_WRITTEN</name>
- <value>54372</value>
- </counter>
- <counter>
- <name>FILE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_WRITE_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_READ</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_WRITE_OPS</name>
- <value>0</value>
- </counter>
- </taskCounterGroup>
- <taskCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
- <counter>
- <name>COMBINE_INPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>COMBINE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_GROUPS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_SHUFFLE_BYTES</name>
- <value>2235</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_RECORDS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SPILLED_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SHUFFLED_MAPS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>FAILED_SHUFFLE</name>
- <value>0</value>
- </counter>
- <counter>
- <name>MERGED_MAP_OUTPUTS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>GC_TIME_MILLIS</name>
- <value>26</value>
- </counter>
- <counter>
- <name>CPU_MILLISECONDS</name>
- <value>860</value>
- </counter>
- <counter>
- <name>PHYSICAL_MEMORY_BYTES</name>
- <value>107839488</value>
- </counter>
- <counter>
- <name>VIRTUAL_MEMORY_BYTES</name>
- <value>1123147776</value>
- </counter>
- <counter>
- <name>COMMITTED_HEAP_BYTES</name>
- <value>57475072</value>
- </counter>
- </taskCounterGroup>
- <taskCounterGroup>
- <counterGroupName>Shuffle Errors</counterGroupName>
- <counter>
- <name>BAD_ID</name>
- <value>0</value>
- </counter>
- <counter>
- <name>CONNECTION</name>
- <value>0</value>
- </counter>
- <counter>
- <name>IO_ERROR</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_LENGTH</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_MAP</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_REDUCE</name>
- <value>0</value>
- </counter>
- </taskCounterGroup>
- <taskCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- </taskCounterGroup>
-</jobTaskCounters>
-+---+
-
-** Task Attempts API
-
- With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <taskAttempts> object
-
- When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects.
- See also {{Task Attempt API}} for syntax of the task object.
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects |
-*---------------+--------------+--------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "taskAttempts" : {
- "taskAttempt" : [
- {
- "assignedContainerId" : "container_1326381300833_0002_01_000002",
- "progress" : 100,
- "elapsedTime" : 2638,
- "state" : "SUCCEEDED",
- "diagnostics" : "",
- "rack" : "/98.139.92.0",
- "nodeHttpAddress" : "host.domain.com:8042",
- "startTime" : 1326381450680,
- "id" : "attempt_1326381300833_2_2_m_0_0",
- "type" : "MAP",
- "finishTime" : 1326381453318
- }
- ]
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 537
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<taskAttempts>
- <taskAttempt>
- <startTime>1326381450680</startTime>
- <finishTime>1326381453318</finishTime>
- <elapsedTime>2638</elapsedTime>
- <progress>100.0</progress>
- <id>attempt_1326381300833_2_2_m_0_0</id>
- <rack>/98.139.92.0</rack>
- <state>SUCCEEDED</state>
- <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
- <diagnostics/>
- <type>MAP</type>
- <assignedContainerId>container_1326381300833_0002_01_000002</assignedContainerId>
- </taskAttempt>
-</taskAttempts>
-+---+
-
-** {Task Attempt API}
-
- A Task Attempt resource contains information about a particular task attempt within a job.
-
-*** URI
-
- Use the following URI to obtain an Task Attempt Object, from a task identified by the {attemptid} value.
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <taskAttempt> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task id |
-*---------------+--------------+--------------------------------+
-| rack | string | The rack |
-*---------------+--------------+--------------------------------+
-| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT_PENDING, SUCCESS_CONTAINER_CLEANUP, SUCCEEDED, FAIL_CONTAINER_CLEANUP, FAIL_TASK_CLEANUP, FAILED, KILL_CONTAINER_CLEANUP, KILL_TASK_CLEANUP, KILLED |
-*---------------+--------------+--------------------------------+
-| type | string | The type of task |
-*---------------+--------------+--------------------------------+
-| assignedContainerId | string | The container id this attempt is assigned to|
-*---------------+--------------+--------------------------------+
-| nodeHttpAddress | string | The http address of the node this task attempt ran on |
-*---------------+--------------+--------------------------------+
-| diagnostics| string | A diagnostics message |
-*---------------+--------------+--------------------------------+
-| progress | float | The progress of the task attempt as a percent|
-*---------------+--------------+--------------------------------+
-| startTime | long | The time in which the task attempt started (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| finishTime | long | The time in which the task attempt finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| elapsedTime | long | The elapsed time since the task attempt started (in ms)|
-*---------------+--------------+--------------------------------+
-
- For reduce task attempts you also have the following fields:
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| mergeFinishTime | long | The time at which merge finished (in ms since epoch)|
-*---------------+--------------+--------------------------------+
-| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish)|
-*---------------+--------------+--------------------------------+
-| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish)|
-*---------------+--------------+--------------------------------+
-| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task)|
-*---------------+--------------+--------------------------------+
-
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "taskAttempt" : {
- "assignedContainerId" : "container_1326381300833_0002_01_000002",
- "progress" : 100,
- "elapsedTime" : 2638,
- "state" : "SUCCEEDED",
- "diagnostics" : "",
- "rack" : "/98.139.92.0",
- "nodeHttpAddress" : "host.domain.com:8042",
- "startTime" : 1326381450680,
- "id" : "attempt_1326381300833_2_2_m_0_0",
- "type" : "MAP",
- "finishTime" : 1326381453318
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 691
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<taskAttempt>
- <startTime>1326381450680</startTime>
- <finishTime>1326381453318</finishTime>
- <elapsedTime>2638</elapsedTime>
- <progress>100.0</progress>
- <id>attempt_1326381300833_2_2_m_0_0</id>
- <rack>/98.139.92.0</rack>
- <state>SUCCEEDED</state>
- <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
- <diagnostics/>
- <type>MAP</type>
- <assignedContainerId>container_1326381300833_0002_01_000002</assignedContainerId>
-</taskAttempt>
-+---+
-
-** Task Attempt Counters API
-
- With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt.
-
-*** URI
-
-------
- * http://<history server http address:port>/ws/v1/history/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters
-------
-
-*** HTTP Operations Supported
-
-------
- * GET
-------
-
-*** Query Parameters Supported
-
-------
- None
-------
-
-*** Elements of the <jobTaskAttemptCounters> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| id | string | The task attempt id |
-*---------------+--------------+-------------------------------+
-| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects |
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <taskAttemptCounterGroup> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| counterGroupName | string | The name of the counter group |
-*---------------+--------------+-------------------------------+
-| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
-*---------------+--------------+-------------------------------+
-
-*** Elements of the <counter> object
-
-*---------------+--------------+-------------------------------+
-|| Item || Data Type || Description |
-*---------------+--------------+-------------------------------+
-| name | string | The name of the counter |
-*---------------+--------------+-------------------------------+
-| value | long | The value of the counter |
-*---------------+--------------+-------------------------------+
-
-*** Response Examples
-
- <<JSON response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/json
- Transfer-Encoding: chunked
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-{
- "jobTaskAttemptCounters" : {
- "taskAttemptCounterGroup" : [
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
- "counter" : [
- {
- "value" : 2363,
- "name" : "FILE_BYTES_READ"
- },
- {
- "value" : 54372,
- "name" : "FILE_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "FILE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "FILE_WRITE_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_READ"
- },
- {
- "value" : 0,
- "name" : "HDFS_BYTES_WRITTEN"
- },
- {
- "value" : 0,
- "name" : "HDFS_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_LARGE_READ_OPS"
- },
- {
- "value" : 0,
- "name" : "HDFS_WRITE_OPS"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "COMBINE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "COMBINE_OUTPUT_RECORDS"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_GROUPS"
- },
- {
- "value" : 2235,
- "name" : "REDUCE_SHUFFLE_BYTES"
- },
- {
- "value" : 460,
- "name" : "REDUCE_INPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "REDUCE_OUTPUT_RECORDS"
- },
- {
- "value" : 0,
- "name" : "SPILLED_RECORDS"
- },
- {
- "value" : 1,
- "name" : "SHUFFLED_MAPS"
- },
- {
- "value" : 0,
- "name" : "FAILED_SHUFFLE"
- },
- {
- "value" : 1,
- "name" : "MERGED_MAP_OUTPUTS"
- },
- {
- "value" : 26,
- "name" : "GC_TIME_MILLIS"
- },
- {
- "value" : 860,
- "name" : "CPU_MILLISECONDS"
- },
- {
- "value" : 107839488,
- "name" : "PHYSICAL_MEMORY_BYTES"
- },
- {
- "value" : 1123147776,
- "name" : "VIRTUAL_MEMORY_BYTES"
- },
- {
- "value" : 57475072,
- "name" : "COMMITTED_HEAP_BYTES"
- }
- ]
- },
- {
- "counterGroupName" : "Shuffle Errors",
- "counter" : [
- {
- "value" : 0,
- "name" : "BAD_ID"
- },
- {
- "value" : 0,
- "name" : "CONNECTION"
- },
- {
- "value" : 0,
- "name" : "IO_ERROR"
- },
- {
- "value" : 0,
- "name" : "WRONG_LENGTH"
- },
- {
- "value" : 0,
- "name" : "WRONG_MAP"
- },
- {
- "value" : 0,
- "name" : "WRONG_REDUCE"
- }
- ]
- },
- {
- "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
- "counter" : [
- {
- "value" : 0,
- "name" : "BYTES_WRITTEN"
- }
- ]
- }
- ],
- "id" : "attempt_1326381300833_2_2_m_0_0"
- }
-}
-+---+
-
- <<XML response>>
-
- HTTP Request:
-
-------
- GET http://<history server http address:port>/ws/v1/history/mapreduce/jobs/job_1326381300833_2_2/tasks/task_1326381300833_2_2_m_0/attempts/attempt_1326381300833_2_2_m_0_0/counters
- Accept: application/xml
-------
-
- Response Header:
-
-+---+
- HTTP/1.1 200 OK
- Content-Type: application/xml
- Content-Length: 2735
- Server: Jetty(6.1.26)
-+---+
-
- Response Body:
-
-+---+
-<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<jobTaskAttemptCounters>
- <id>attempt_1326381300833_2_2_m_0_0</id>
- <taskAttemptCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
- <counter>
- <name>FILE_BYTES_READ</name>
- <value>2363</value>
- </counter>
- <counter>
- <name>FILE_BYTES_WRITTEN</name>
- <value>54372</value>
- </counter>
- <counter>
- <name>FILE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>FILE_WRITE_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_READ</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_LARGE_READ_OPS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>HDFS_WRITE_OPS</name>
- <value>0</value>
- </counter>
- </taskAttemptCounterGroup>
- <taskAttemptCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
- <counter>
- <name>COMBINE_INPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>COMBINE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_GROUPS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_SHUFFLE_BYTES</name>
- <value>2235</value>
- </counter>
- <counter>
- <name>REDUCE_INPUT_RECORDS</name>
- <value>460</value>
- </counter>
- <counter>
- <name>REDUCE_OUTPUT_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SPILLED_RECORDS</name>
- <value>0</value>
- </counter>
- <counter>
- <name>SHUFFLED_MAPS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>FAILED_SHUFFLE</name>
- <value>0</value>
- </counter>
- <counter>
- <name>MERGED_MAP_OUTPUTS</name>
- <value>1</value>
- </counter>
- <counter>
- <name>GC_TIME_MILLIS</name>
- <value>26</value>
- </counter>
- <counter>
- <name>CPU_MILLISECONDS</name>
- <value>860</value>
- </counter>
- <counter>
- <name>PHYSICAL_MEMORY_BYTES</name>
- <value>107839488</value>
- </counter>
- <counter>
- <name>VIRTUAL_MEMORY_BYTES</name>
- <value>1123147776</value>
- </counter>
- <counter>
- <name>COMMITTED_HEAP_BYTES</name>
- <value>57475072</value>
- </counter>
- </taskAttemptCounterGroup>
- <taskAttemptCounterGroup>
- <counterGroupName>Shuffle Errors</counterGroupName>
- <counter>
- <name>BAD_ID</name>
- <value>0</value>
- </counter>
- <counter>
- <name>CONNECTION</name>
- <value>0</value>
- </counter>
- <counter>
- <name>IO_ERROR</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_LENGTH</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_MAP</name>
- <value>0</value>
- </counter>
- <counter>
- <name>WRONG_REDUCE</name>
- <value>0</value>
- </counter>
- </taskAttemptCounterGroup>
- <taskAttemptCounterGroup>
- <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
- <counter>
- <name>BYTES_WRITTEN</name>
- <value>0</value>
- </counter>
- </taskAttemptCounterGroup>
-</jobTaskAttemptCounters>
-+---+
[44/50] [abbrv] hadoop git commit: HADOOP-11599. Client#getTimeout
should use IPC_CLIENT_PING_DEFAULT when IPC_CLIENT_PING_KEY is not
configured. Contributed by zhihai xu.
Posted by zj...@apache.org.
HADOOP-11599. Client#getTimeout should use IPC_CLIENT_PING_DEFAULT when IPC_CLIENT_PING_KEY is not configured. Contributed by zhihai xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f56a4cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f56a4cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f56a4cb
Branch: refs/heads/YARN-2928
Commit: 3f56a4cb0c57583e285e85a4d0c1584c4de9f1f1
Parents: b6fc1f3
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Wed Feb 18 17:32:50 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Wed Feb 18 17:35:11 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/ipc/Client.java | 3 ++-
.../src/test/java/org/apache/hadoop/ipc/TestIPC.java | 6 ++++++
3 files changed, 11 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f56a4cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b3b2c95..f248555 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -959,6 +959,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11295. RPC Server Reader thread can't shutdown if RPCCallQueue is
full. (Ming Ma via kihwal)
+ HADOOP-11599. Client#getTimeout should use IPC_CLIENT_PING_DEFAULT when
+ IPC_CLIENT_PING_KEY is not configured. (zhihai xu via ozawa)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f56a4cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index bdcb96c..3f93c42 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -236,7 +236,8 @@ public class Client {
* @return the timeout period in milliseconds. -1 if no timeout value is set
*/
final public static int getTimeout(Configuration conf) {
- if (!conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true)) {
+ if (!conf.getBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_PING_DEFAULT)) {
return getPingInterval(conf);
}
return -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f56a4cb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
index 04a7412..eb19f48 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestIPC.java
@@ -1235,6 +1235,12 @@ public class TestIPC {
}
}
+ @Test
+ public void testClientGetTimeout() throws IOException {
+ Configuration config = new Configuration();
+ assertEquals(Client.getTimeout(config), -1);
+ }
+
private void assertRetriesOnSocketTimeouts(Configuration conf,
int maxTimeoutRetries) throws IOException {
SocketFactory mockFactory = Mockito.mock(SocketFactory.class);
[50/50] [abbrv] hadoop git commit: Merge remote-tracking branch
'apache/trunk' into YARN-2928
Posted by zj...@apache.org.
Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a3d0af5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a3d0af5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a3d0af5
Branch: refs/heads/YARN-2928
Commit: 5a3d0af568c4410428e5d6022118e9b1e1475de5
Parents: e278615 2aa9979
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Feb 18 13:18:29 2015 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Wed Feb 18 13:18:29 2015 -0800
----------------------------------------------------------------------
dev-support/smart-apply-patch.sh | 4 +-
.../client/KerberosAuthenticator.java | 26 +-
.../server/AuthenticationToken.java | 162 +-
.../security/authentication/util/AuthToken.java | 218 ++
.../hadoop-auth/src/site/apt/BuildingIt.apt.vm | 70 -
.../src/site/apt/Configuration.apt.vm | 377 ---
.../hadoop-auth/src/site/apt/Examples.apt.vm | 133 -
.../hadoop-auth/src/site/apt/index.apt.vm | 59 -
.../hadoop-auth/src/site/markdown/BuildingIt.md | 56 +
.../src/site/markdown/Configuration.md | 341 +++
.../hadoop-auth/src/site/markdown/Examples.md | 109 +
.../hadoop-auth/src/site/markdown/index.md | 43 +
.../server/TestAuthenticationToken.java | 100 -
.../authentication/util/TestAuthToken.java | 127 +
hadoop-common-project/hadoop-common/CHANGES.txt | 64 +-
.../hadoop-common/src/main/bin/hadoop | 5 +
.../src/main/bin/hadoop-config.cmd | 4 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 9 +-
.../hadoop-common/src/main/bin/hadoop-daemon.sh | 3 +
.../src/main/bin/hadoop-daemons.sh | 16 +-
.../src/main/bin/hadoop-functions.sh | 14 +
.../org/apache/hadoop/conf/Configuration.java | 6 +-
.../java/org/apache/hadoop/fs/CanUnbuffer.java | 36 +
.../org/apache/hadoop/fs/FSDataInputStream.java | 12 +-
.../apache/hadoop/fs/shell/XAttrCommands.java | 10 +-
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 4 +-
.../org/apache/hadoop/ha/HealthMonitor.java | 35 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 3 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 1 +
.../java/org/apache/hadoop/log/LogLevel.java | 10 +-
.../metrics/spi/AbstractMetricsContext.java | 12 +-
.../java/org/apache/hadoop/net/NetUtils.java | 1 +
.../hadoop/security/alias/CredentialShell.java | 19 +-
.../src/main/resources/core-default.xml | 77 +-
.../src/site/markdown/ClusterSetup.md | 2 +-
.../src/site/markdown/CommandsManual.md | 10 +-
.../src/site/markdown/SecureMode.md | 44 +-
.../hadoop-common/src/site/markdown/Tracing.md | 55 +-
.../apache/hadoop/conf/TestConfiguration.java | 29 +-
.../org/apache/hadoop/ha/DummyHAService.java | 73 +-
.../org/apache/hadoop/ha/TestHealthMonitor.java | 4 +-
.../java/org/apache/hadoop/io/TestMapFile.java | 5 +-
.../java/org/apache/hadoop/ipc/TestIPC.java | 6 +
.../java/org/apache/hadoop/ipc/TestRPC.java | 68 +
.../org/apache/hadoop/net/TestNetUtils.java | 11 +
.../hadoop/security/alias/TestCredShell.java | 15 +
.../hadoop-kms/src/site/apt/index.apt.vm | 1020 -------
.../hadoop-kms/src/site/markdown/index.md.vm | 864 ++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 109 +-
.../bkjournal/BookKeeperJournalManager.java | 47 +-
.../bkjournal/TestBookKeeperJournalManager.java | 153 +-
.../TestBookKeeperSpeculativeRead.java | 167 ++
.../hadoop-hdfs/src/main/bin/hdfs | 5 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 77 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 30 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 39 +-
.../hadoop/hdfs/DistributedFileSystem.java | 60 +-
.../java/org/apache/hadoop/hdfs/PeerCache.java | 9 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 18 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 8 +-
.../hdfs/protocol/DatanodeInfoWithStorage.java | 4 +
.../hadoop/hdfs/protocol/LocatedBlock.java | 47 +-
.../datatransfer/DataTransferProtocol.java | 6 +-
.../hdfs/protocol/datatransfer/Receiver.java | 4 +-
.../hdfs/protocol/datatransfer/Sender.java | 8 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 2 +-
.../ClientNamenodeProtocolTranslatorPB.java | 4 +-
.../DatanodeProtocolClientSideTranslatorPB.java | 9 +-
.../DatanodeProtocolServerSideTranslatorPB.java | 7 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 46 +
.../hadoop/hdfs/server/balancer/Balancer.java | 7 +-
.../blockmanagement/DatanodeDescriptor.java | 39 +-
.../server/blockmanagement/DatanodeManager.java | 19 +-
.../blockmanagement/HeartbeatManager.java | 8 +-
.../hdfs/server/datanode/BPOfferService.java | 8 +-
.../hdfs/server/datanode/BPServiceActor.java | 69 +-
.../server/datanode/BPServiceActorAction.java | 32 +
.../datanode/BPServiceActorActionException.java | 33 +
.../hdfs/server/datanode/BlockReceiver.java | 12 +-
.../hdfs/server/datanode/BlockScanner.java | 32 +
.../hdfs/server/datanode/BlockSender.java | 3 +
.../hadoop/hdfs/server/datanode/DataNode.java | 8 +-
.../hdfs/server/datanode/DataXceiver.java | 34 +-
.../hdfs/server/datanode/DataXceiverServer.java | 4 +-
.../hdfs/server/datanode/ErrorReportAction.java | 83 +
.../server/datanode/ReportBadBlockAction.java | 108 +
.../hdfs/server/datanode/VolumeScanner.java | 133 +-
.../server/datanode/fsdataset/FsDatasetSpi.java | 21 +
.../datanode/fsdataset/impl/FsDatasetImpl.java | 148 +-
.../datanode/fsdataset/impl/FsVolumeList.java | 41 +-
.../fsdataset/impl/VolumeFailureInfo.java | 82 +
.../server/datanode/metrics/FSDatasetMBean.java | 19 +
.../server/datanode/web/DatanodeHttpServer.java | 2 +-
.../datanode/web/webhdfs/ParameterParser.java | 2 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 6 +-
.../namenode/DirectoryWithQuotaFeature.java | 111 +-
.../hdfs/server/namenode/FSDirAttrOp.java | 36 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 2 +-
.../hdfs/server/namenode/FSDirectory.java | 39 +-
.../hadoop/hdfs/server/namenode/FSImage.java | 16 +-
.../hdfs/server/namenode/FSImageFormat.java | 9 +-
.../server/namenode/FSImageFormatPBINode.java | 6 +-
.../server/namenode/FSImageSerialization.java | 2 +-
.../hdfs/server/namenode/FSNamesystem.java | 65 +-
.../hadoop/hdfs/server/namenode/INode.java | 18 +-
.../hdfs/server/namenode/INodeDirectory.java | 12 +-
.../namenode/INodeDirectoryAttributes.java | 8 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 46 +-
.../hdfs/server/namenode/INodeReference.java | 4 +-
.../hdfs/server/namenode/INodeSymlink.java | 2 +-
.../hdfs/server/namenode/ImageServlet.java | 2 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 4 +-
.../server/namenode/NameNodeHttpServer.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 10 +-
.../hadoop/hdfs/server/namenode/Quota.java | 10 +-
.../hdfs/server/namenode/QuotaCounts.java | 98 +-
.../hdfs/server/namenode/SecondaryNameNode.java | 4 +-
.../namenode/metrics/FSNamesystemMBean.java | 13 +
.../snapshot/FileWithSnapshotFeature.java | 14 +-
.../namenode/snapshot/SnapshotDiffInfo.java | 9 +-
.../hdfs/server/protocol/DatanodeProtocol.java | 5 +-
.../server/protocol/VolumeFailureSummary.java | 72 +
.../offlineImageViewer/PBImageTextWriter.java | 2 +-
.../hadoop/hdfs/web/URLConnectionFactory.java | 7 +-
.../src/main/native/fuse-dfs/fuse_connect.c | 16 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 28 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 9 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +-
.../src/main/proto/DatanodeProtocol.proto | 17 +-
.../src/main/proto/datatransfer.proto | 3 +
.../src/main/resources/hdfs-default.xml | 6 +
.../src/main/webapps/hdfs/dfshealth.html | 33 +
.../src/main/webapps/hdfs/dfshealth.js | 43 +
.../src/main/webapps/hdfs/explorer.js | 7 +
.../src/site/apt/ArchivalStorage.apt.vm | 233 --
.../site/apt/CentralizedCacheManagement.apt.vm | 344 ---
.../src/site/apt/ExtendedAttributes.apt.vm | 97 -
.../src/site/apt/FaultInjectFramework.apt.vm | 312 --
.../hadoop-hdfs/src/site/apt/Federation.apt.vm | 339 ---
.../src/site/apt/HDFSCommands.apt.vm | 797 ------
.../site/apt/HDFSHighAvailabilityWithNFS.apt.vm | 859 ------
.../site/apt/HDFSHighAvailabilityWithQJM.apt.vm | 816 ------
.../hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm | 510 ----
.../src/site/apt/HdfsEditsViewer.apt.vm | 104 -
.../src/site/apt/HdfsImageViewer.apt.vm | 247 --
.../src/site/apt/HdfsMultihoming.apt.vm | 145 -
.../src/site/apt/HdfsNfsGateway.apt.vm | 364 ---
.../src/site/apt/HdfsPermissionsGuide.apt.vm | 438 ---
.../src/site/apt/HdfsQuotaAdminGuide.apt.vm | 116 -
.../src/site/apt/HdfsUserGuide.apt.vm | 556 ----
.../hadoop-hdfs/src/site/apt/LibHdfs.apt.vm | 101 -
.../src/site/apt/SLGUserGuide.apt.vm | 195 --
.../src/site/apt/ShortCircuitLocalReads.apt.vm | 112 -
.../src/site/apt/TransparentEncryption.apt.vm | 290 --
.../hadoop-hdfs/src/site/apt/ViewFs.apt.vm | 304 --
.../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 2628 -----------------
.../src/site/markdown/ArchivalStorage.md | 160 ++
.../site/markdown/CentralizedCacheManagement.md | 268 ++
.../src/site/markdown/ExtendedAttributes.md | 98 +
.../src/site/markdown/FaultInjectFramework.md | 254 ++
.../hadoop-hdfs/src/site/markdown/Federation.md | 254 ++
.../src/site/markdown/HDFSCommands.md | 514 ++++
.../markdown/HDFSHighAvailabilityWithNFS.md | 678 +++++
.../markdown/HDFSHighAvailabilityWithQJM.md | 642 +++++
.../hadoop-hdfs/src/site/markdown/HdfsDesign.md | 240 ++
.../src/site/markdown/HdfsEditsViewer.md | 69 +
.../src/site/markdown/HdfsImageViewer.md | 172 ++
.../src/site/markdown/HdfsMultihoming.md | 127 +
.../src/site/markdown/HdfsNfsGateway.md | 254 ++
.../src/site/markdown/HdfsPermissionsGuide.md | 284 ++
.../src/site/markdown/HdfsQuotaAdminGuide.md | 93 +
.../src/site/markdown/HdfsUserGuide.md | 375 +++
.../hadoop-hdfs/src/site/markdown/LibHdfs.md | 92 +
.../src/site/markdown/SLGUserGuide.md | 157 +
.../src/site/markdown/ShortCircuitLocalReads.md | 87 +
.../src/site/markdown/TransparentEncryption.md | 268 ++
.../hadoop-hdfs/src/site/markdown/ViewFs.md | 242 ++
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 1939 +++++++++++++
.../java/org/apache/hadoop/fs/TestUnbuffer.java | 127 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 27 +-
.../hadoop/hdfs/TestDataTransferProtocol.java | 2 +-
.../hdfs/server/balancer/TestBalancer.java | 71 +-
.../blockmanagement/TestBlockManager.java | 3 +-
.../TestNameNodePrunesMissingStorages.java | 4 +-
.../TestOverReplicatedBlocks.java | 2 +-
.../blockmanagement/TestReplicationPolicy.java | 2 +-
.../TestReplicationPolicyConsiderLoad.java | 8 +-
.../TestReplicationPolicyWithNodeGroup.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 33 +
.../server/datanode/TestBPOfferService.java | 161 +-
.../hdfs/server/datanode/TestBlockRecovery.java | 4 +-
.../hdfs/server/datanode/TestBlockScanner.java | 131 +
.../TestDataNodeVolumeFailureReporting.java | 421 ++-
.../hdfs/server/datanode/TestDiskError.java | 2 +-
.../server/datanode/TestFsDatasetCache.java | 3 +-
.../hdfs/server/datanode/TestStorageReport.java | 4 +-
.../extdataset/ExternalDatasetImpl.java | 30 +
.../fsdataset/impl/FsVolumeListTest.java | 101 -
.../fsdataset/impl/TestFsDatasetImpl.java | 6 +-
.../fsdataset/impl/TestFsVolumeList.java | 104 +
.../web/webhdfs/TestParameterParser.java | 13 +
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/NameNodeAdapter.java | 2 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestDiskspaceQuotaUpdate.java | 14 +-
.../namenode/TestFavoredNodesEndToEnd.java | 29 +
.../hdfs/server/namenode/TestMalformedURLs.java | 59 +
.../server/namenode/TestQuotaByStorageType.java | 193 +-
.../snapshot/TestRenameWithSnapshots.java | 10 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 4 +-
.../src/test/resources/hdfs-site.malformed.xml | 143 +
hadoop-mapreduce-project/CHANGES.txt | 34 +
hadoop-mapreduce-project/bin/mapred | 5 +
.../bin/mr-jobhistory-daemon.sh | 3 +
.../jobhistory/JobHistoryEventHandler.java | 7 +-
.../mapreduce/v2/app/webapp/CountersBlock.java | 6 +-
.../org/apache/hadoop/mapred/IndexCache.java | 2 +-
.../java/org/apache/hadoop/mapred/MapTask.java | 4 +
.../org/apache/hadoop/mapred/QueueManager.java | 4 +-
.../apache/hadoop/mapred/TaskLogAppender.java | 2 +-
.../mapred/lib/CombineFileRecordReader.java | 5 -
.../java/org/apache/hadoop/mapreduce/Job.java | 47 -
.../apache/hadoop/mapreduce/JobSubmitter.java | 15 +-
.../hadoop/mapreduce/lib/chain/Chain.java | 5 +-
.../lib/fieldsel/FieldSelectionHelper.java | 2 +-
.../lib/input/CombineFileRecordReader.java | 5 -
.../mapreduce/security/SecureShuffleUtils.java | 15 +-
.../mapreduce/task/reduce/InMemoryReader.java | 4 +-
.../task/reduce/ShuffleSchedulerImpl.java | 9 +-
.../org/apache/hadoop/mapreduce/tools/CLI.java | 21 +-
.../hadoop/mapreduce/util/ResourceBundles.java | 2 +-
.../src/site/apt/DistributedCacheDeploy.apt.vm | 151 -
.../src/site/apt/EncryptedShuffle.apt.vm | 320 ---
.../src/site/apt/MapReduceTutorial.apt.vm | 1605 -----------
...pReduce_Compatibility_Hadoop1_Hadoop2.apt.vm | 114 -
.../src/site/apt/MapredAppMasterRest.apt.vm | 2709 ------------------
.../src/site/apt/MapredCommands.apt.vm | 233 --
.../apt/PluggableShuffleAndPluggableSort.apt.vm | 98 -
.../site/markdown/DistributedCacheDeploy.md.vm | 119 +
.../src/site/markdown/EncryptedShuffle.md | 255 ++
.../src/site/markdown/MapReduceTutorial.md | 1156 ++++++++
.../MapReduce_Compatibility_Hadoop1_Hadoop2.md | 69 +
.../src/site/markdown/MapredAppMasterRest.md | 2397 ++++++++++++++++
.../src/site/markdown/MapredCommands.md | 153 +
.../PluggableShuffleAndPluggableSort.md | 73 +
.../apache/hadoop/mapreduce/tools/TestCLI.java | 45 +
.../hadoop/mapreduce/v2/hs/CompletedJob.java | 8 +-
.../src/site/apt/HistoryServerRest.apt.vm | 2672 -----------------
.../src/site/markdown/HistoryServerRest.md | 2361 +++++++++++++++
.../TestClientProtocolProviderImpls.java | 71 +-
hadoop-project/src/site/apt/index.apt.vm | 73 -
hadoop-project/src/site/markdown/index.md.vm | 72 +
.../org/apache/hadoop/fs/s3a/Constants.java | 4 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 2 +
.../apache/hadoop/fs/s3a/S3AInputStream.java | 20 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 83 +-
hadoop-tools/hadoop-distcp/README | 2 -
.../src/site/markdown/DistCp.md.vm | 4 +-
.../hadoop/mapred/gridmix/TestHighRamJob.java | 11 +-
.../hadoop-openstack/src/site/apt/index.apt.vm | 686 -----
.../hadoop-openstack/src/site/markdown/index.md | 544 ++++
.../src/site/resources/css/site.css | 30 +
.../src/site/apt/SchedulerLoadSimulator.apt.vm | 439 ---
.../src/site/markdown/SchedulerLoadSimulator.md | 357 +++
.../src/site/apt/HadoopStreaming.apt.vm | 792 -----
.../src/site/markdown/HadoopStreaming.md.vm | 559 ++++
hadoop-yarn-project/CHANGES.txt | 65 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +
.../hadoop-yarn/bin/yarn-daemon.sh | 3 +
.../hadoop-yarn/bin/yarn-daemons.sh | 19 +-
.../dev-support/findbugs-exclude.xml | 29 +-
.../hadoop/yarn/api/records/ContainerId.java | 5 +
.../hadoop/yarn/client/cli/RMAdminCLI.java | 29 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 18 +-
.../nodelabels/CommonNodeLabelsManager.java | 7 +-
.../apache/hadoop/yarn/util/ConverterUtils.java | 7 +
.../src/main/resources/yarn-default.xml | 5 +-
.../hadoop/yarn/util/TestConverterUtils.java | 15 +
.../server/timeline/LeveldbTimelineStore.java | 2 +-
.../server/timeline/TimelineStoreTestUtils.java | 3 +
.../containermanager/AuxServices.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 4 +-
.../application/ApplicationImpl.java | 2 +-
.../container/ContainerImpl.java | 2 +-
.../logaggregation/AppLogAggregatorImpl.java | 15 +
.../loghandler/NonAggregatingLogHandler.java | 63 +-
.../recovery/NMLeveldbStateStoreService.java | 67 +-
.../recovery/NMNullStateStoreService.java | 16 +
.../recovery/NMStateStoreService.java | 35 +
.../impl/container-executor.c | 3 +-
.../test/test-container-executor.c | 24 +-
.../yarn_server_nodemanager_recovery.proto | 4 +
.../TestDockerContainerExecutorWithMocks.java | 6 +-
.../TestNonAggregatingLogHandler.java | 79 +-
.../recovery/NMMemoryStateStoreService.java | 79 +-
.../TestNMLeveldbStateStoreService.java | 51 +
.../ApplicationMasterService.java | 16 +-
.../resourcemanager/ResourceTrackerService.java | 4 +-
.../amlauncher/ApplicationMasterLauncher.java | 1 +
.../nodelabels/RMNodeLabelsManager.java | 20 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 52 +-
.../scheduler/AppSchedulingInfo.java | 3 +-
.../scheduler/capacity/AbstractCSQueue.java | 240 +-
.../scheduler/capacity/CSQueue.java | 49 +-
.../scheduler/capacity/CSQueueUtils.java | 160 +-
.../CapacitySchedulerConfiguration.java | 73 +-
.../scheduler/capacity/LeafQueue.java | 196 +-
.../scheduler/capacity/ParentQueue.java | 92 +-
.../scheduler/capacity/PlanQueue.java | 13 +-
.../scheduler/capacity/QueueCapacities.java | 68 +-
.../scheduler/capacity/ReservationQueue.java | 10 +-
.../scheduler/fair/AllocationConfiguration.java | 13 +-
.../fair/AllocationFileLoaderService.java | 2 +-
.../scheduler/fair/FSOpDurations.java | 3 +
.../yarn/server/resourcemanager/MockAM.java | 28 +-
.../server/resourcemanager/TestRMRestart.java | 4 +
.../nodelabels/TestRMNodeLabelsManager.java | 63 +-
.../attempt/TestRMAppAttemptTransitions.java | 2 +-
.../scheduler/capacity/TestCSQueueUtils.java | 24 +-
.../capacity/TestCapacityScheduler.java | 6 +-
.../TestCapacitySchedulerNodeLabelUpdate.java | 3 +-
.../scheduler/capacity/TestQueueCapacities.java | 2 +-
.../scheduler/capacity/TestQueueParsing.java | 14 +-
.../security/TestAMRMTokens.java | 45 +-
.../webapp/TestRMWebServicesCapacitySched.java | 4 +-
.../yarn/server/webproxy/ProxyUriUtils.java | 11 +-
.../hadoop/yarn/server/webproxy/ProxyUtils.java | 127 +
.../yarn/server/webproxy/WebAppProxy.java | 14 +-
.../yarn/server/webproxy/WebAppProxyServer.java | 20 +-
.../server/webproxy/WebAppProxyServlet.java | 131 +-
.../webproxy/amfilter/AmFilterInitializer.java | 2 +-
.../server/webproxy/amfilter/AmIpFilter.java | 48 +-
.../server/webproxy/TestWebAppProxyServlet.java | 84 +-
.../server/webproxy/amfilter/TestAmFilter.java | 39 +-
.../src/site/apt/ResourceManagerRestart.apt.vm | 182 +-
.../src/site/apt/YarnCommands.apt.vm | 21 +-
337 files changed, 23330 insertions(+), 23584 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a3d0af5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
[22/50] [abbrv] hadoop git commit: MAPREDUCE-6260. Convert site
documentation to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
new file mode 100644
index 0000000..b11b0ae
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredAppMasterRest.md
@@ -0,0 +1,2397 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+MapReduce Application Master REST API's.
+========================================
+
+* [MapReduce Application Master REST API's.](#MapReduce_Application_Master_REST_APIs.)
+ * [Overview](#Overview)
+ * [Mapreduce Application Master Info API](#Mapreduce_Application_Master_Info_API)
+ * [Jobs API](#Jobs_API)
+ * [Job API](#Job_API)
+ * [Job Attempts API](#Job_Attempts_API)
+ * [Job Counters API](#Job_Counters_API)
+ * [Job Conf API](#Job_Conf_API)
+ * [Tasks API](#Tasks_API)
+ * [Task API](#Task_API)
+ * [Task Counters API](#Task_Counters_API)
+ * [Task Attempts API](#Task_Attempts_API)
+ * [Task Attempt API](#Task_Attempt_API)
+ * [Task Attempt Counters API](#Task_Attempt_Counters_API)
+
+Overview
+--------
+
+The MapReduce Application Master REST API's allow the user to get status on the running MapReduce application master. Currently this is the equivalent to a running MapReduce job. The information includes the jobs the app master is running and all the job particulars like tasks, counters, configuration, attempts, etc. The application master should be accessed via the proxy. This proxy is configurable to run either on the resource manager or on a separate host. The proxy URL usually looks like: `http://<proxy http address:port>/proxy/appid`.
+
+Mapreduce Application Master Info API
+-------------------------------------
+
+The MapReduce application master information resource provides overall information about that mapreduce application master. This includes application id, time it was started, user, name, etc.
+
+### URI
+
+Both of the following URI's give you the MapReduce application master information, from an application id identified by the appid value.
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/info
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *info* object
+
+When you make a request for the mapreduce application master information, the information will be returned as an info object.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| appId | long | The application id |
+| startedOn | long | The time the application started (in ms since epoch) |
+| name | string | The name of the application |
+| user | string | The user name of the user who started the application |
+| elapsedTime | long | The time since the application was started (in ms) |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0003/ws/v1/mapreduce/info
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "info" : {
+ "appId" : "application_1326232085508_0003",
+ "startedOn" : 1326238244047,
+ "user" : "user1",
+ "name" : "Sleep job",
+ "elapsedTime" : 32374
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ Accept: application/xml
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0003/ws/v1/mapreduce/info
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 223
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <info>
+ <appId>application_1326232085508_0003</appId>
+ <name>Sleep job</name>
+ <user>user1</user>
+ <startedOn>1326238244047</startedOn>
+ <elapsedTime>32407</elapsedTime>
+ </info>
+
+Jobs API
+--------
+
+The jobs resource provides a list of the jobs running on this application master. See also [Job API](#Job_API) for syntax of the job object.
+
+### URI
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *jobs* object
+
+When you make a request for the list of jobs, the information will be returned as a collection of job objects. See also [Job API](#Job_API) for syntax of the job object.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| job | array of job objects(JSON)/Zero or more job objects(XML) | The collection of job objects |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobs" : {
+ "job" : [
+ {
+ "runningReduceAttempts" : 1,
+ "reduceProgress" : 100,
+ "failedReduceAttempts" : 0,
+ "newMapAttempts" : 0,
+ "mapsRunning" : 0,
+ "state" : "RUNNING",
+ "successfulReduceAttempts" : 0,
+ "reducesRunning" : 1,
+ "acls" : [
+ {
+ "value" : " ",
+ "name" : "mapreduce.job.acl-modify-job"
+ },
+ {
+ "value" : " ",
+ "name" : "mapreduce.job.acl-view-job"
+ }
+ ],
+ "reducesPending" : 0,
+ "user" : "user1",
+ "reducesTotal" : 1,
+ "mapsCompleted" : 1,
+ "startTime" : 1326238769379,
+ "id" : "job_1326232085508_4_4",
+ "successfulMapAttempts" : 1,
+ "runningMapAttempts" : 0,
+ "newReduceAttempts" : 0,
+ "name" : "Sleep job",
+ "mapsPending" : 0,
+ "elapsedTime" : 59377,
+ "reducesCompleted" : 0,
+ "mapProgress" : 100,
+ "diagnostics" : "",
+ "failedMapAttempts" : 0,
+ "killedReduceAttempts" : 0,
+ "mapsTotal" : 1,
+ "uberized" : false,
+ "killedMapAttempts" : 0,
+ "finishTime" : 0
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 1214
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobs>
+ <job>
+ <startTime>1326238769379</startTime>
+ <finishTime>0</finishTime>
+ <elapsedTime>59416</elapsedTime>
+ <id>job_1326232085508_4_4</id>
+ <name>Sleep job</name>
+ <user>user1</user>
+ <state>RUNNING</state>
+ <mapsTotal>1</mapsTotal>
+ <mapsCompleted>1</mapsCompleted>
+ <reducesTotal>1</reducesTotal>
+ <reducesCompleted>0</reducesCompleted>
+ <mapProgress>100.0</mapProgress>
+ <reduceProgress>100.0</reduceProgress>
+ <mapsPending>0</mapsPending>
+ <mapsRunning>0</mapsRunning>
+ <reducesPending>0</reducesPending>
+ <reducesRunning>1</reducesRunning>
+ <uberized>false</uberized>
+ <diagnostics/>
+ <newReduceAttempts>0</newReduceAttempts>
+ <runningReduceAttempts>1</runningReduceAttempts>
+ <failedReduceAttempts>0</failedReduceAttempts>
+ <killedReduceAttempts>0</killedReduceAttempts>
+ <successfulReduceAttempts>0</successfulReduceAttempts>
+ <newMapAttempts>0</newMapAttempts>
+ <runningMapAttempts>0</runningMapAttempts>
+ <failedMapAttempts>0</failedMapAttempts>
+ <killedMapAttempts>0</killedMapAttempts>
+ <successfulMapAttempts>1</successfulMapAttempts>
+ <acls>
+ <name>mapreduce.job.acl-modify-job</name>
+ <value> </value>
+ </acls>
+ <acls>
+ <name>mapreduce.job.acl-view-job</name>
+ <value> </value>
+ </acls>
+ </job>
+ </jobs>
+
+Job API
+-------
+
+A job resource contains information about a particular job that was started by this application master. Certain fields are only accessible if user has permissions - depends on acl settings.
+
+### URI
+
+Use the following URI to obtain a job object, for a job identified by the jobid value.
+
+ * http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid}
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *job* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The job id |
+| name | string | The job name |
+| user | string | The user name |
+| state | string | the job state - valid values are: NEW, INITED, RUNNING, SUCCEEDED, FAILED, KILL\_WAIT, KILLED, ERROR |
+| startTime | long | The time the job started (in ms since epoch) |
+| finishTime | long | The time the job finished (in ms since epoch) |
+| elapsedTime | long | The elapsed time since job started (in ms) |
+| mapsTotal | int | The total number of maps |
+| mapsCompleted | int | The number of completed maps |
+| reducesTotal | int | The total number of reduces |
+| reducesCompleted | int | The number of completed reduces |
+| diagnostics | string | A diagnostic message |
+| uberized | boolean | Indicates if the job was an uber job - ran completely in the application master |
+| mapsPending | int | The number of maps still to be run |
+| mapsRunning | int | The number of running maps |
+| reducesPending | int | The number of reduces still to be run |
+| reducesRunning | int | The number of running reduces |
+| newReduceAttempts | int | The number of new reduce attempts |
+| runningReduceAttempts | int | The number of running reduce attempts |
+| failedReduceAttempts | int | The number of failed reduce attempts |
+| killedReduceAttempts | int | The number of killed reduce attempts |
+| successfulReduceAttempts | int | The number of successful reduce attempts |
+| newMapAttempts | int | The number of new map attempts |
+| runningMapAttempts | int | The number of running map attempts |
+| failedMapAttempts | int | The number of failed map attempts |
+| killedMapAttempts | int | The number of killed map attempts |
+| successfulMapAttempts | int | The number of successful map attempts |
+| acls | array of acls(json)/zero or more acls objects(xml) | A collection of acls objects |
+
+### Elements of the *acls* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| value | string | The acl value |
+| name | string | The acl name |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Server: Jetty(6.1.26)
+ Content-Length: 720
+
+Response Body:
+
+ {
+ "job" : {
+ "runningReduceAttempts" : 1,
+ "reduceProgress" : 100,
+ "failedReduceAttempts" : 0,
+ "newMapAttempts" : 0,
+ "mapsRunning" : 0,
+ "state" : "RUNNING",
+ "successfulReduceAttempts" : 0,
+ "reducesRunning" : 1,
+ "acls" : [
+ {
+ "value" : " ",
+ "name" : "mapreduce.job.acl-modify-job"
+ },
+ {
+ "value" : " ",
+ "name" : "mapreduce.job.acl-view-job"
+ }
+ ],
+ "reducesPending" : 0,
+ "user" : "user1",
+ "reducesTotal" : 1,
+ "mapsCompleted" : 1,
+ "startTime" : 1326238769379,
+ "id" : "job_1326232085508_4_4",
+ "successfulMapAttempts" : 1,
+ "runningMapAttempts" : 0,
+ "newReduceAttempts" : 0,
+ "name" : "Sleep job",
+ "mapsPending" : 0,
+ "elapsedTime" : 59437,
+ "reducesCompleted" : 0,
+ "mapProgress" : 100,
+ "diagnostics" : "",
+ "failedMapAttempts" : 0,
+ "killedReduceAttempts" : 0,
+ "mapsTotal" : 1,
+ "uberized" : false,
+ "killedMapAttempts" : 0,
+ "finishTime" : 0
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 1201
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <job>
+ <startTime>1326238769379</startTime>
+ <finishTime>0</finishTime>
+ <elapsedTime>59474</elapsedTime>
+ <id>job_1326232085508_4_4</id>
+ <name>Sleep job</name>
+ <user>user1</user>
+ <state>RUNNING</state>
+ <mapsTotal>1</mapsTotal>
+ <mapsCompleted>1</mapsCompleted>
+ <reducesTotal>1</reducesTotal>
+ <reducesCompleted>0</reducesCompleted>
+ <mapProgress>100.0</mapProgress>
+ <reduceProgress>100.0</reduceProgress>
+ <mapsPending>0</mapsPending>
+ <mapsRunning>0</mapsRunning>
+ <reducesPending>0</reducesPending>
+ <reducesRunning>1</reducesRunning>
+ <uberized>false</uberized>
+ <diagnostics/>
+ <newReduceAttempts>0</newReduceAttempts>
+ <runningReduceAttempts>1</runningReduceAttempts>
+ <failedReduceAttempts>0</failedReduceAttempts>
+ <killedReduceAttempts>0</killedReduceAttempts>
+ <successfulReduceAttempts>0</successfulReduceAttempts>
+ <newMapAttempts>0</newMapAttempts>
+ <runningMapAttempts>0</runningMapAttempts>
+ <failedMapAttempts>0</failedMapAttempts>
+ <killedMapAttempts>0</killedMapAttempts>
+ <successfulMapAttempts>1</successfulMapAttempts>
+ <acls>
+ <name>mapreduce.job.acl-modify-job</name>
+ <value> </value>
+ </acls>
+ <acls>
+ <name>mapreduce.job.acl-view-job</name> <value> </value>
+ </acls>
+ </job>
+
+Job Attempts API
+----------------
+
+With the job attempts API, you can obtain a collection of resources that represent the job attempts. When you run a GET operation on this resource, you obtain a collection of Job Attempt Objects.
+
+### URI
+
+ * http://<history server http address:port>/ws/v1/history/jobs/{jobid}/jobattempts
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *jobAttempts* object
+
+When you make a request for the list of job attempts, the information will be returned as an array of job attempt objects.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| jobAttempt | array of job attempt objects(JSON)/zero or more job attempt objects(XML) | The collection of job attempt objects |
+
+### Elements of the *jobAttempt* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The job attempt id |
+| nodeId | string | The node id of the node the attempt ran on |
+| nodeHttpAddress | string | The node http address of the node the attempt ran on |
+| logsLink | string | The http link to the job attempt logs |
+| containerId | string | The id of the container for the job attempt |
+| startTime | long | The start time of the attempt (in ms since epoch) |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobAttempts" : {
+ "jobAttempt" : [
+ {
+ "nodeId" : "host.domain.com:8041",
+ "nodeHttpAddress" : "host.domain.com:8042",
+ "startTime" : 1326238773493,
+ "id" : 1,
+ "logsLink" : "http://host.domain.com:8042/node/containerlogs/container_1326232085508_0004_01_000001",
+ "containerId" : "container_1326232085508_0004_01_000001"
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/jobattempts
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 498
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobAttempts>
+ <jobAttempt>
+ <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
+ <nodeId>host.domain.com:8041</nodeId>
+ <id>1</id>
+ <startTime>1326238773493</startTime>
+ <containerId>container_1326232085508_0004_01_000001</containerId>
+ <logsLink>http://host.domain.com:8042/node/containerlogs/container_1326232085508_0004_01_000001</logsLink>
+ </jobAttempt>
+ </jobAttempts>
+
+Job Counters API
+----------------
+
+With the job counters API, you can object a collection of resources that represent all the counters for that job.
+
+### URI
+
+ * http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/{jobid}/counters
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *jobCounters* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The job id |
+| counterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
+
+### Elements of the *counterGroup* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| counterGroupName | string | The name of the counter group |
+| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
+
+### Elements of the *counter* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the counter |
+| reduceCounterValue | long | The counter value of reduce tasks |
+| mapCounterValue | long | The counter value of map tasks |
+| totalCounterValue | long | The counter value of all tasks |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobCounters" : {
+ "id" : "job_1326232085508_4_4",
+ "counterGroup" : [
+ {
+ "counterGroupName" : "Shuffle Errors",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "BAD_ID"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "CONNECTION"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "IO_ERROR"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "WRONG_LENGTH"
+ }, {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "WRONG_MAP"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "WRONG_REDUCE"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2483,
+ "name" : "FILE_BYTES_READ"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 108763,
+ "name" : "FILE_BYTES_WRITTEN"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FILE_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FILE_LARGE_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FILE_WRITE_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 48,
+ "name" : "HDFS_BYTES_READ"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "HDFS_BYTES_WRITTEN"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "HDFS_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "HDFS_LARGE_READ_OPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "HDFS_WRITE_OPS"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "MAP_INPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1200,
+ "name" : "MAP_OUTPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 4800,
+ "name" : "MAP_OUTPUT_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2235,
+ "name" : "MAP_OUTPUT_MATERIALIZED_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 48,
+ "name" : "SPLIT_RAW_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "COMBINE_INPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "COMBINE_OUTPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 460,
+ "name" : "REDUCE_INPUT_GROUPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2235,
+ "name" : "REDUCE_SHUFFLE_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 460,
+ "name" : "REDUCE_INPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "REDUCE_OUTPUT_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1200,
+ "name" : "SPILLED_RECORDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "SHUFFLED_MAPS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "FAILED_SHUFFLE"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1,
+ "name" : "MERGED_MAP_OUTPUTS"
+ }, {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 58,
+ "name" : "GC_TIME_MILLIS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 1580,
+ "name" : "CPU_MILLISECONDS"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 462643200,
+ "name" : "PHYSICAL_MEMORY_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 2149728256,
+ "name" : "VIRTUAL_MEMORY_BYTES"
+ },
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 357957632,
+ "name" : "COMMITTED_HEAP_BYTES"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "BYTES_READ"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+ "counter" : [
+ {
+ "reduceCounterValue" : 0,
+ "mapCounterValue" : 0,
+ "totalCounterValue" : 0,
+ "name" : "BYTES_WRITTEN"
+ }
+ ]
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/counters
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 7027
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobCounters>
+ <id>job_1326232085508_4_4</id>
+ <counterGroup>
+ <counterGroupName>Shuffle Errors</counterGroupName>
+ <counter>
+ <name>BAD_ID</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>CONNECTION</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>IO_ERROR</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>WRONG_LENGTH</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>WRONG_MAP</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>WRONG_REDUCE</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
+ <counter>
+ <name>FILE_BYTES_READ</name>
+ <totalCounterValue>2483</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_BYTES_WRITTEN</name>
+ <totalCounterValue>108763</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_READ_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_LARGE_READ_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FILE_WRITE_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_READ</name>
+ <totalCounterValue>48</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_WRITTEN</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_READ_OPS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_LARGE_READ_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>HDFS_WRITE_OPS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
+ <counter>
+ <name>MAP_INPUT_RECORDS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MAP_OUTPUT_RECORDS</name>
+ <totalCounterValue>1200</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MAP_OUTPUT_BYTES</name>
+ <totalCounterValue>4800</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MAP_OUTPUT_MATERIALIZED_BYTES</name>
+ <totalCounterValue>2235</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>SPLIT_RAW_BYTES</name>
+ <totalCounterValue>48</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>COMBINE_INPUT_RECORDS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>COMBINE_OUTPUT_RECORDS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_GROUPS</name>
+ <totalCounterValue>460</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_SHUFFLE_BYTES</name>
+ <totalCounterValue>2235</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_RECORDS</name>
+ <totalCounterValue>460</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>REDUCE_OUTPUT_RECORDS</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>SPILLED_RECORDS</name>
+ <totalCounterValue>1200</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>SHUFFLED_MAPS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>FAILED_SHUFFLE</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>MERGED_MAP_OUTPUTS</name>
+ <totalCounterValue>1</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>GC_TIME_MILLIS</name>
+ <totalCounterValue>58</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>CPU_MILLISECONDS</name>
+ <totalCounterValue>1580</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>PHYSICAL_MEMORY_BYTES</name>
+ <totalCounterValue>462643200</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>VIRTUAL_MEMORY_BYTES</name>
+ <totalCounterValue>2149728256</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ <counter>
+ <name>COMMITTED_HEAP_BYTES</name>
+ <totalCounterValue>357957632</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.input.FileInputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_READ</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter> </counterGroup> <counterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
+ <counter> <name>BYTES_WRITTEN</name>
+ <totalCounterValue>0</totalCounterValue>
+ <mapCounterValue>0</mapCounterValue>
+ <reduceCounterValue>0</reduceCounterValue>
+ </counter>
+ </counterGroup>
+ </jobCounters>
+
+Job Conf API
+------------
+
+A job configuration resource contains information about the job configuration for this job.
+
+### URI
+
+Use the following URI to obtain th job configuration information, from a job identified by the jobid value.
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/conf
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *conf* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| path | string | The path to the job configuration file |
+| property | array of the configuration properties(JSON)/zero or more property objects(XML) | Collection of property objects |
+
+### Elements of the *property* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the configuration property |
+| value | string | The value of the configuration property |
+| source | string | The location this configuration object came from. If there is more then one of these it shows the history with the latest source at the end of the list. |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+This is a small snippet of the output as the output if very large. The real output contains every property in your job configuration file.
+
+ {
+ "conf" : {
+ "path" : "hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml",
+ "property" : [
+ {
+ "value" : "/home/hadoop/hdfs/data",
+ "name" : "dfs.datanode.data.dir",
+ "source" : ["hdfs-site.xml", "job.xml"]
+ },
+ {
+ "value" : "org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer",
+ "name" : "hadoop.http.filter.initializers"
+ "source" : ["programmatically", "job.xml"]
+ },
+ {
+ "value" : "/home/hadoop/tmp",
+ "name" : "mapreduce.cluster.temp.dir"
+ "source" : ["mapred-site.xml"]
+ },
+ ...
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/conf
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 552
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <conf>
+ <path>hdfs://host.domain.com:9000/user/user1/.staging/job_1326232085508_0004/job.xml</path>
+ <property>
+ <name>dfs.datanode.data.dir</name>
+ <value>/home/hadoop/hdfs/data</value>
+ <source>hdfs-site.xml</source>
+ <source>job.xml</source>
+ </property>
+ <property>
+ <name>hadoop.http.filter.initializers</name>
+ <value>org.apache.hadoop.yarn.server.webproxy.amfilter.AmFilterInitializer</value>
+ <source>programmatically</source>
+ <source>job.xml</source>
+ </property>
+ <property>
+ <name>mapreduce.cluster.temp.dir</name>
+ <value>/home/hadoop/tmp</value>
+ <source>mapred-site.xml</source>
+ </property>
+ ...
+ </conf>
+
+Tasks API
+---------
+
+With the tasks API, you can obtain a collection of resources that represent all the tasks for a job. When you run a GET operation on this resource, you obtain a collection of Task Objects.
+
+### URI
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ * type - type of task, valid values are m or r. m for map task or r for reduce task.
+
+### Elements of the *tasks* object
+
+When you make a request for the list of tasks , the information will be returned as an array of task objects. See also [Task API](#Task_API) for syntax of the task object.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| task | array of task objects(JSON)/zero or more task objects(XML) | The collection of task objects |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "tasks" : {
+ "task" : [
+ {
+ "progress" : 100,
+ "elapsedTime" : 2768,
+ "state" : "SUCCEEDED",
+ "startTime" : 1326238773493,
+ "id" : "task_1326232085508_4_4_m_0",
+ "type" : "MAP",
+ "successfulAttempt" : "attempt_1326232085508_4_4_m_0_0",
+ "finishTime" : 1326238776261
+ },
+ {
+ "progress" : 100,
+ "elapsedTime" : 0,
+ "state" : "RUNNING",
+ "startTime" : 1326238777460,
+ "id" : "task_1326232085508_4_4_r_0",
+ "type" : "REDUCE",
+ "successfulAttempt" : "",
+ "finishTime" : 0
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 603
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <tasks>
+ <task>
+ <startTime>1326238773493</startTime>
+ <finishTime>1326238776261</finishTime>
+ <elapsedTime>2768</elapsedTime>
+ <progress>100.0</progress>
+ <id>task_1326232085508_4_4_m_0</id>
+ <state>SUCCEEDED</state>
+ <type>MAP</type>
+ <successfulAttempt>attempt_1326232085508_4_4_m_0_0</successfulAttempt>
+ </task>
+ <task>
+ <startTime>1326238777460</startTime>
+ <finishTime>0</finishTime>
+ <elapsedTime>0</elapsedTime>
+ <progress>100.0</progress>
+ <id>task_1326232085508_4_4_r_0</id>
+ <state>RUNNING</state>
+ <type>REDUCE</type>
+ <successfulAttempt/>
+ </task>
+ </tasks>
+
+Task API
+--------
+
+A Task resource contains information about a particular task within a job.
+
+### URI
+
+Use the following URI to obtain an Task Object, from a task identified by the taskid value.
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *task* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task id |
+| state | string | The state of the task - valid values are: NEW, SCHEDULED, RUNNING, SUCCEEDED, FAILED, KILL\_WAIT, KILLED |
+| type | string | The task type - MAP or REDUCE |
+| successfulAttempt | string | The the id of the last successful attempt |
+| progress | float | The progress of the task as a percent |
+| startTime | long | The time in which the task started (in ms since epoch) |
+| finishTime | long | The time in which the task finished (in ms since epoch) |
+| elapsedTime | long | The elapsed time since the application started (in ms) |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "task" : {
+ "progress" : 100,
+ "elapsedTime" : 0,
+ "state" : "RUNNING",
+ "startTime" : 1326238777460,
+ "id" : "task_1326232085508_4_4_r_0",
+ "type" : "REDUCE",
+ "successfulAttempt" : "",
+ "finishTime" : 0
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 299
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <task>
+ <startTime>1326238777460</startTime>
+ <finishTime>0</finishTime>
+ <elapsedTime>0</elapsedTime>
+ <progress>100.0</progress>
+ <id>task_1326232085508_4_4_r_0</id>
+ <state>RUNNING</state>
+ <type>REDUCE</type>
+ <successfulAttempt/>
+ </task>
+
+Task Counters API
+-----------------
+
+With the task counters API, you can object a collection of resources that represent all the counters for that task.
+
+### URI
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/counters
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *jobTaskCounters* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task id |
+| taskcounterGroup | array of counterGroup objects(JSON)/zero or more counterGroup objects(XML) | A collection of counter group objects |
+
+### Elements of the *counterGroup* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| counterGroupName | string | The name of the counter group |
+| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
+
+### Elements of the *counter* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the counter |
+| value | long | The value of the counter |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobTaskCounters" : {
+ "id" : "task_1326232085508_4_4_r_0",
+ "taskCounterGroup" : [
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
+ "counter" : [
+ {
+ "value" : 2363,
+ "name" : "FILE_BYTES_READ"
+ },
+ {
+ "value" : 54372,
+ "name" : "FILE_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_WRITE_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_READ"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_WRITE_OPS"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "COMBINE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "COMBINE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_GROUPS"
+ },
+ {
+ "value" : 2235,
+ "name" : "REDUCE_SHUFFLE_BYTES"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "REDUCE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "SPILLED_RECORDS"
+ },
+ {
+ "value" : 1,
+ "name" : "SHUFFLED_MAPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FAILED_SHUFFLE"
+ },
+ {
+ "value" : 1,
+ "name" : "MERGED_MAP_OUTPUTS"
+ },
+ {
+ "value" : 26,
+ "name" : "GC_TIME_MILLIS"
+ },
+ {
+ "value" : 860,
+ "name" : "CPU_MILLISECONDS"
+ },
+ {
+ "value" : 107839488,
+ "name" : "PHYSICAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 1123147776,
+ "name" : "VIRTUAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 57475072,
+ "name" : "COMMITTED_HEAP_BYTES"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "Shuffle Errors",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BAD_ID"
+ },
+ {
+ "value" : 0,
+ "name" : "CONNECTION"
+ },
+ {
+ "value" : 0,
+ "name" : "IO_ERROR"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_LENGTH"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_MAP"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_REDUCE"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BYTES_WRITTEN"
+ }
+ ]
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/counters
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 2660
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobTaskCounters>
+ <id>task_1326232085508_4_4_r_0</id>
+ <taskCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
+ <counter>
+ <name>FILE_BYTES_READ</name>
+ <value>2363</value>
+ </counter>
+ <counter>
+ <name>FILE_BYTES_WRITTEN</name>
+ <value>54372</value>
+ </counter>
+ <counter>
+ <name>FILE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_READ</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ </taskCounterGroup>
+ <taskCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
+ <counter>
+ <name>COMBINE_INPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>COMBINE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_GROUPS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_SHUFFLE_BYTES</name>
+ <value>2235</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_RECORDS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SPILLED_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SHUFFLED_MAPS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>FAILED_SHUFFLE</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>MERGED_MAP_OUTPUTS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>GC_TIME_MILLIS</name>
+ <value>26</value>
+ </counter>
+ <counter>
+ <name>CPU_MILLISECONDS</name>
+ <value>860</value>
+ </counter>
+ <counter>
+ <name>PHYSICAL_MEMORY_BYTES</name>
+ <value>107839488</value>
+ </counter>
+ <counter>
+ <name>VIRTUAL_MEMORY_BYTES</name>
+ <value>1123147776</value>
+ </counter>
+ <counter>
+ <name>COMMITTED_HEAP_BYTES</name>
+ <value>57475072</value>
+ </counter>
+ </taskCounterGroup>
+ <taskCounterGroup>
+ <counterGroupName>Shuffle Errors</counterGroupName>
+ <counter>
+ <name>BAD_ID</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>CONNECTION</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>IO_ERROR</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_LENGTH</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_MAP</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_REDUCE</name>
+ <value>0</value>
+ </counter>
+ </taskCounterGroup>
+ <taskCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ </taskCounterGroup>
+ </jobTaskCounters>
+
+Task Attempts API
+-----------------
+
+With the task attempts API, you can obtain a collection of resources that represent a task attempt within a job. When you run a GET operation on this resource, you obtain a collection of Task Attempt Objects.
+
+### URI
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *taskAttempts* object
+
+When you make a request for the list of task attempts, the information will be returned as an array of task attempt objects. See also [Task Attempt API](#Task_Attempt_API) for syntax of the task object.
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| taskAttempt | array of task attempt objects(JSON)/zero or more task attempt objects(XML) | The collection of task attempt objects |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "taskAttempts" : {
+ "taskAttempt" : [
+ {
+ "elapsedMergeTime" : 47,
+ "shuffleFinishTime" : 1326238780052,
+ "assignedContainerId" : "container_1326232085508_0004_01_000003",
+ "progress" : 100,
+ "elapsedTime" : 0,
+ "state" : "RUNNING",
+ "elapsedShuffleTime" : 2592,
+ "mergeFinishTime" : 1326238780099,
+ "rack" : "/98.139.92.0",
+ "elapsedReduceTime" : 0,
+ "nodeHttpAddress" : "host.domain.com:8042",
+ "type" : "REDUCE",
+ "startTime" : 1326238777460,
+ "id" : "attempt_1326232085508_4_4_r_0_0",
+ "finishTime" : 0
+ }
+ ]
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 807
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <taskAttempts>
+ <taskAttempt xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:type="reduceTaskAttemptInfo">
+ <startTime>1326238777460</startTime>
+ <finishTime>0</finishTime>
+ <elapsedTime>0</elapsedTime>
+ <progress>100.0</progress>
+ <id>attempt_1326232085508_4_4_r_0_0</id>
+ <rack>/98.139.92.0</rack>
+ <state>RUNNING</state>
+ <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
+ <type>REDUCE</type>
+ <assignedContainerId>container_1326232085508_0004_01_000003</assignedContainerId>
+ <shuffleFinishTime>1326238780052</shuffleFinishTime>
+ <mergeFinishTime>1326238780099</mergeFinishTime>
+ <elapsedShuffleTime>2592</elapsedShuffleTime>
+ <elapsedMergeTime>47</elapsedMergeTime>
+ <elapsedReduceTime>0</elapsedReduceTime>
+ </taskAttempt>
+ </taskAttempts>
+
+Task Attempt API
+----------------
+
+A Task Attempt resource contains information about a particular task attempt within a job.
+
+### URI
+
+Use the following URI to obtain an Task Attempt Object, from a task identified by the attemptid value.
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *taskAttempt* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task id |
+| rack | string | The rack |
+| state | string | The state of the task attempt - valid values are: NEW, UNASSIGNED, ASSIGNED, RUNNING, COMMIT\_PENDING, SUCCESS\_CONTAINER\_CLEANUP, SUCCEEDED, FAIL\_CONTAINER\_CLEANUP, FAIL\_TASK\_CLEANUP, FAILED, KILL\_CONTAINER\_CLEANUP, KILL\_TASK\_CLEANUP, KILLED |
+| type | string | The type of task |
+| assignedContainerId | string | The container id this attempt is assigned to |
+| nodeHttpAddress | string | The http address of the node this task attempt ran on |
+| diagnostics | string | The diagnostics message |
+| progress | float | The progress of the task attempt as a percent |
+| startTime | long | The time in which the task attempt started (in ms since epoch) |
+| finishTime | long | The time in which the task attempt finished (in ms since epoch) |
+| elapsedTime | long | The elapsed time since the task attempt started (in ms) |
+
+For reduce task attempts you also have the following fields:
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| shuffleFinishTime | long | The time at which shuffle finished (in ms since epoch) |
+| mergeFinishTime | long | The time at which merge finished (in ms since epoch) |
+| elapsedShuffleTime | long | The time it took for the shuffle phase to complete (time in ms between reduce task start and shuffle finish) |
+| elapsedMergeTime | long | The time it took for the merge phase to complete (time in ms between the shuffle finish and merge finish) |
+| elapsedReduceTime | long | The time it took for the reduce phase to complete (time in ms between merge finish to end of reduce task) |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "taskAttempt" : {
+ "elapsedMergeTime" : 47,
+ "shuffleFinishTime" : 1326238780052,
+ "assignedContainerId" : "container_1326232085508_0004_01_000003",
+ "progress" : 100,
+ "elapsedTime" : 0,
+ "state" : "RUNNING",
+ "elapsedShuffleTime" : 2592,
+ "mergeFinishTime" : 1326238780099,
+ "rack" : "/98.139.92.0",
+ "elapsedReduceTime" : 0,
+ "nodeHttpAddress" : "host.domain.com:8042",
+ "startTime" : 1326238777460,
+ "id" : "attempt_1326232085508_4_4_r_0_0",
+ "type" : "REDUCE",
+ "finishTime" : 0
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 691
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <taskAttempt>
+ <startTime>1326238777460</startTime>
+ <finishTime>0</finishTime>
+ <elapsedTime>0</elapsedTime>
+ <progress>100.0</progress>
+ <id>attempt_1326232085508_4_4_r_0_0</id>
+ <rack>/98.139.92.0</rack>
+ <state>RUNNING</state>
+ <nodeHttpAddress>host.domain.com:8042</nodeHttpAddress>
+ <type>REDUCE</type>
+ <assignedContainerId>container_1326232085508_0004_01_000003</assignedContainerId>
+ <shuffleFinishTime>1326238780052</shuffleFinishTime>
+ <mergeFinishTime>1326238780099</mergeFinishTime>
+ <elapsedShuffleTime>2592</elapsedShuffleTime>
+ <elapsedMergeTime>47</elapsedMergeTime>
+ <elapsedReduceTime>0</elapsedReduceTime>
+ </taskAttempt>
+
+Task Attempt Counters API
+-------------------------
+
+With the task attempt counters API, you can object a collection of resources that represent al the counters for that task attempt.
+
+### URI
+
+ * http://<proxy http address:port>/proxy/{appid}/ws/v1/mapreduce/jobs/{jobid}/tasks/{taskid}/attempt/{attemptid}/counters
+
+### HTTP Operations Supported
+
+ * GET
+
+### Query Parameters Supported
+
+ None
+
+### Elements of the *jobTaskAttemptCounters* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| id | string | The task attempt id |
+| taskAttemptcounterGroup | array of task attempt counterGroup objects(JSON)/zero or more task attempt counterGroup objects(XML) | A collection of task attempt counter group objects |
+
+### Elements of the *taskAttemptCounterGroup* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| counterGroupName | string | The name of the counter group |
+| counter | array of counter objects(JSON)/zero or more counter objects(XML) | A collection of counter objects |
+
+### Elements of the *counter* object
+
+| Item | Data Type | Description |
+|:---- |:---- |:---- |
+| name | string | The name of the counter |
+| value | long | The value of the counter |
+
+### Response Examples
+
+**JSON response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/json
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ {
+ "jobTaskAttemptCounters" : {
+ "taskAttemptCounterGroup" : [
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.FileSystemCounter",
+ "counter" : [
+ {
+ "value" : 2363,
+ "name" : "FILE_BYTES_READ"
+ },
+ {
+ "value" : 54372,
+ "name" : "FILE_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FILE_WRITE_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_READ"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_BYTES_WRITTEN"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_LARGE_READ_OPS"
+ },
+ {
+ "value" : 0,
+ "name" : "HDFS_WRITE_OPS"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.TaskCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "COMBINE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "COMBINE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_GROUPS"
+ },
+ {
+ "value" : 2235,
+ "name" : "REDUCE_SHUFFLE_BYTES"
+ },
+ {
+ "value" : 460,
+ "name" : "REDUCE_INPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "REDUCE_OUTPUT_RECORDS"
+ },
+ {
+ "value" : 0,
+ "name" : "SPILLED_RECORDS"
+ },
+ {
+ "value" : 1,
+ "name" : "SHUFFLED_MAPS"
+ },
+ {
+ "value" : 0,
+ "name" : "FAILED_SHUFFLE"
+ },
+ {
+ "value" : 1,
+ "name" : "MERGED_MAP_OUTPUTS"
+ },
+ {
+ "value" : 26,
+ "name" : "GC_TIME_MILLIS"
+ },
+ {
+ "value" : 860,
+ "name" : "CPU_MILLISECONDS"
+ },
+ {
+ "value" : 107839488,
+ "name" : "PHYSICAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 1123147776,
+ "name" : "VIRTUAL_MEMORY_BYTES"
+ },
+ {
+ "value" : 57475072,
+ "name" : "COMMITTED_HEAP_BYTES"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "Shuffle Errors",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BAD_ID"
+ },
+ {
+ "value" : 0,
+ "name" : "CONNECTION"
+ },
+ {
+ "value" : 0,
+ "name" : "IO_ERROR"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_LENGTH"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_MAP"
+ },
+ {
+ "value" : 0,
+ "name" : "WRONG_REDUCE"
+ }
+ ]
+ },
+ {
+ "counterGroupName" : "org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter",
+ "counter" : [
+ {
+ "value" : 0,
+ "name" : "BYTES_WRITTEN"
+ }
+ ]
+ }
+ ],
+ "id" : "attempt_1326232085508_4_4_r_0_0"
+ }
+ }
+
+**XML response**
+
+HTTP Request:
+
+ GET http://<proxy http address:port>/proxy/application_1326232085508_0004/ws/v1/mapreduce/jobs/job_1326232085508_4_4/tasks/task_1326232085508_4_4_r_0/attempts/attempt_1326232085508_4_4_r_0_0/counters
+ Accept: application/xml
+
+Response Header:
+
+ HTTP/1.1 200 OK
+ Content-Type: application/xml
+ Content-Length: 2735
+ Server: Jetty(6.1.26)
+
+Response Body:
+
+ <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+ <jobTaskAttemptCounters>
+ <id>attempt_1326232085508_4_4_r_0_0</id>
+ <taskAttemptCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.FileSystemCounter</counterGroupName>
+ <counter>
+ <name>FILE_BYTES_READ</name>
+ <value>2363</value>
+ </counter>
+ <counter>
+ <name>FILE_BYTES_WRITTEN</name>
+ <value>54372</value>
+ </counter>
+ <counter>
+ <name>FILE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>FILE_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_READ</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_LARGE_READ_OPS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>HDFS_WRITE_OPS</name>
+ <value>0</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ <taskAttemptCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.TaskCounter</counterGroupName>
+ <counter>
+ <name>COMBINE_INPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>COMBINE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_GROUPS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_SHUFFLE_BYTES</name>
+ <value>2235</value>
+ </counter>
+ <counter>
+ <name>REDUCE_INPUT_RECORDS</name>
+ <value>460</value>
+ </counter>
+ <counter>
+ <name>REDUCE_OUTPUT_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SPILLED_RECORDS</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>SHUFFLED_MAPS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>FAILED_SHUFFLE</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>MERGED_MAP_OUTPUTS</name>
+ <value>1</value>
+ </counter>
+ <counter>
+ <name>GC_TIME_MILLIS</name>
+ <value>26</value>
+ </counter>
+ <counter>
+ <name>CPU_MILLISECONDS</name>
+ <value>860</value>
+ </counter>
+ <counter>
+ <name>PHYSICAL_MEMORY_BYTES</name>
+ <value>107839488</value>
+ </counter>
+ <counter>
+ <name>VIRTUAL_MEMORY_BYTES</name>
+ <value>1123147776</value>
+ </counter>
+ <counter>
+ <name>COMMITTED_HEAP_BYTES</name>
+ <value>57475072</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ <taskAttemptCounterGroup>
+ <counterGroupName>Shuffle Errors</counterGroupName>
+ <counter>
+ <name>BAD_ID</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>CONNECTION</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>IO_ERROR</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_LENGTH</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_MAP</name>
+ <value>0</value>
+ </counter>
+ <counter>
+ <name>WRONG_REDUCE</name>
+ <value>0</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ <taskAttemptCounterGroup>
+ <counterGroupName>org.apache.hadoop.mapreduce.lib.output.FileOutputFormatCounter</counterGroupName>
+ <counter>
+ <name>BYTES_WRITTEN</name>
+ <value>0</value>
+ </counter>
+ </taskAttemptCounterGroup>
+ </jobTaskAttemptCounters>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8b787e2f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
new file mode 100644
index 0000000..ab0dc9d
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/site/markdown/MapredCommands.md
@@ -0,0 +1,153 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+MapReduce Commands Guide
+========================
+
+* [Overview](#Overview)
+* [User Commands](#User_Commands)
+ * [archive](#archive)
+ * [classpath](#classpath)
+ * [distcp](#distcp)
+ * [job](#job)
+ * [pipes](#pipes)
+ * [queue](#queue)
+ * [version](#version)
+* [Administration Commands](#Administration_Commands)
+ * [historyserver](#historyserver)
+ * [hsadmin](#hsadmin)
+
+Overview
+--------
+
+All mapreduce commands are invoked by the `bin/mapred` script. Running the mapred script without any arguments prints the description for all commands.
+
+Usage: `mapred [SHELL_OPTIONS] COMMAND [GENERIC_OPTIONS] [COMMAND_OPTIONS]`
+
+Hadoop has an option parsing framework that employs parsing generic options as well as running classes.
+
+| COMMAND\_OPTIONS | Description |
+|:---- |:---- |
+| SHELL\_OPTIONS | The common set of shell options. These are documented on the [Hadoop Commands Reference](../../hadoop-project-dist/hadoop-common/CommandsManual.html#Shell_Options) page. |
+| GENERIC\_OPTIONS | The common set of options supported by multiple commands. See the [Hadoop Commands Reference](../../hadoop-project-dist/hadoop-common/CommandsManual.html#Generic_Options) for more information. |
+| COMMAND COMMAND\_OPTIONS | Various commands with their options are described in the following sections. The commands have been grouped into [User Commands](#User_Commands) and [Administration Commands](#Administration_Commands). |
+
+User Commands
+-------------
+
+Commands useful for users of a hadoop cluster.
+
+### `archive`
+
+Creates a hadoop archive. More information can be found at
+[Hadoop Archives Guide](./HadoopArchives.html).
+
+### `classpath`
+
+Prints the class path needed to get the Hadoop jar and the required libraries.
+
+Usage: `mapred classpath`
+
+### `distcp`
+
+Copy file or directories recursively. More information can be found at
+[Hadoop DistCp Guide](./DistCp.html).
+
+### `job`
+
+Command to interact with Map Reduce Jobs.
+
+Usage: `mapred job | [GENERIC_OPTIONS] | [-submit <job-file>] | [-status <job-id>] | [-counter <job-id> <group-name> <counter-name>] | [-kill <job-id>] | [-events <job-id> <from-event-#> <#-of-events>] | [-history [all] <jobOutputDir>] | [-list [all]] | [-kill-task <task-id>] | [-fail-task <task-id>] | [-set-priority <job-id> <priority>]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| -submit *job-file* | Submits the job. |
+| -status *job-id* | Prints the map and reduce completion percentage and all job counters. |
+| -counter *job-id* *group-name* *counter-name* | Prints the counter value. |
+| -kill *job-id* | Kills the job. |
+| -events *job-id* *from-event-\#* *\#-of-events* | Prints the events' details received by jobtracker for the given range. |
+| -history [all]*jobOutputDir* | Prints job details, failed and killed tip details. More details about the job such as successful tasks and task attempts made for each task can be viewed by specifying the [all] option. |
+| -list [all] | Displays jobs which are yet to complete. `-list all` displays all jobs. |
+| -kill-task *task-id* | Kills the task. Killed tasks are NOT counted against failed attempts. |
+| -fail-task *task-id* | Fails the task. Failed tasks are counted against failed attempts. |
+| -set-priority *job-id* *priority* | Changes the priority of the job. Allowed priority values are VERY\_HIGH, HIGH, NORMAL, LOW, VERY\_LOW |
+
+### `pipes`
+
+Runs a pipes job.
+
+Usage: `mapred pipes [-conf <path>] [-jobconf <key=value>, <key=value>, ...] [-input <path>] [-output <path>] [-jar <jar file>] [-inputformat <class>] [-map <class>] [-partitioner <class>] [-reduce <class>] [-writer <class>] [-program <executable>] [-reduces <num>]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| -conf *path* | Configuration for job |
+| -jobconf *key=value*, *key=value*, ... | Add/override configuration for job |
+| -input *path* | Input directory |
+| -output *path* | Output directory |
+| -jar *jar file* | Jar filename |
+| -inputformat *class* | InputFormat class |
+| -map *class* | Java Map class |
+| -partitioner *class* | Java Partitioner |
+| -reduce *class* | Java Reduce class |
+| -writer *class* | Java RecordWriter |
+| -program *executable* | Executable URI |
+| -reduces *num* | Number of reduces |
+
+### `queue`
+
+command to interact and view Job Queue information
+
+Usage: `mapred queue [-list] | [-info <job-queue-name> [-showJobs]] | [-showacls]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| -list | Gets list of Job Queues configured in the system. Along with scheduling information associated with the job queues. |
+| -info *job-queue-name* [-showJobs] | Displays the job queue information and associated scheduling information of particular job queue. If `-showJobs` options is present a list of jobs submitted to the particular job queue is displayed. |
+| -showacls | Displays the queue name and associated queue operations allowed for the current user. The list consists of only those queues to which the user has access. |
+
+### `version`
+
+Prints the version.
+
+Usage: `mapred version`
+
+Administration Commands
+-----------------------
+
+Commands useful for administrators of a hadoop cluster.
+
+### `historyserver`
+
+Start JobHistoryServer.
+
+Usage: `mapred historyserver`
+
+### `hsadmin`
+
+Runs a MapReduce hsadmin client for execute JobHistoryServer administrative commands.
+
+Usage: `mapred hsadmin [-refreshUserToGroupsMappings] | [-refreshSuperUserGroupsConfiguration] | [-refreshAdminAcls] | [-refreshLoadedJobCache] | [-refreshLogRetentionSettings] | [-refreshJobRetentionSettings] | [-getGroups [username]] | [-help [cmd]]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| -refreshUserToGroupsMappings | Refresh user-to-groups mappings |
+| -refreshSuperUserGroupsConfiguration | Refresh superuser proxy groups mappings |
+| -refreshAdminAcls | Refresh acls for administration of Job history server |
+| -refreshLoadedJobCache | Refresh loaded job cache of Job history server |
+| -refreshJobRetentionSettings | Refresh job history period, job cleaner settings |
+| -refreshLogRetentionSettings | Refresh log retention period and log retention check interval |
+| -getGroups [username] | Get the groups which given user belongs to |
+| -help [cmd] | Displays help for the given command or all commands if none is specified. |
+
+
[17/50] [abbrv] hadoop git commit: HDFS-4266. BKJM: Separate write
and ack quorum (Rakesh R via umamahesh)
Posted by zj...@apache.org.
HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0412de1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0412de1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0412de1
Branch: refs/heads/YARN-2928
Commit: f0412de1c1d42b3c2a92531f81d97a24df920523
Parents: f24a567
Author: Uma Maheswara Rao G <um...@apache.org>
Authored: Tue Feb 17 21:28:49 2015 +0530
Committer: Uma Maheswara Rao G <um...@apache.org>
Committed: Tue Feb 17 21:28:49 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../bkjournal/BookKeeperJournalManager.java | 15 +-
.../bkjournal/TestBookKeeperJournalManager.java | 153 ++++++++++++++++++-
3 files changed, 163 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0412de1/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fcf5994..f28e41e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -639,6 +639,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7797. Add audit log for setQuota operation (Rakesh R via umamahesh)
+ HDFS-4266. BKJM: Separate write and ack quorum (Rakesh R via umamahesh)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0412de1/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
index 51905c0..89fa84c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/main/java/org/apache/hadoop/contrib/bkjournal/BookKeeperJournalManager.java
@@ -152,6 +152,13 @@ public class BookKeeperJournalManager implements JournalManager {
= "dfs.namenode.bookkeeperjournal.readEntryTimeoutSec";
public static final int BKJM_BOOKKEEPER_READ_ENTRY_TIMEOUT_DEFAULT = 5;
+ public static final String BKJM_BOOKKEEPER_ACK_QUORUM_SIZE
+ = "dfs.namenode.bookkeeperjournal.ack.quorum-size";
+
+ public static final String BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC
+ = "dfs.namenode.bookkeeperjournal.addEntryTimeoutSec";
+ public static final int BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_DEFAULT = 5;
+
private ZooKeeper zkc;
private final Configuration conf;
private final BookKeeper bkc;
@@ -162,6 +169,8 @@ public class BookKeeperJournalManager implements JournalManager {
private final MaxTxId maxTxId;
private final int ensembleSize;
private final int quorumSize;
+ private final int ackQuorumSize;
+ private final int addEntryTimeout;
private final String digestpw;
private final int speculativeReadTimeout;
private final int readEntryTimeout;
@@ -184,6 +193,9 @@ public class BookKeeperJournalManager implements JournalManager {
BKJM_BOOKKEEPER_ENSEMBLE_SIZE_DEFAULT);
quorumSize = conf.getInt(BKJM_BOOKKEEPER_QUORUM_SIZE,
BKJM_BOOKKEEPER_QUORUM_SIZE_DEFAULT);
+ ackQuorumSize = conf.getInt(BKJM_BOOKKEEPER_ACK_QUORUM_SIZE, quorumSize);
+ addEntryTimeout = conf.getInt(BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC,
+ BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_DEFAULT);
speculativeReadTimeout = conf.getInt(
BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_MS,
BKJM_BOOKKEEPER_SPECULATIVE_READ_TIMEOUT_DEFAULT);
@@ -216,6 +228,7 @@ public class BookKeeperJournalManager implements JournalManager {
ClientConfiguration clientConf = new ClientConfiguration();
clientConf.setSpeculativeReadTimeout(speculativeReadTimeout);
clientConf.setReadEntryTimeout(readEntryTimeout);
+ clientConf.setAddEntryTimeout(addEntryTimeout);
bkc = new BookKeeper(clientConf, zkc);
} catch (KeeperException e) {
throw new IOException("Error initializing zk", e);
@@ -403,7 +416,7 @@ public class BookKeeperJournalManager implements JournalManager {
// bookkeeper errored on last stream, clean up ledger
currentLedger.close();
}
- currentLedger = bkc.createLedger(ensembleSize, quorumSize,
+ currentLedger = bkc.createLedger(ensembleSize, quorumSize, ackQuorumSize,
BookKeeper.DigestType.MAC,
digestpw.getBytes(Charsets.UTF_8));
} catch (BKException bke) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0412de1/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
index 44e4bef..07fcd72 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/bkjournal/src/test/java/org/apache/hadoop/contrib/bkjournal/TestBookKeeperJournalManager.java
@@ -32,6 +32,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Random;
+import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executors;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Callable;
@@ -67,6 +68,7 @@ public class TestBookKeeperJournalManager {
private ZooKeeper zkc;
private static BKJMUtil bkutil;
static int numBookies = 3;
+ private BookieServer newBookie;
@BeforeClass
public static void setupBookkeeper() throws Exception {
@@ -87,6 +89,9 @@ public class TestBookKeeperJournalManager {
@After
public void teardown() throws Exception {
zkc.close();
+ if (newBookie != null) {
+ newBookie.shutdown();
+ }
}
private NamespaceInfo newNSInfo() {
@@ -377,7 +382,8 @@ public class TestBookKeeperJournalManager {
*/
@Test
public void testAllBookieFailure() throws Exception {
- BookieServer bookieToFail = bkutil.newBookie();
+ // bookie to fail
+ newBookie = bkutil.newBookie();
BookieServer replacementBookie = null;
try {
@@ -408,7 +414,7 @@ public class TestBookKeeperJournalManager {
}
out.setReadyToFlush();
out.flush();
- bookieToFail.shutdown();
+ newBookie.shutdown();
assertEquals("New bookie didn't die",
numBookies, bkutil.checkBookiesUp(numBookies, 10));
@@ -449,7 +455,7 @@ public class TestBookKeeperJournalManager {
if (replacementBookie != null) {
replacementBookie.shutdown();
}
- bookieToFail.shutdown();
+ newBookie.shutdown();
if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) {
LOG.warn("Not all bookies from this test shut down, expect errors");
@@ -464,7 +470,7 @@ public class TestBookKeeperJournalManager {
*/
@Test
public void testOneBookieFailure() throws Exception {
- BookieServer bookieToFail = bkutil.newBookie();
+ newBookie = bkutil.newBookie();
BookieServer replacementBookie = null;
try {
@@ -500,7 +506,7 @@ public class TestBookKeeperJournalManager {
replacementBookie = bkutil.newBookie();
assertEquals("replacement bookie didn't start",
ensembleSize+1, bkutil.checkBookiesUp(ensembleSize+1, 10));
- bookieToFail.shutdown();
+ newBookie.shutdown();
assertEquals("New bookie didn't die",
ensembleSize, bkutil.checkBookiesUp(ensembleSize, 10));
@@ -518,7 +524,7 @@ public class TestBookKeeperJournalManager {
if (replacementBookie != null) {
replacementBookie.shutdown();
}
- bookieToFail.shutdown();
+ newBookie.shutdown();
if (bkutil.checkBookiesUp(numBookies, 30) != numBookies) {
LOG.warn("Not all bookies from this test shut down, expect errors");
@@ -822,6 +828,141 @@ public class TestBookKeeperJournalManager {
assertTrue("No thread managed to complete formatting", numCompleted > 0);
}
+ @Test(timeout = 120000)
+ public void testDefaultAckQuorum() throws Exception {
+ newBookie = bkutil.newBookie();
+ int ensembleSize = numBookies + 1;
+ int quorumSize = numBookies + 1;
+ // ensure that the journal manager has to use all bookies,
+ // so that a failure will fail the journal manager
+ Configuration conf = new Configuration();
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE,
+ ensembleSize);
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE,
+ quorumSize);
+ // sets 2 secs
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC,
+ 2);
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
+ BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi);
+ bkjm.format(nsi);
+ CountDownLatch sleepLatch = new CountDownLatch(1);
+ sleepBookie(sleepLatch, newBookie);
+
+ EditLogOutputStream out = bkjm.startLogSegment(1,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ int numTransactions = 100;
+ for (long i = 1; i <= numTransactions; i++) {
+ FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
+ op.setTransactionId(i);
+ out.write(op);
+ }
+ try {
+ out.close();
+ bkjm.finalizeLogSegment(1, numTransactions);
+
+ List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
+ bkjm.selectInputStreams(in, 1, true);
+ try {
+ assertEquals(numTransactions,
+ FSEditLogTestUtil.countTransactionsInStream(in.get(0)));
+ } finally {
+ in.get(0).close();
+ }
+ fail("Should throw exception as not enough non-faulty bookies available!");
+ } catch (IOException ioe) {
+ // expected
+ }
+ }
+
+ /**
+ * Test ack quorum feature supported by bookkeeper. Keep ack quorum bookie
+ * alive and sleep all the other bookies. Now the client would wait for the
+ * acknowledgement from the ack size bookies and after receiving the success
+ * response will continue writing. Non ack client will hang long time to add
+ * entries.
+ */
+ @Test(timeout = 120000)
+ public void testAckQuorum() throws Exception {
+ // slow bookie
+ newBookie = bkutil.newBookie();
+ // make quorum size and ensemble size same to avoid the interleave writing
+ // of the ledger entries
+ int ensembleSize = numBookies + 1;
+ int quorumSize = numBookies + 1;
+ int ackSize = numBookies;
+ // ensure that the journal manager has to use all bookies,
+ // so that a failure will fail the journal manager
+ Configuration conf = new Configuration();
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ENSEMBLE_SIZE,
+ ensembleSize);
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_QUORUM_SIZE,
+ quorumSize);
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ACK_QUORUM_SIZE,
+ ackSize);
+ // sets 60 minutes
+ conf.setInt(BookKeeperJournalManager.BKJM_BOOKKEEPER_ADD_ENTRY_TIMEOUT_SEC,
+ 3600);
+
+ NamespaceInfo nsi = newNSInfo();
+ BookKeeperJournalManager bkjm = new BookKeeperJournalManager(conf,
+ BKJMUtil.createJournalURI("/hdfsjournal-onebookiefailure"), nsi);
+ bkjm.format(nsi);
+ CountDownLatch sleepLatch = new CountDownLatch(1);
+ sleepBookie(sleepLatch, newBookie);
+
+ EditLogOutputStream out = bkjm.startLogSegment(1,
+ NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION);
+ int numTransactions = 100;
+ for (long i = 1; i <= numTransactions; i++) {
+ FSEditLogOp op = FSEditLogTestUtil.getNoOpInstance();
+ op.setTransactionId(i);
+ out.write(op);
+ }
+ out.close();
+ bkjm.finalizeLogSegment(1, numTransactions);
+
+ List<EditLogInputStream> in = new ArrayList<EditLogInputStream>();
+ bkjm.selectInputStreams(in, 1, true);
+ try {
+ assertEquals(numTransactions,
+ FSEditLogTestUtil.countTransactionsInStream(in.get(0)));
+ } finally {
+ sleepLatch.countDown();
+ in.get(0).close();
+ bkjm.close();
+ }
+ }
+
+ /**
+ * Sleep a bookie until I count down the latch
+ *
+ * @param latch
+ * Latch to wait on
+ * @param bookie
+ * bookie server
+ * @throws Exception
+ */
+ private void sleepBookie(final CountDownLatch l, final BookieServer bookie)
+ throws Exception {
+
+ Thread sleeper = new Thread() {
+ public void run() {
+ try {
+ bookie.suspendProcessing();
+ l.await(60, TimeUnit.SECONDS);
+ bookie.resumeProcessing();
+ } catch (Exception e) {
+ LOG.error("Error suspending bookie", e);
+ }
+ }
+ };
+ sleeper.setName("BookieServerSleeper-" + bookie.getBookie().getId());
+ sleeper.start();
+ }
+
+
private String startAndFinalizeLogSegment(BookKeeperJournalManager bkjm,
int startTxid, int endTxid) throws IOException, KeeperException,
InterruptedException {
[49/50] [abbrv] hadoop git commit: HDFS-7772. Document hdfs balancer
-exclude/-include option in HDFSCommands.html. Contributed by Xiaoyu Yao.
Posted by zj...@apache.org.
HDFS-7772. Document hdfs balancer -exclude/-include option in HDFSCommands.html. Contributed by Xiaoyu Yao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2aa9979a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2aa9979a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2aa9979a
Branch: refs/heads/YARN-2928
Commit: 2aa9979a713ab79853885264ad7739c48226aaa4
Parents: f5da556
Author: cnauroth <cn...@apache.org>
Authored: Wed Feb 18 11:46:57 2015 -0800
Committer: cnauroth <cn...@apache.org>
Committed: Wed Feb 18 12:03:07 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/balancer/Balancer.java | 7 +++----
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 17 +++++++++++++----
3 files changed, 19 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa9979a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ec1c837..70eae1c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -653,6 +653,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7804. correct the haadmin command usage in #HDFSHighAvailabilityWithQJM.html
(Brahma Reddy Battula via umamahesh)
+ HDFS-7772. Document hdfs balancer -exclude/-include option in
+ HDFSCommands.html (Xiaoyu Yao via cnauroth)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa9979a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 5b87cb5..71338e6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -170,15 +170,14 @@ public class Balancer {
private static final long GB = 1L << 30; //1GB
private static final long MAX_SIZE_TO_MOVE = 10*GB;
- private static final String USAGE = "Usage: java "
- + Balancer.class.getSimpleName()
+ private static final String USAGE = "Usage: hdfs balancer"
+ "\n\t[-policy <policy>]\tthe balancing policy: "
+ BalancingPolicy.Node.INSTANCE.getName() + " or "
+ BalancingPolicy.Pool.INSTANCE.getName()
+ "\n\t[-threshold <threshold>]\tPercentage of disk capacity"
- + "\n\t[-exclude [-f <hosts-file> | comma-sperated list of hosts]]"
+ + "\n\t[-exclude [-f <hosts-file> | <comma-separated list of hosts>]]"
+ "\tExcludes the specified datanodes."
- + "\n\t[-include [-f <hosts-file> | comma-sperated list of hosts]]"
+ + "\n\t[-include [-f <hosts-file> | <comma-separated list of hosts>]]"
+ "\tIncludes only the specified datanodes."
+ "\n\t[-idleiterations <idleiterations>]"
+ "\tNumber of consecutive idle iterations (-1 for Infinite) before exit.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2aa9979a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 6a7f34c..0573158 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -245,13 +245,22 @@ Commands useful for administrators of a hadoop cluster.
### `balancer`
-Usage: `hdfs balancer [-threshold <threshold>] [-policy <policy>] [-idleiterations <idleiterations>]`
+Usage:
+
+ hdfs balancer
+ [-threshold <threshold>]
+ [-policy <policy>]
+ [-exclude [-f <hosts-file> | <comma-separated list of hosts>]]
+ [-include [-f <hosts-file> | <comma-separated list of hosts>]]
+ [-idleiterations <idleiterations>]
| COMMAND\_OPTION | Description |
|:---- |:---- |
-| `-policy` *policy* | `datanode` (default): Cluster is balanced if each datanode is balanced.<br/> `blockpool`: Cluster is balanced if each block pool in each datanode is balanced. |
-| `-threshold` *threshold* | Percentage of disk capacity. This overwrites the default threshold. |
-| `-idleiterations` *iterations* | Maximum number of idle iterations before exit. This overwrites the default idleiterations(5). |
+| `-policy` \<policy\> | `datanode` (default): Cluster is balanced if each datanode is balanced.<br/> `blockpool`: Cluster is balanced if each block pool in each datanode is balanced. |
+| `-threshold` \<threshold\> | Percentage of disk capacity. This overwrites the default threshold. |
+| `-exclude -f` \<hosts-file\> \| \<comma-separated list of hosts\> | Excludes the specified datanodes from being balanced by the balancer. |
+| `-include -f` \<hosts-file\> \| \<comma-separated list of hosts\> | Includes only the specified datanodes to be balanced by the balancer. |
+| `-idleiterations` \<iterations\> | Maximum number of idle iterations before exit. This overwrites the default idleiterations(5). |
Runs a cluster balancing utility. An administrator can simply press Ctrl-C to stop the rebalancing process. See [Balancer](./HdfsUserGuide.html#Balancer) for more details.
[43/50] [abbrv] hadoop git commit: HADOOP-11593. Convert site
documentation from apt to markdown (stragglers) (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
HADOOP-11593. Convert site documentation from apt to markdown (stragglers) (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6fc1f3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6fc1f3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6fc1f3e
Branch: refs/heads/YARN-2928
Commit: b6fc1f3e4355be913b7d4f6ccd48c0c26b66d039
Parents: 7c78204
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 17 21:30:24 2015 -1000
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 17 21:30:24 2015 -1000
----------------------------------------------------------------------
.../hadoop-auth/src/site/apt/BuildingIt.apt.vm | 70 --
.../src/site/apt/Configuration.apt.vm | 377 -------
.../hadoop-auth/src/site/apt/Examples.apt.vm | 133 ---
.../hadoop-auth/src/site/apt/index.apt.vm | 59 -
.../hadoop-auth/src/site/markdown/BuildingIt.md | 56 +
.../src/site/markdown/Configuration.md | 341 ++++++
.../hadoop-auth/src/site/markdown/Examples.md | 109 ++
.../hadoop-auth/src/site/markdown/index.md | 43 +
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../hadoop-kms/src/site/apt/index.apt.vm | 1020 ------------------
.../hadoop-kms/src/site/markdown/index.md.vm | 864 +++++++++++++++
hadoop-project/src/site/apt/index.apt.vm | 73 --
hadoop-project/src/site/markdown/index.md.vm | 72 ++
.../hadoop-openstack/src/site/apt/index.apt.vm | 686 ------------
.../hadoop-openstack/src/site/markdown/index.md | 544 ++++++++++
.../src/site/resources/css/site.css | 30 +
.../src/site/apt/SchedulerLoadSimulator.apt.vm | 439 --------
.../src/site/markdown/SchedulerLoadSimulator.md | 357 ++++++
.../src/site/apt/HadoopStreaming.apt.vm | 792 --------------
.../src/site/markdown/HadoopStreaming.md.vm | 559 ++++++++++
20 files changed, 2978 insertions(+), 3649 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm
deleted file mode 100644
index 2ca2f0a..0000000
--- a/hadoop-common-project/hadoop-auth/src/site/apt/BuildingIt.apt.vm
+++ /dev/null
@@ -1,70 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Auth, Java HTTP SPNEGO ${project.version} - Building It
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Auth, Java HTTP SPNEGO ${project.version} - Building It
-
-* Requirements
-
- * Java 6+
-
- * Maven 3+
-
- * Kerberos KDC (for running Kerberos test cases)
-
-* Building
-
- Use Maven goals: clean, test, compile, package, install
-
- Available profiles: docs, testKerberos
-
-* Testing
-
- By default Kerberos testcases are not run.
-
- The requirements to run Kerberos testcases are a running KDC, a keytab
- file with a client principal and a kerberos principal.
-
- To run Kerberos tescases use the <<<testKerberos>>> Maven profile:
-
-+---+
-$ mvn test -PtestKerberos
-+---+
-
- The following Maven <<<-D>>> options can be used to change the default
- values:
-
- * <<<hadoop-auth.test.kerberos.realm>>>: default value <<LOCALHOST>>
-
- * <<<hadoop-auth.test.kerberos.client.principal>>>: default value <<client>>
-
- * <<<hadoop-auth.test.kerberos.server.principal>>>: default value
- <<HTTP/localhost>> (it must start 'HTTP/')
-
- * <<<hadoop-auth.test.kerberos.keytab.file>>>: default value
- <<${HOME}/${USER}.keytab>>
-
-** Generating Documentation
-
- To create the documentation use the <<<docs>>> Maven profile:
-
-+---+
-$ mvn package -Pdocs
-+---+
-
- The generated documentation is available at
- <<<hadoop-auth/target/site/>>>.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm
deleted file mode 100644
index 88248e5..0000000
--- a/hadoop-common-project/hadoop-auth/src/site/apt/Configuration.apt.vm
+++ /dev/null
@@ -1,377 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Auth, Java HTTP SPNEGO ${project.version} - Server Side
- Configuration
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Auth, Java HTTP SPNEGO ${project.version} - Server Side
-Configuration
-
-* Server Side Configuration Setup
-
- The AuthenticationFilter filter is Hadoop Auth's server side component.
-
- This filter must be configured in front of all the web application resources
- that required authenticated requests. For example:
-
- The Hadoop Auth and dependent JAR files must be in the web application
- classpath (commonly the <<<WEB-INF/lib>>> directory).
-
- Hadoop Auth uses SLF4J-API for logging. Auth Maven POM dependencies define
- the SLF4J API dependency but it does not define the dependency on a concrete
- logging implementation, this must be addded explicitly to the web
- application. For example, if the web applicationan uses Log4j, the
- SLF4J-LOG4J12 and LOG4J jar files must be part part of the web application
- classpath as well as the Log4j configuration file.
-
-** Common Configuration parameters
-
- * <<<config.prefix>>>: If specified, all other configuration parameter names
- must start with the prefix. The default value is no prefix.
-
- * <<<[PREFIX.]type>>>: the authentication type keyword (<<<simple>>> or
- <<<kerberos>>>) or a Authentication handler implementation.
-
- * <<<[PREFIX.]signature.secret>>>: When <<<signer.secret.provider>>> is set to
- <<<string>>> or not specified, this is the value for the secret used to sign
- the HTTP cookie.
-
- * <<<[PREFIX.]token.validity>>>: The validity -in seconds- of the generated
- authentication token. The default value is <<<3600>>> seconds. This is also
- used for the rollover interval when <<<signer.secret.provider>>> is set to
- <<<random>>> or <<<zookeeper>>>.
-
- * <<<[PREFIX.]cookie.domain>>>: domain to use for the HTTP cookie that stores
- the authentication token.
-
- * <<<[PREFIX.]cookie.path>>>: path to use for the HTTP cookie that stores the
- authentication token.
-
- * <<<signer.secret.provider>>>: indicates the name of the SignerSecretProvider
- class to use. Possible values are: <<<string>>>, <<<random>>>,
- <<<zookeeper>>>, or a classname. If not specified, the <<<string>>>
- implementation will be used; and failing that, the <<<random>>>
- implementation will be used.
-
-** Kerberos Configuration
-
- <<IMPORTANT>>: A KDC must be configured and running.
-
- To use Kerberos SPNEGO as the authentication mechanism, the authentication
- filter must be configured with the following init parameters:
-
- * <<<[PREFIX.]type>>>: the keyword <<<kerberos>>>.
-
- * <<<[PREFIX.]kerberos.principal>>>: The web-application Kerberos principal
- name. The Kerberos principal name must start with <<<HTTP/...>>>. For
- example: <<<...@LOCALHOST>>>. There is no default value.
-
- * <<<[PREFIX.]kerberos.keytab>>>: The path to the keytab file containing
- the credentials for the kerberos principal. For example:
- <<</Users/tucu/tucu.keytab>>>. There is no default value.
-
- <<Example>>:
-
-+---+
-<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
- ...
-
- <filter>
- <filter-name>kerberosFilter</filter-name>
- <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
- <init-param>
- <param-name>type</param-name>
- <param-value>kerberos</param-value>
- </init-param>
- <init-param>
- <param-name>token.validity</param-name>
- <param-value>30</param-value>
- </init-param>
- <init-param>
- <param-name>cookie.domain</param-name>
- <param-value>.foo.com</param-value>
- </init-param>
- <init-param>
- <param-name>cookie.path</param-name>
- <param-value>/</param-value>
- </init-param>
- <init-param>
- <param-name>kerberos.principal</param-name>
- <param-value>HTTP/localhost@LOCALHOST</param-value>
- </init-param>
- <init-param>
- <param-name>kerberos.keytab</param-name>
- <param-value>/tmp/auth.keytab</param-value>
- </init-param>
- </filter>
-
- <filter-mapping>
- <filter-name>kerberosFilter</filter-name>
- <url-pattern>/kerberos/*</url-pattern>
- </filter-mapping>
-
- ...
-</web-app>
-+---+
-
-** Pseudo/Simple Configuration
-
- To use Pseudo/Simple as the authentication mechanism (trusting the value of
- the query string parameter 'user.name'), the authentication filter must be
- configured with the following init parameters:
-
- * <<<[PREFIX.]type>>>: the keyword <<<simple>>>.
-
- * <<<[PREFIX.]simple.anonymous.allowed>>>: is a boolean parameter that
- indicates if anonymous requests are allowed or not. The default value is
- <<<false>>>.
-
- <<Example>>:
-
-+---+
-<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
- ...
-
- <filter>
- <filter-name>simpleFilter</filter-name>
- <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
- <init-param>
- <param-name>type</param-name>
- <param-value>simple</param-value>
- </init-param>
- <init-param>
- <param-name>token.validity</param-name>
- <param-value>30</param-value>
- </init-param>
- <init-param>
- <param-name>cookie.domain</param-name>
- <param-value>.foo.com</param-value>
- </init-param>
- <init-param>
- <param-name>cookie.path</param-name>
- <param-value>/</param-value>
- </init-param>
- <init-param>
- <param-name>simple.anonymous.allowed</param-name>
- <param-value>false</param-value>
- </init-param>
- </filter>
-
- <filter-mapping>
- <filter-name>simpleFilter</filter-name>
- <url-pattern>/simple/*</url-pattern>
- </filter-mapping>
-
- ...
-</web-app>
-+---+
-
-** AltKerberos Configuration
-
- <<IMPORTANT>>: A KDC must be configured and running.
-
- The AltKerberos authentication mechanism is a partially implemented derivative
- of the Kerberos SPNEGO authentication mechanism which allows a "mixed" form of
- authentication where Kerberos SPNEGO is used by non-browsers while an
- alternate form of authentication (to be implemented by the user) is used for
- browsers. To use AltKerberos as the authentication mechanism (besides
- providing an implementation), the authentication filter must be configured
- with the following init parameters, in addition to the previously mentioned
- Kerberos SPNEGO ones:
-
- * <<<[PREFIX.]type>>>: the full class name of the implementation of
- AltKerberosAuthenticationHandler to use.
-
- * <<<[PREFIX.]alt-kerberos.non-browser.user-agents>>>: a comma-separated
- list of which user-agents should be considered non-browsers.
-
- <<Example>>:
-
-+---+
-<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
- ...
-
- <filter>
- <filter-name>kerberosFilter</filter-name>
- <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
- <init-param>
- <param-name>type</param-name>
- <param-value>org.my.subclass.of.AltKerberosAuthenticationHandler</param-value>
- </init-param>
- <init-param>
- <param-name>alt-kerberos.non-browser.user-agents</param-name>
- <param-value>java,curl,wget,perl</param-value>
- </init-param>
- <init-param>
- <param-name>token.validity</param-name>
- <param-value>30</param-value>
- </init-param>
- <init-param>
- <param-name>cookie.domain</param-name>
- <param-value>.foo.com</param-value>
- </init-param>
- <init-param>
- <param-name>cookie.path</param-name>
- <param-value>/</param-value>
- </init-param>
- <init-param>
- <param-name>kerberos.principal</param-name>
- <param-value>HTTP/localhost@LOCALHOST</param-value>
- </init-param>
- <init-param>
- <param-name>kerberos.keytab</param-name>
- <param-value>/tmp/auth.keytab</param-value>
- </init-param>
- </filter>
-
- <filter-mapping>
- <filter-name>kerberosFilter</filter-name>
- <url-pattern>/kerberos/*</url-pattern>
- </filter-mapping>
-
- ...
-</web-app>
-+---+
-
-** SignerSecretProvider Configuration
-
- The SignerSecretProvider is used to provide more advanced behaviors for the
- secret used for signing the HTTP Cookies.
-
- These are the relevant configuration properties:
-
- * <<<signer.secret.provider>>>: indicates the name of the
- SignerSecretProvider class to use. Possible values are: "string",
- "random", "zookeeper", or a classname. If not specified, the "string"
- implementation will be used; and failing that, the "random" implementation
- will be used.
-
- * <<<[PREFIX.]signature.secret>>>: When <<<signer.secret.provider>>> is set
- to <<<string>>> or not specified, this is the value for the secret used to
- sign the HTTP cookie.
-
- * <<<[PREFIX.]token.validity>>>: The validity -in seconds- of the generated
- authentication token. The default value is <<<3600>>> seconds. This is
- also used for the rollover interval when <<<signer.secret.provider>>> is
- set to <<<random>>> or <<<zookeeper>>>.
-
- The following configuration properties are specific to the <<<zookeeper>>>
- implementation:
-
- * <<<signer.secret.provider.zookeeper.connection.string>>>: Indicates the
- ZooKeeper connection string to connect with.
-
- * <<<signer.secret.provider.zookeeper.path>>>: Indicates the ZooKeeper path
- to use for storing and retrieving the secrets. All servers
- that need to coordinate their secret should point to the same path
-
- * <<<signer.secret.provider.zookeeper.auth.type>>>: Indicates the auth type
- to use. Supported values are <<<none>>> and <<<sasl>>>. The default
- value is <<<none>>>.
-
- * <<<signer.secret.provider.zookeeper.kerberos.keytab>>>: Set this to the
- path with the Kerberos keytab file. This is only required if using
- Kerberos.
-
- * <<<signer.secret.provider.zookeeper.kerberos.principal>>>: Set this to the
- Kerberos principal to use. This only required if using Kerberos.
-
- <<Example>>:
-
-+---+
-<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
- ...
-
- <filter>
- <!-- AuthenticationHandler configs not shown -->
- <init-param>
- <param-name>signer.secret.provider</param-name>
- <param-value>string</param-value>
- </init-param>
- <init-param>
- <param-name>signature.secret</param-name>
- <param-value>my_secret</param-value>
- </init-param>
- </filter>
-
- ...
-</web-app>
-+---+
-
- <<Example>>:
-
-+---+
-<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
- ...
-
- <filter>
- <!-- AuthenticationHandler configs not shown -->
- <init-param>
- <param-name>signer.secret.provider</param-name>
- <param-value>random</param-value>
- </init-param>
- <init-param>
- <param-name>token.validity</param-name>
- <param-value>30</param-value>
- </init-param>
- </filter>
-
- ...
-</web-app>
-+---+
-
- <<Example>>:
-
-+---+
-<web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
- ...
-
- <filter>
- <!-- AuthenticationHandler configs not shown -->
- <init-param>
- <param-name>signer.secret.provider</param-name>
- <param-value>zookeeper</param-value>
- </init-param>
- <init-param>
- <param-name>token.validity</param-name>
- <param-value>30</param-value>
- </init-param>
- <init-param>
- <param-name>signer.secret.provider.zookeeper.connection.string</param-name>
- <param-value>zoo1:2181,zoo2:2181,zoo3:2181</param-value>
- </init-param>
- <init-param>
- <param-name>signer.secret.provider.zookeeper.path</param-name>
- <param-value>/myapp/secrets</param-value>
- </init-param>
- <init-param>
- <param-name>signer.secret.provider.zookeeper.use.kerberos.acls</param-name>
- <param-value>true</param-value>
- </init-param>
- <init-param>
- <param-name>signer.secret.provider.zookeeper.kerberos.keytab</param-name>
- <param-value>/tmp/auth.keytab</param-value>
- </init-param>
- <init-param>
- <param-name>signer.secret.provider.zookeeper.kerberos.principal</param-name>
- <param-value>HTTP/localhost@LOCALHOST</param-value>
- </init-param>
- </filter>
-
- ...
-</web-app>
-+---+
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm
deleted file mode 100644
index 1b1afd5..0000000
--- a/hadoop-common-project/hadoop-auth/src/site/apt/Examples.apt.vm
+++ /dev/null
@@ -1,133 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Auth, Java HTTP SPNEGO ${project.version} - Examples
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Auth, Java HTTP SPNEGO ${project.version} - Examples
-
-* Accessing a Hadoop Auth protected URL Using a browser
-
- <<IMPORTANT:>> The browser must support HTTP Kerberos SPNEGO. For example,
- Firefox or Internet Explorer.
-
- For Firefox access the low level configuration page by loading the
- <<<about:config>>> page. Then go to the
- <<<network.negotiate-auth.trusted-uris>>> preference and add the hostname or
- the domain of the web server that is HTTP Kerberos SPNEGO protected (if using
- multiple domains and hostname use comma to separate them).
-
-* Accessing a Hadoop Auth protected URL Using <<<curl>>>
-
- <<IMPORTANT:>> The <<<curl>>> version must support GSS, run <<<curl -V>>>.
-
-+---+
-$ curl -V
-curl 7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8l zlib/1.2.3
-Protocols: tftp ftp telnet dict ldap http file https ftps
-Features: GSS-Negotiate IPv6 Largefile NTLM SSL libz
-+---+
-
- Login to the KDC using <<kinit>> and then use <<<curl>>> to fetch protected
- URL:
-
-+---+
-$ kinit
-Please enter the password for tucu@LOCALHOST:
-$ curl --negotiate -u foo -b ~/cookiejar.txt -c ~/cookiejar.txt http://localhost:8080/hadoop-auth-examples/kerberos/who
-Enter host password for user 'tucu':
-
-Hello Hadoop Auth Examples!
-+---+
-
- * The <<<--negotiate>>> option enables SPNEGO in <<<curl>>>.
-
- * The <<<-u foo>>> option is required but the user ignored (the principal
- that has been kinit-ed is used).
-
- * The <<<-b>>> and <<<-c>>> are use to store and send HTTP Cookies.
-
-* Using the Java Client
-
- Use the <<<AuthenticatedURL>>> class to obtain an authenticated HTTP
- connection:
-
-+---+
-...
-URL url = new URL("http://localhost:8080/hadoop-auth/kerberos/who");
-AuthenticatedURL.Token token = new AuthenticatedURL.Token();
-...
-HttpURLConnection conn = new AuthenticatedURL(url, token).openConnection();
-...
-conn = new AuthenticatedURL(url, token).openConnection();
-...
-+---+
-
-* Building and Running the Examples
-
- Download Hadoop-Auth's source code, the examples are in the
- <<<src/main/examples>>> directory.
-
-** Server Example:
-
- Edit the <<<hadoop-auth-examples/src/main/webapp/WEB-INF/web.xml>>> and set the
- right configuration init parameters for the <<<AuthenticationFilter>>>
- definition configured for Kerberos (the right Kerberos principal and keytab
- file must be specified). Refer to the {{{./Configuration.html}Configuration
- document}} for details.
-
- Create the web application WAR file by running the <<<mvn package>>> command.
-
- Deploy the WAR file in a servlet container. For example, if using Tomcat,
- copy the WAR file to Tomcat's <<<webapps/>>> directory.
-
- Start the servlet container.
-
-** Accessing the server using <<<curl>>>
-
- Try accessing protected resources using <<<curl>>>. The protected resources
- are:
-
-+---+
-$ kinit
-Please enter the password for tucu@LOCALHOST:
-
-$ curl http://localhost:8080/hadoop-auth-examples/anonymous/who
-
-$ curl http://localhost:8080/hadoop-auth-examples/simple/who?user.name=foo
-
-$ curl --negotiate -u foo -b ~/cookiejar.txt -c ~/cookiejar.txt http://localhost:8080/hadoop-auth-examples/kerberos/who
-+---+
-
-** Accessing the server using the Java client example
-
-+---+
-$ kinit
-Please enter the password for tucu@LOCALHOST:
-
-$ cd examples
-
-$ mvn exec:java -Durl=http://localhost:8080/hadoop-auth-examples/kerberos/who
-
-....
-
-Token value: "u=tucu,p=tucu@LOCALHOST,t=kerberos,e=1295305313146,s=sVZ1mpSnC5TKhZQE3QLN5p2DWBo="
-Status code: 200 OK
-
-You are: user[tucu] principal[tucu@LOCALHOST]
-
-....
-
-+---+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-auth/src/site/apt/index.apt.vm
deleted file mode 100644
index bf85f7f..0000000
--- a/hadoop-common-project/hadoop-auth/src/site/apt/index.apt.vm
+++ /dev/null
@@ -1,59 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Auth, Java HTTP SPNEGO ${project.version}
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Auth, Java HTTP SPNEGO ${project.version}
-
- Hadoop Auth is a Java library consisting of a client and a server
- components to enable Kerberos SPNEGO authentication for HTTP.
-
- Hadoop Auth also supports additional authentication mechanisms on the client
- and the server side via 2 simple interfaces.
-
- Additionally, it provides a partially implemented derivative of the Kerberos
- SPNEGO authentication to allow a "mixed" form of authentication where Kerberos
- SPNEGO is used by non-browsers while an alternate form of authentication
- (to be implemented by the user) is used for browsers.
-
-* License
-
- Hadoop Auth is distributed under {{{http://www.apache.org/licenses/}Apache
- License 2.0}}.
-
-* How Does Auth Works?
-
- Hadoop Auth enforces authentication on protected resources, once authentiation
- has been established it sets a signed HTTP Cookie that contains an
- authentication token with the user name, user principal, authentication type
- and expiration time.
-
- Subsequent HTTP client requests presenting the signed HTTP Cookie have access
- to the protected resources until the HTTP Cookie expires.
-
- The secret used to sign the HTTP Cookie has multiple implementations that
- provide different behaviors, including a hardcoded secret string, a rolling
- randomly generated secret, and a rolling randomly generated secret
- synchronized between multiple servers using ZooKeeper.
-
-* User Documentation
-
- * {{{./Examples.html}Examples}}
-
- * {{{./Configuration.html}Configuration}}
-
- * {{{./BuildingIt.html}Building It}}
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md b/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md
new file mode 100644
index 0000000..53a49d4
--- /dev/null
+++ b/hadoop-common-project/hadoop-auth/src/site/markdown/BuildingIt.md
@@ -0,0 +1,56 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop Auth, Java HTTP SPNEGO - Building It
+===========================================
+
+Requirements
+------------
+
+* Java 6+
+* Maven 3+
+* Kerberos KDC (for running Kerberos test cases)
+
+Building
+--------
+
+Use Maven goals: clean, test, compile, package, install
+
+Available profiles: docs, testKerberos
+
+Testing
+-------
+
+By default Kerberos testcases are not run.
+
+The requirements to run Kerberos testcases are a running KDC, a keytab file with a client principal and a kerberos principal.
+
+To run Kerberos tescases use the `testKerberos` Maven profile:
+
+ $ mvn test -PtestKerberos
+
+The following Maven `-D` options can be used to change the default values:
+
+* `hadoop-auth.test.kerberos.realm`: default value **LOCALHOST**
+* `hadoop-auth.test.kerberos.client.principal`: default value **client**
+* `hadoop-auth.test.kerberos.server.principal`: default value **HTTP/localhost** (it must start 'HTTP/')
+* `hadoop-auth.test.kerberos.keytab.file`: default value **$HOME/$USER.keytab**
+
+### Generating Documentation
+
+To create the documentation use the `docs` Maven profile:
+
+ $ mvn package -Pdocs
+
+The generated documentation is available at `hadoop-auth/target/site/`.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md b/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
new file mode 100644
index 0000000..9d076bb
--- /dev/null
+++ b/hadoop-common-project/hadoop-auth/src/site/markdown/Configuration.md
@@ -0,0 +1,341 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop Auth, Java HTTP SPNEGO - Server Side Configuration
+=========================================================
+
+Server Side Configuration Setup
+-------------------------------
+
+The AuthenticationFilter filter is Hadoop Auth's server side component.
+
+This filter must be configured in front of all the web application resources that required authenticated requests. For example:
+
+The Hadoop Auth and dependent JAR files must be in the web application classpath (commonly the `WEB-INF/lib` directory).
+
+Hadoop Auth uses SLF4J-API for logging. Auth Maven POM dependencies define the SLF4J API dependency but it does not define the dependency on a concrete logging implementation, this must be addded explicitly to the web application. For example, if the web applicationan uses Log4j, the SLF4J-LOG4J12 and LOG4J jar files must be part part of the web application classpath as well as the Log4j configuration file.
+
+### Common Configuration parameters
+
+* `config.prefix`: If specified, all other configuration parameter names
+ must start with the prefix. The default value is no prefix.
+
+* `[PREFIX.]type`: the authentication type keyword (`simple` or \
+ `kerberos`) or a Authentication handler implementation.
+
+* `[PREFIX.]signature.secret`: When `signer.secret.provider` is set to
+ `string` or not specified, this is the value for the secret used to sign
+ the HTTP cookie.
+
+* `[PREFIX.]token.validity`: The validity -in seconds- of the generated
+ authentication token. The default value is `3600` seconds. This is also
+ used for the rollover interval when `signer.secret.provider` is set to
+ `random` or `zookeeper`.
+
+* `[PREFIX.]cookie.domain`: domain to use for the HTTP cookie that stores
+ the authentication token.
+
+* `[PREFIX.]cookie.path`: path to use for the HTTP cookie that stores the
+ authentication token.
+
+* `signer.secret.provider`: indicates the name of the SignerSecretProvider
+ class to use. Possible values are: `string`, `random`,
+ `zookeeper`, or a classname. If not specified, the `string`
+ implementation will be used; and failing that, the `random`
+ implementation will be used.
+
+### Kerberos Configuration
+
+**IMPORTANT**: A KDC must be configured and running.
+
+To use Kerberos SPNEGO as the authentication mechanism, the authentication filter must be configured with the following init parameters:
+
+* `[PREFIX.]type`: the keyword `kerberos`.
+
+* `[PREFIX.]kerberos.principal`: The web-application Kerberos principal
+ name. The Kerberos principal name must start with `HTTP/...`. For
+ example: `HTTP/localhost@LOCALHOST`. There is no default value.
+
+* `[PREFIX.]kerberos.keytab`: The path to the keytab file containing
+ the credentials for the kerberos principal. For example:
+ `/Users/tucu/tucu.keytab`. There is no default value.
+
+**Example**:
+
+ <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+ ...
+
+ <filter>
+ <filter-name>kerberosFilter</filter-name>
+ <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+ <init-param>
+ <param-name>type</param-name>
+ <param-value>kerberos</param-value>
+ </init-param>
+ <init-param>
+ <param-name>token.validity</param-name>
+ <param-value>30</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cookie.domain</param-name>
+ <param-value>.foo.com</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cookie.path</param-name>
+ <param-value>/</param-value>
+ </init-param>
+ <init-param>
+ <param-name>kerberos.principal</param-name>
+ <param-value>HTTP/localhost@LOCALHOST</param-value>
+ </init-param>
+ <init-param>
+ <param-name>kerberos.keytab</param-name>
+ <param-value>/tmp/auth.keytab</param-value>
+ </init-param>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>kerberosFilter</filter-name>
+ <url-pattern>/kerberos/*</url-pattern>
+ </filter-mapping>
+
+ ...
+ </web-app>
+
+### Pseudo/Simple Configuration
+
+To use Pseudo/Simple as the authentication mechanism (trusting the value of the query string parameter 'user.name'), the authentication filter must be configured with the following init parameters:
+
+* `[PREFIX.]type`: the keyword `simple`.
+
+* `[PREFIX.]simple.anonymous.allowed`: is a boolean parameter that
+ indicates if anonymous requests are allowed or not. The default value is
+ `false`.
+
+**Example**:
+
+ <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+ ...
+
+ <filter>
+ <filter-name>simpleFilter</filter-name>
+ <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+ <init-param>
+ <param-name>type</param-name>
+ <param-value>simple</param-value>
+ </init-param>
+ <init-param>
+ <param-name>token.validity</param-name>
+ <param-value>30</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cookie.domain</param-name>
+ <param-value>.foo.com</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cookie.path</param-name>
+ <param-value>/</param-value>
+ </init-param>
+ <init-param>
+ <param-name>simple.anonymous.allowed</param-name>
+ <param-value>false</param-value>
+ </init-param>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>simpleFilter</filter-name>
+ <url-pattern>/simple/*</url-pattern>
+ </filter-mapping>
+
+ ...
+ </web-app>
+
+### AltKerberos Configuration
+
+**IMPORTANT**: A KDC must be configured and running.
+
+The AltKerberos authentication mechanism is a partially implemented derivative of the Kerberos SPNEGO authentication mechanism which allows a "mixed" form of authentication where Kerberos SPNEGO is used by non-browsers while an alternate form of authentication (to be implemented by the user) is used for browsers. To use AltKerberos as the authentication mechanism (besides providing an implementation), the authentication filter must be configured with the following init parameters, in addition to the previously mentioned Kerberos SPNEGO ones:
+
+* `[PREFIX.]type`: the full class name of the implementation of
+ AltKerberosAuthenticationHandler to use.
+
+* `[PREFIX.]alt-kerberos.non-browser.user-agents`: a comma-separated
+ list of which user-agents should be considered non-browsers.
+
+**Example**:
+
+ <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+ ...
+
+ <filter>
+ <filter-name>kerberosFilter</filter-name>
+ <filter-class>org.apache.hadoop.security.auth.server.AuthenticationFilter</filter-class>
+ <init-param>
+ <param-name>type</param-name>
+ <param-value>org.my.subclass.of.AltKerberosAuthenticationHandler</param-value>
+ </init-param>
+ <init-param>
+ <param-name>alt-kerberos.non-browser.user-agents</param-name>
+ <param-value>java,curl,wget,perl</param-value>
+ </init-param>
+ <init-param>
+ <param-name>token.validity</param-name>
+ <param-value>30</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cookie.domain</param-name>
+ <param-value>.foo.com</param-value>
+ </init-param>
+ <init-param>
+ <param-name>cookie.path</param-name>
+ <param-value>/</param-value>
+ </init-param>
+ <init-param>
+ <param-name>kerberos.principal</param-name>
+ <param-value>HTTP/localhost@LOCALHOST</param-value>
+ </init-param>
+ <init-param>
+ <param-name>kerberos.keytab</param-name>
+ <param-value>/tmp/auth.keytab</param-value>
+ </init-param>
+ </filter>
+
+ <filter-mapping>
+ <filter-name>kerberosFilter</filter-name>
+ <url-pattern>/kerberos/*</url-pattern>
+ </filter-mapping>
+
+ ...
+ </web-app>
+
+### SignerSecretProvider Configuration
+
+The SignerSecretProvider is used to provide more advanced behaviors for the secret used for signing the HTTP Cookies.
+
+These are the relevant configuration properties:
+
+* `signer.secret.provider`: indicates the name of the
+ SignerSecretProvider class to use. Possible values are: "string",
+ "random", "zookeeper", or a classname. If not specified, the "string"
+ implementation will be used; and failing that, the "random" implementation
+ will be used.
+
+* `[PREFIX.]signature.secret`: When `signer.secret.provider` is set
+ to `string` or not specified, this is the value for the secret used to
+ sign the HTTP cookie.
+
+* `[PREFIX.]token.validity`: The validity -in seconds- of the generated
+ authentication token. The default value is `3600` seconds. This is
+ also used for the rollover interval when `signer.secret.provider` is
+ set to `random` or `zookeeper`.
+
+The following configuration properties are specific to the `zookeeper` implementation:
+
+* `signer.secret.provider.zookeeper.connection.string`: Indicates the
+ ZooKeeper connection string to connect with.
+
+* `signer.secret.provider.zookeeper.path`: Indicates the ZooKeeper path
+ to use for storing and retrieving the secrets. All servers
+ that need to coordinate their secret should point to the same path
+
+* `signer.secret.provider.zookeeper.auth.type`: Indicates the auth type
+ to use. Supported values are `none` and `sasl`. The default
+ value is `none`.
+
+* `signer.secret.provider.zookeeper.kerberos.keytab`: Set this to the
+ path with the Kerberos keytab file. This is only required if using
+ Kerberos.
+
+* `signer.secret.provider.zookeeper.kerberos.principal`: Set this to the
+ Kerberos principal to use. This only required if using Kerberos.
+
+**Example**:
+
+ <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+ ...
+
+ <filter>
+ <!-- AuthenticationHandler configs not shown -->
+ <init-param>
+ <param-name>signer.secret.provider</param-name>
+ <param-value>string</param-value>
+ </init-param>
+ <init-param>
+ <param-name>signature.secret</param-name>
+ <param-value>my_secret</param-value>
+ </init-param>
+ </filter>
+
+ ...
+ </web-app>
+
+**Example**:
+
+ <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+ ...
+
+ <filter>
+ <!-- AuthenticationHandler configs not shown -->
+ <init-param>
+ <param-name>signer.secret.provider</param-name>
+ <param-value>random</param-value>
+ </init-param>
+ <init-param>
+ <param-name>token.validity</param-name>
+ <param-value>30</param-value>
+ </init-param>
+ </filter>
+
+ ...
+ </web-app>
+
+**Example**:
+
+ <web-app version="2.5" xmlns="http://java.sun.com/xml/ns/javaee">
+ ...
+
+ <filter>
+ <!-- AuthenticationHandler configs not shown -->
+ <init-param>
+ <param-name>signer.secret.provider</param-name>
+ <param-value>zookeeper</param-value>
+ </init-param>
+ <init-param>
+ <param-name>token.validity</param-name>
+ <param-value>30</param-value>
+ </init-param>
+ <init-param>
+ <param-name>signer.secret.provider.zookeeper.connection.string</param-name>
+ <param-value>zoo1:2181,zoo2:2181,zoo3:2181</param-value>
+ </init-param>
+ <init-param>
+ <param-name>signer.secret.provider.zookeeper.path</param-name>
+ <param-value>/myapp/secrets</param-value>
+ </init-param>
+ <init-param>
+ <param-name>signer.secret.provider.zookeeper.use.kerberos.acls</param-name>
+ <param-value>true</param-value>
+ </init-param>
+ <init-param>
+ <param-name>signer.secret.provider.zookeeper.kerberos.keytab</param-name>
+ <param-value>/tmp/auth.keytab</param-value>
+ </init-param>
+ <init-param>
+ <param-name>signer.secret.provider.zookeeper.kerberos.principal</param-name>
+ <param-value>HTTP/localhost@LOCALHOST</param-value>
+ </init-param>
+ </filter>
+
+ ...
+ </web-app>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/markdown/Examples.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/Examples.md b/hadoop-common-project/hadoop-auth/src/site/markdown/Examples.md
new file mode 100644
index 0000000..7efb642
--- /dev/null
+++ b/hadoop-common-project/hadoop-auth/src/site/markdown/Examples.md
@@ -0,0 +1,109 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop Auth, Java HTTP SPNEGO - Examples
+========================================
+
+Accessing a Hadoop Auth protected URL Using a browser
+-----------------------------------------------------
+
+**IMPORTANT:** The browser must support HTTP Kerberos SPNEGO. For example, Firefox or Internet Explorer.
+
+For Firefox access the low level configuration page by loading the `about:config` page. Then go to the `network.negotiate-auth.trusted-uris` preference and add the hostname or the domain of the web server that is HTTP Kerberos SPNEGO protected (if using multiple domains and hostname use comma to separate them).
+
+Accessing a Hadoop Auth protected URL Using `curl`
+--------------------------------------------------
+
+**IMPORTANT:** The `curl` version must support GSS, run `curl -V`.
+
+ $ curl -V
+ curl 7.19.7 (universal-apple-darwin10.0) libcurl/7.19.7 OpenSSL/0.9.8l zlib/1.2.3
+ Protocols: tftp ftp telnet dict ldap http file https ftps
+ Features: GSS-Negotiate IPv6 Largefile NTLM SSL libz
+
+Login to the KDC using **kinit** and then use `curl` to fetch protected URL:
+
+ $ kinit
+ Please enter the password for tucu@LOCALHOST:
+ $ curl --negotiate -u foo -b ~/cookiejar.txt -c ~/cookiejar.txt http://localhost:8080/hadoop-auth-examples/kerberos/who
+ Enter host password for user 'tucu':
+
+ Hello Hadoop Auth Examples!
+
+* The `--negotiate` option enables SPNEGO in `curl`.
+
+* The `-u foo` option is required but the user ignored (the principal
+ that has been kinit-ed is used).
+
+* The `-b` and `-c` are use to store and send HTTP Cookies.
+
+Using the Java Client
+---------------------
+
+Use the `AuthenticatedURL` class to obtain an authenticated HTTP connection:
+
+ ...
+ URL url = new URL("http://localhost:8080/hadoop-auth/kerberos/who");
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ ...
+ HttpURLConnection conn = new AuthenticatedURL(url, token).openConnection();
+ ...
+ conn = new AuthenticatedURL(url, token).openConnection();
+ ...
+
+Building and Running the Examples
+---------------------------------
+
+Download Hadoop-Auth's source code, the examples are in the `src/main/examples` directory.
+
+### Server Example:
+
+Edit the `hadoop-auth-examples/src/main/webapp/WEB-INF/web.xml` and set the right configuration init parameters for the `AuthenticationFilter` definition configured for Kerberos (the right Kerberos principal and keytab file must be specified). Refer to the [Configuration document](./Configuration.html) for details.
+
+Create the web application WAR file by running the `mvn package` command.
+
+Deploy the WAR file in a servlet container. For example, if using Tomcat, copy the WAR file to Tomcat's `webapps/` directory.
+
+Start the servlet container.
+
+### Accessing the server using `curl`
+
+Try accessing protected resources using `curl`. The protected resources are:
+
+ $ kinit
+ Please enter the password for tucu@LOCALHOST:
+
+ $ curl http://localhost:8080/hadoop-auth-examples/anonymous/who
+
+ $ curl http://localhost:8080/hadoop-auth-examples/simple/who?user.name=foo
+
+ $ curl --negotiate -u foo -b ~/cookiejar.txt -c ~/cookiejar.txt http://localhost:8080/hadoop-auth-examples/kerberos/who
+
+### Accessing the server using the Java client example
+
+ $ kinit
+ Please enter the password for tucu@LOCALHOST:
+
+ $ cd examples
+
+ $ mvn exec:java -Durl=http://localhost:8080/hadoop-auth-examples/kerberos/who
+
+ ....
+
+ Token value: "u=tucu,p=tucu@LOCALHOST,t=kerberos,e=1295305313146,s=sVZ1mpSnC5TKhZQE3QLN5p2DWBo="
+ Status code: 200 OK
+
+ You are: user[tucu] principal[tucu@LOCALHOST]
+
+ ....
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-auth/src/site/markdown/index.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/site/markdown/index.md b/hadoop-common-project/hadoop-auth/src/site/markdown/index.md
new file mode 100644
index 0000000..8573b18
--- /dev/null
+++ b/hadoop-common-project/hadoop-auth/src/site/markdown/index.md
@@ -0,0 +1,43 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop Auth, Java HTTP SPNEGO
+=============================
+
+Hadoop Auth is a Java library consisting of a client and a server components to enable Kerberos SPNEGO authentication for HTTP.
+
+Hadoop Auth also supports additional authentication mechanisms on the client and the server side via 2 simple interfaces.
+
+Additionally, it provides a partially implemented derivative of the Kerberos SPNEGO authentication to allow a "mixed" form of authentication where Kerberos SPNEGO is used by non-browsers while an alternate form of authentication (to be implemented by the user) is used for browsers.
+
+License
+-------
+
+Hadoop Auth is distributed under [Apache License 2.0](http://www.apache.org/licenses/).
+
+How Does Auth Works?
+--------------------
+
+Hadoop Auth enforces authentication on protected resources, once authentiation has been established it sets a signed HTTP Cookie that contains an authentication token with the user name, user principal, authentication type and expiration time.
+
+Subsequent HTTP client requests presenting the signed HTTP Cookie have access to the protected resources until the HTTP Cookie expires.
+
+The secret used to sign the HTTP Cookie has multiple implementations that provide different behaviors, including a hardcoded secret string, a rolling randomly generated secret, and a rolling randomly generated secret synchronized between multiple servers using ZooKeeper.
+
+User Documentation
+------------------
+
+* [Examples](./Examples.html)
+* [Configuration](./Configuration.html)
+* [Building It](./BuildingIt.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index d8a85f7..b3b2c95 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -181,6 +181,9 @@ Trunk (Unreleased)
HADOOP-11596. Allow smart-apply-patch.sh to add new files in binary git
patches (raviprak)
+ HADOOP-11593. Convert site documentation from apt to markdown (stragglers)
+ (Masatake Iwasaki via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6fc1f3e/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm b/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
deleted file mode 100644
index a2dcce3..0000000
--- a/hadoop-common-project/hadoop-kms/src/site/apt/index.apt.vm
+++ /dev/null
@@ -1,1020 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop KMS - Documentation Sets ${project.version}
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Key Management Server (KMS) - Documentation Sets ${project.version}
-
- Hadoop KMS is a cryptographic key management server based on Hadoop's
- <<KeyProvider>> API.
-
- It provides a client and a server components which communicate over
- HTTP using a REST API.
-
- The client is a KeyProvider implementation interacts with the KMS
- using the KMS HTTP REST API.
-
- KMS and its client have built-in security and they support HTTP SPNEGO
- Kerberos authentication and HTTPS secure transport.
-
- KMS is a Java web-application and it runs using a pre-configured Tomcat
- bundled with the Hadoop distribution.
-
-* KMS Client Configuration
-
- The KMS client <<<KeyProvider>>> uses the <<kms>> scheme, and the embedded
- URL must be the URL of the KMS. For example, for a KMS running
- on <<<http://localhost:16000/kms>>>, the KeyProvider URI is
- <<<kms://http@localhost:16000/kms>>>. And, for a KMS running on
- <<<https://localhost:16000/kms>>>, the KeyProvider URI is
- <<<kms://https@localhost:16000/kms>>>
-
-* KMS
-
-** KMS Configuration
-
- Configure the KMS backing KeyProvider properties
- in the <<<etc/hadoop/kms-site.xml>>> configuration file:
-
-+---+
- <property>
- <name>hadoop.kms.key.provider.uri</name>
- <value>jceks://file@/${user.home}/kms.keystore</value>
- </property>
-
- <property>
- <name>hadoop.security.keystore.java-keystore-provider.password-file</name>
- <value>kms.keystore.password</value>
- </property>
-+---+
-
- The password file is looked up in the Hadoop's configuration directory via the
- classpath.
-
- NOTE: You need to restart the KMS for the configuration changes to take
- effect.
-
-** KMS Cache
-
- KMS caches keys for short period of time to avoid excessive hits to the
- underlying key provider.
-
- The Cache is enabled by default (can be dissabled by setting the
- <<<hadoop.kms.cache.enable>>> boolean property to false)
-
- The cache is used with the following 3 methods only, <<<getCurrentKey()>>>
- and <<<getKeyVersion()>>> and <<<getMetadata()>>>.
-
- For the <<<getCurrentKey()>>> method, cached entries are kept for a maximum
- of 30000 millisecond regardless the number of times the key is being access
- (to avoid stale keys to be considered current).
-
- For the <<<getKeyVersion()>>> method, cached entries are kept with a default
- inactivity timeout of 600000 milliseconds (10 mins). This time out is
- configurable via the following property in the <<<etc/hadoop/kms-site.xml>>>
- configuration file:
-
-+---+
- <property>
- <name>hadoop.kms.cache.enable</name>
- <value>true</value>
- </property>
-
- <property>
- <name>hadoop.kms.cache.timeout.ms</name>
- <value>600000</value>
- </property>
-
- <property>
- <name>hadoop.kms.current.key.cache.timeout.ms</name>
- <value>30000</value>
- </property>
-+---+
-
-** KMS Aggregated Audit logs
-
- Audit logs are aggregated for API accesses to the GET_KEY_VERSION,
- GET_CURRENT_KEY, DECRYPT_EEK, GENERATE_EEK operations.
-
- Entries are grouped by the (user,key,operation) combined key for a
- configurable aggregation interval after which the number of accesses to the
- specified end-point by the user for a given key is flushed to the audit log.
-
- The Aggregation interval is configured via the property :
-
-+---+
- <property>
- <name>hadoop.kms.aggregation.delay.ms</name>
- <value>10000</value>
- </property>
-+---+
-
-
-** Start/Stop the KMS
-
- To start/stop KMS use KMS's bin/kms.sh script. For example:
-
-+---+
-hadoop-${project.version} $ sbin/kms.sh start
-+---+
-
- NOTE: Invoking the script without any parameters list all possible
- parameters (start, stop, run, etc.). The <<<kms.sh>>> script is a wrapper
- for Tomcat's <<<catalina.sh>>> script that sets the environment variables
- and Java System properties required to run KMS.
-
-** Embedded Tomcat Configuration
-
- To configure the embedded Tomcat go to the <<<share/hadoop/kms/tomcat/conf>>>.
-
- KMS pre-configures the HTTP and Admin ports in Tomcat's <<<server.xml>>> to
- 16000 and 16001.
-
- Tomcat logs are also preconfigured to go to Hadoop's <<<logs/>>> directory.
-
- The following environment variables (which can be set in KMS's
- <<<etc/hadoop/kms-env.sh>>> script) can be used to alter those values:
-
- * KMS_HTTP_PORT
-
- * KMS_ADMIN_PORT
-
- * KMS_MAX_THREADS
-
- * KMS_LOG
-
- NOTE: You need to restart the KMS for the configuration changes to take
- effect.
-
-** Loading native libraries
-
- The following environment variable (which can be set in KMS's
- <<<etc/hadoop/kms-env.sh>>> script) can be used to specify the location
- of any required native libraries. For eg. Tomact native Apache Portable
- Runtime (APR) libraries:
-
- * JAVA_LIBRARY_PATH
-
-** KMS Security Configuration
-
-*** Enabling Kerberos HTTP SPNEGO Authentication
-
- Configure the Kerberos <<<etc/krb5.conf>>> file with the information of your
- KDC server.
-
- Create a service principal and its keytab for the KMS, it must be an
- <<<HTTP>>> service principal.
-
- Configure KMS <<<etc/hadoop/kms-site.xml>>> with the correct security values,
- for example:
-
-+---+
- <property>
- <name>hadoop.kms.authentication.type</name>
- <value>kerberos</value>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.kerberos.keytab</name>
- <value>${user.home}/kms.keytab</value>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.kerberos.principal</name>
- <value>HTTP/localhost</value>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.kerberos.name.rules</name>
- <value>DEFAULT</value>
- </property>
-+---+
-
- NOTE: You need to restart the KMS for the configuration changes to take
- effect.
-
-*** KMS Proxyuser Configuration
-
- Each proxyuser must be configured in <<<etc/hadoop/kms-site.xml>>> using the
- following properties:
-
-+---+
- <property>
- <name>hadoop.kms.proxyuser.#USER#.users</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.proxyuser.#USER#.groups</name>
- <value>*</value>
- </property>
-
- <property>
- <name>hadoop.kms.proxyuser.#USER#.hosts</name>
- <value>*</value>
- </property>
-+---+
-
- <<<#USER#>>> is the username of the proxyuser to configure.
-
- The <<<users>>> property indicates the users that can be impersonated.
-
- The <<<groups>>> property indicates the groups users being impersonated must
- belong to.
-
- At least one of the <<<users>>> or <<<groups>>> properties must be defined.
- If both are specified, then the configured proxyuser will be able to
- impersonate and user in the <<<users>>> list and any user belonging to one of
- the groups in the <<<groups>>> list.
-
- The <<<hosts>>> property indicates from which host the proxyuser can make
- impersonation requests.
-
- If <<<users>>>, <<<groups>>> or <<<hosts>>> has a <<<*>>>, it means there are
- no restrictions for the proxyuser regarding users, groups or hosts.
-
-*** KMS over HTTPS (SSL)
-
- To configure KMS to work over HTTPS the following 2 properties must be
- set in the <<<etc/hadoop/kms_env.sh>>> script (shown with default values):
-
- * KMS_SSL_KEYSTORE_FILE=${HOME}/.keystore
-
- * KMS_SSL_KEYSTORE_PASS=password
-
- In the KMS <<<tomcat/conf>>> directory, replace the <<<server.xml>>> file
- with the provided <<<ssl-server.xml>>> file.
-
- You need to create an SSL certificate for the KMS. As the
- <<<kms>>> Unix user, using the Java <<<keytool>>> command to create the
- SSL certificate:
-
-+---+
-$ keytool -genkey -alias tomcat -keyalg RSA
-+---+
-
- You will be asked a series of questions in an interactive prompt. It will
- create the keystore file, which will be named <<.keystore>> and located in the
- <<<kms>>> user home directory.
-
- The password you enter for "keystore password" must match the value of the
- <<<KMS_SSL_KEYSTORE_PASS>>> environment variable set in the
- <<<kms-env.sh>>> script in the configuration directory.
-
- The answer to "What is your first and last name?" (i.e. "CN") must be the
- hostname of the machine where the KMS will be running.
-
- NOTE: You need to restart the KMS for the configuration changes to take
- effect.
-
-*** KMS Access Control
-
- KMS ACLs configuration are defined in the KMS <<<etc/hadoop/kms-acls.xml>>>
- configuration file. This file is hot-reloaded when it changes.
-
- KMS supports both fine grained access control as well as blacklist for kms
- operations via a set ACL configuration properties.
-
- A user accessing KMS is first checked for inclusion in the Access Control
- List for the requested operation and then checked for exclusion in the
- Black list for the operation before access is granted.
-
-
-+---+
- <property>
- <name>hadoop.kms.acl.CREATE</name>
- <value>*</value>
- <description>
- ACL for create-key operations.
- If the user is not in the GET ACL, the key material is not returned
- as part of the response.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.CREATE</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for create-key operations.
- If the user is in the Blacklist, the key material is not returned
- as part of the response.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.DELETE</name>
- <value>*</value>
- <description>
- ACL for delete-key operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.DELETE</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for delete-key operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.ROLLOVER</name>
- <value>*</value>
- <description>
- ACL for rollover-key operations.
- If the user is not in the GET ACL, the key material is not returned
- as part of the response.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.ROLLOVER</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for rollover-key operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GET</name>
- <value>*</value>
- <description>
- ACL for get-key-version and get-current-key operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.GET</name>
- <value>hdfs,foo</value>
- <description>
- ACL for get-key-version and get-current-key operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GET_KEYS</name>
- <value>*</value>
- <description>
- ACL for get-keys operation.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.GET_KEYS</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for get-keys operation.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GET_METADATA</name>
- <value>*</value>
- <description>
- ACL for get-key-metadata and get-keys-metadata operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.GET_METADATA</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for get-key-metadata and get-keys-metadata operations.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.SET_KEY_MATERIAL</name>
- <value>*</value>
- <description>
- Complimentary ACL for CREATE and ROLLOVER operation to allow the client
- to provide the key material when creating or rolling a key.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.SET_KEY_MATERIAL</name>
- <value>hdfs,foo</value>
- <description>
- Complimentary Blacklist for CREATE and ROLLOVER operation to allow the client
- to provide the key material when creating or rolling a key.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.GENERATE_EEK</name>
- <value>*</value>
- <description>
- ACL for generateEncryptedKey
- CryptoExtension operations
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.blacklist.GENERATE_EEK</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for generateEncryptedKey
- CryptoExtension operations
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.acl.DECRYPT_EEK</name>
- <value>*</value>
- <description>
- ACL for decrypt EncryptedKey
- CryptoExtension operations
- </description>
- </property>
-</configuration>
-
- <property>
- <name>hadoop.kms.blacklist.DECRYPT_EEK</name>
- <value>hdfs,foo</value>
- <description>
- Blacklist for decrypt EncryptedKey
- CryptoExtension operations
- </description>
- </property>
-</configuration>
-
-+---+
-
-*** Key Access Control
-
- KMS supports access control for all non-read operations at the Key level.
- All Key Access operations are classified as :
-
- * MANAGEMENT - createKey, deleteKey, rolloverNewVersion
-
- * GENERATE_EEK - generateEncryptedKey, warmUpEncryptedKeys
-
- * DECRYPT_EEK - decryptEncryptedKey
-
- * READ - getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata,
- getCurrentKey
-
- * ALL - all of the above
-
- These can be defined in the KMS <<<etc/hadoop/kms-acls.xml>>> as follows
-
- For all keys for which a key access has not been explicitly configured, It
- is possible to configure a default key access control for a subset of the
- operation types.
-
- It is also possible to configure a "whitelist" key ACL for a subset of the
- operation types. The whitelist key ACL is a whitelist in addition to the
- explicit or default per-key ACL. That is, if no per-key ACL is explicitly
- set, a user will be granted access if they are present in the default per-key
- ACL or the whitelist key ACL. If a per-key ACL is explicitly set, a user
- will be granted access if they are present in the per-key ACL or the
- whitelist key ACL.
-
- If no ACL is configured for a specific key AND no default ACL is configured
- AND no root key ACL is configured for the requested operation,
- then access will be DENIED.
-
- <<NOTE:>> The default and whitelist key ACL does not support <<<ALL>>>
- operation qualifier.
-
-+---+
- <property>
- <name>key.acl.testKey1.MANAGEMENT</name>
- <value>*</value>
- <description>
- ACL for create-key, deleteKey and rolloverNewVersion operations.
- </description>
- </property>
-
- <property>
- <name>key.acl.testKey2.GENERATE_EEK</name>
- <value>*</value>
- <description>
- ACL for generateEncryptedKey operations.
- </description>
- </property>
-
- <property>
- <name>key.acl.testKey3.DECRYPT_EEK</name>
- <value>admink3</value>
- <description>
- ACL for decryptEncryptedKey operations.
- </description>
- </property>
-
- <property>
- <name>key.acl.testKey4.READ</name>
- <value>*</value>
- <description>
- ACL for getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata,
- getCurrentKey operations
- </description>
- </property>
-
- <property>
- <name>key.acl.testKey5.ALL</name>
- <value>*</value>
- <description>
- ACL for ALL operations.
- </description>
- </property>
-
- <property>
- <name>whitelist.key.acl.MANAGEMENT</name>
- <value>admin1</value>
- <description>
- whitelist ACL for MANAGEMENT operations for all keys.
- </description>
- </property>
-
- <!--
- 'testKey3' key ACL is defined. Since a 'whitelist'
- key is also defined for DECRYPT_EEK, in addition to
- admink3, admin1 can also perform DECRYPT_EEK operations
- on 'testKey3'
- -->
- <property>
- <name>whitelist.key.acl.DECRYPT_EEK</name>
- <value>admin1</value>
- <description>
- whitelist ACL for DECRYPT_EEK operations for all keys.
- </description>
- </property>
-
- <property>
- <name>default.key.acl.MANAGEMENT</name>
- <value>user1,user2</value>
- <description>
- default ACL for MANAGEMENT operations for all keys that are not
- explicitly defined.
- </description>
- </property>
-
- <property>
- <name>default.key.acl.GENERATE_EEK</name>
- <value>user1,user2</value>
- <description>
- default ACL for GENERATE_EEK operations for all keys that are not
- explicitly defined.
- </description>
- </property>
-
- <property>
- <name>default.key.acl.DECRYPT_EEK</name>
- <value>user1,user2</value>
- <description>
- default ACL for DECRYPT_EEK operations for all keys that are not
- explicitly defined.
- </description>
- </property>
-
- <property>
- <name>default.key.acl.READ</name>
- <value>user1,user2</value>
- <description>
- default ACL for READ operations for all keys that are not
- explicitly defined.
- </description>
- </property>
-+---+
-
-** KMS Delegation Token Configuration
-
- KMS delegation token secret manager can be configured with the following
- properties:
-
-+---+
- <property>
- <name>hadoop.kms.authentication.delegation-token.update-interval.sec</name>
- <value>86400</value>
- <description>
- How often the master key is rotated, in seconds. Default value 1 day.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.delegation-token.max-lifetime.sec</name>
- <value>604800</value>
- <description>
- Maximum lifetime of a delagation token, in seconds. Default value 7 days.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.delegation-token.renew-interval.sec</name>
- <value>86400</value>
- <description>
- Renewal interval of a delagation token, in seconds. Default value 1 day.
- </description>
- </property>
-
- <property>
- <name>hadoop.kms.authentication.delegation-token.removal-scan-interval.sec</name>
- <value>3600</value>
- <description>
- Scan interval to remove expired delegation tokens.
- </description>
- </property>
-+---+
-
-
-** Using Multiple Instances of KMS Behind a Load-Balancer or VIP
-
- KMS supports multiple KMS instances behind a load-balancer or VIP for
- scalability and for HA purposes.
-
- When using multiple KMS instances behind a load-balancer or VIP, requests from
- the same user may be handled by different KMS instances.
-
- KMS instances behind a load-balancer or VIP must be specially configured to
- work properly as a single logical service.
-
-*** HTTP Kerberos Principals Configuration
-
- When KMS instances are behind a load-balancer or VIP, clients will use the
- hostname of the VIP. For Kerberos SPNEGO authentication, the hostname of the
- URL is used to construct the Kerberos service name of the server,
- <<<HTTP/#HOSTNAME#>>>. This means that all KMS instances must have a Kerberos
- service name with the load-balancer or VIP hostname.
-
- In order to be able to access directly a specific KMS instance, the KMS
- instance must also have Keberos service name with its own hostname. This is
- required for monitoring and admin purposes.
-
- Both Kerberos service principal credentials (for the load-balancer/VIP
- hostname and for the actual KMS instance hostname) must be in the keytab file
- configured for authentication. And the principal name specified in the
- configuration must be '*'. For example:
-
-+---+
- <property>
- <name>hadoop.kms.authentication.kerberos.principal</name>
- <value>*</value>
- </property>
-+---+
-
- <<NOTE:>> If using HTTPS, the SSL certificate used by the KMS instance must
- be configured to support multiple hostnames (see Java 7
- <<<keytool>>> SAN extension support for details on how to do this).
-
-*** HTTP Authentication Signature
-
- KMS uses Hadoop Authentication for HTTP authentication. Hadoop Authentication
- issues a signed HTTP Cookie once the client has authenticated successfully.
- This HTTP Cookie has an expiration time, after which it will trigger a new
- authentication sequence. This is done to avoid triggering the authentication
- on every HTTP request of a client.
-
- A KMS instance must verify the HTTP Cookie signatures signed by other KMS
- instances. To do this all KMS instances must share the signing secret.
-
- This secret sharing can be done using a Zookeeper service which is configured
- in KMS with the following properties in the <<<kms-site.xml>>>:
-
-+---+
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider</name>
- <value>zookeeper</value>
- <description>
- Indicates how the secret to sign the authentication cookies will be
- stored. Options are 'random' (default), 'string' and 'zookeeper'.
- If using a setup with multiple KMS instances, 'zookeeper' should be used.
- </description>
- </property>
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.path</name>
- <value>/hadoop-kms/hadoop-auth-signature-secret</value>
- <description>
- The Zookeeper ZNode path where the KMS instances will store and retrieve
- the secret from.
- </description>
- </property>
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.connection.string</name>
- <value>#HOSTNAME#:#PORT#,...</value>
- <description>
- The Zookeeper connection string, a list of hostnames and port comma
- separated.
- </description>
- </property>
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.auth.type</name>
- <value>kerberos</value>
- <description>
- The Zookeeper authentication type, 'none' or 'sasl' (Kerberos).
- </description>
- </property>
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.keytab</name>
- <value>/etc/hadoop/conf/kms.keytab</value>
- <description>
- The absolute path for the Kerberos keytab with the credentials to
- connect to Zookeeper.
- </description>
- </property>
- <property>
- <name>hadoop.kms.authentication.signer.secret.provider.zookeeper.kerberos.principal</name>
- <value>kms/#HOSTNAME#</value>
- <description>
- The Kerberos service principal used to connect to Zookeeper.
- </description>
- </property>
-+---+
-
-*** Delegation Tokens
-
- TBD
-
-** KMS HTTP REST API
-
-*** Create a Key
-
- <REQUEST:>
-
-+---+
-POST http://HOST:PORT/kms/v1/keys
-Content-Type: application/json
-
-{
- "name" : "<key-name>",
- "cipher" : "<cipher>",
- "length" : <length>, //int
- "material" : "<material>", //base64
- "description" : "<description>"
-}
-+---+
-
- <RESPONSE:>
-
-+---+
-201 CREATED
-LOCATION: http://HOST:PORT/kms/v1/key/<key-name>
-Content-Type: application/json
-
-{
- "name" : "versionName",
- "material" : "<material>", //base64, not present without GET ACL
-}
-+---+
-
-*** Rollover Key
-
- <REQUEST:>
-
-+---+
-POST http://HOST:PORT/kms/v1/key/<key-name>
-Content-Type: application/json
-
-{
- "material" : "<material>",
-}
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-{
- "name" : "versionName",
- "material" : "<material>", //base64, not present without GET ACL
-}
-+---+
-
-*** Delete Key
-
- <REQUEST:>
-
-+---+
-DELETE http://HOST:PORT/kms/v1/key/<key-name>
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-+---+
-
-*** Get Key Metadata
-
- <REQUEST:>
-
-+---+
-GET http://HOST:PORT/kms/v1/key/<key-name>/_metadata
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-{
- "name" : "<key-name>",
- "cipher" : "<cipher>",
- "length" : <length>, //int
- "description" : "<description>",
- "created" : <millis-epoc>, //long
- "versions" : <versions> //int
-}
-+---+
-
-*** Get Current Key
-
- <REQUEST:>
-
-+---+
-GET http://HOST:PORT/kms/v1/key/<key-name>/_currentversion
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-{
- "name" : "versionName",
- "material" : "<material>", //base64
-}
-+---+
-
-
-*** Generate Encrypted Key for Current KeyVersion
-
- <REQUEST:>
-
-+---+
-GET http://HOST:PORT/kms/v1/key/<key-name>/_eek?eek_op=generate&num_keys=<number-of-keys-to-generate>
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-[
- {
- "versionName" : "encryptionVersionName",
- "iv" : "<iv>", //base64
- "encryptedKeyVersion" : {
- "versionName" : "EEK",
- "material" : "<material>", //base64
- }
- },
- {
- "versionName" : "encryptionVersionName",
- "iv" : "<iv>", //base64
- "encryptedKeyVersion" : {
- "versionName" : "EEK",
- "material" : "<material>", //base64
- }
- },
- ...
-]
-+---+
-
-*** Decrypt Encrypted Key
-
- <REQUEST:>
-
-+---+
-POST http://HOST:PORT/kms/v1/keyversion/<version-name>/_eek?ee_op=decrypt
-Content-Type: application/json
-
-{
- "name" : "<key-name>",
- "iv" : "<iv>", //base64
- "material" : "<material>", //base64
-}
-
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-{
- "name" : "EK",
- "material" : "<material>", //base64
-}
-+---+
-
-
-*** Get Key Version
-
- <REQUEST:>
-
-+---+
-GET http://HOST:PORT/kms/v1/keyversion/<version-name>
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-{
- "name" : "versionName",
- "material" : "<material>", //base64
-}
-+---+
-
-*** Get Key Versions
-
- <REQUEST:>
-
-+---+
-GET http://HOST:PORT/kms/v1/key/<key-name>/_versions
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-[
- {
- "name" : "versionName",
- "material" : "<material>", //base64
- },
- {
- "name" : "versionName",
- "material" : "<material>", //base64
- },
- ...
-]
-+---+
-
-*** Get Key Names
-
- <REQUEST:>
-
-+---+
-GET http://HOST:PORT/kms/v1/keys/names
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-[
- "<key-name>",
- "<key-name>",
- ...
-]
-+---+
-
-*** Get Keys Metadata
-
-+---+
-GET http://HOST:PORT/kms/v1/keys/metadata?key=<key-name>&key=<key-name>,...
-+---+
-
- <RESPONSE:>
-
-+---+
-200 OK
-Content-Type: application/json
-
-[
- {
- "name" : "<key-name>",
- "cipher" : "<cipher>",
- "length" : <length>, //int
- "description" : "<description>",
- "created" : <millis-epoc>, //long
- "versions" : <versions> //int
- },
- {
- "name" : "<key-name>",
- "cipher" : "<cipher>",
- "length" : <length>, //int
- "description" : "<description>",
- "created" : <millis-epoc>, //long
- "versions" : <versions> //int
- },
- ...
-]
-+---+
[03/50] [abbrv] hadoop git commit: HADOOP-11589.
NetUtils.createSocketAddr should trim the input URI. Contributed by Rakesh R.
Posted by zj...@apache.org.
HADOOP-11589. NetUtils.createSocketAddr should trim the input URI. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ef950ea1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ef950ea1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ef950ea1
Branch: refs/heads/YARN-2928
Commit: ef950ea1c08f5a95137a95df6d4c90241a25b70e
Parents: b7d4828
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Sun Feb 15 00:30:46 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Sun Feb 15 00:30:46 2015 +0900
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/net/NetUtils.java | 1 +
.../test/java/org/apache/hadoop/net/TestNetUtils.java | 11 +++++++++++
3 files changed, 15 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef950ea1/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 99320cb..522ec47 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -602,6 +602,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-9869. Configuration.getSocketAddr()/getEnum() should use
getTrimmed(). (Tsuyoshi Ozawa via aajisaka)
+ HADOOP-11589. NetUtils.createSocketAddr should trim the input URI.
+ (Rakesh R via ozawa)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef950ea1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
index ef1092b..e475149 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetUtils.java
@@ -189,6 +189,7 @@ public class NetUtils {
throw new IllegalArgumentException("Target address cannot be null." +
helpText);
}
+ target = target.trim();
boolean hasScheme = target.contains("://");
URI uri = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ef950ea1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
index 319e8a9..a12054b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/net/TestNetUtils.java
@@ -643,6 +643,17 @@ public class TestNetUtils {
assertNotNull(NetUtils.getHostNameOfIP("127.0.0.1:1"));
}
+ @Test
+ public void testTrimCreateSocketAddress() {
+ Configuration conf = new Configuration();
+ NetUtils.addStaticResolution("host", "127.0.0.1");
+ final String defaultAddr = "host:1 ";
+
+ InetSocketAddress addr = NetUtils.createSocketAddr(defaultAddr);
+ conf.setSocketAddr("myAddress", addr);
+ assertEquals(defaultAddr.trim(), NetUtils.getHostPortString(addr));
+ }
+
private <T> void assertBetterArrayEquals(T[] expect, T[]got) {
String expectStr = StringUtils.join(expect, ", ");
String gotStr = StringUtils.join(got, ", ");
[36/50] [abbrv] hadoop git commit: HDFS-7780. Update use of Iterator
to Iterable in DataXceiverServer and SnapshotDiffInfo. Contributed by Ray
Chiang.
Posted by zj...@apache.org.
HDFS-7780. Update use of Iterator to Iterable in DataXceiverServer and SnapshotDiffInfo. Contributed by Ray Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6dc8812a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6dc8812a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6dc8812a
Branch: refs/heads/YARN-2928
Commit: 6dc8812a95bf369ec1f2e3d8a9473033172736cd
Parents: 409113d
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Feb 17 14:47:53 2015 -0800
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Feb 17 14:47:53 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/datanode/DataXceiverServer.java | 4 ++--
.../hdfs/server/namenode/snapshot/SnapshotDiffInfo.java | 9 +++++----
3 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dc8812a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 391005c..308b61f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -647,6 +647,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7795. Show warning if not all favored nodes were chosen by namenode
(kihwal)
+ HDFS-7780. Update use of Iterator to Iterable in DataXceiverServer and
+ SnapshotDiffInfo. (Ray Chiang via aajisaka)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dc8812a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
index 9bf95eb..caf6eaa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiverServer.java
@@ -252,9 +252,9 @@ class DataXceiverServer implements Runnable {
// be set true before calling this method.
synchronized void restartNotifyPeers() {
assert (datanode.shouldRun == true && datanode.shutdownForUpgrade);
- for (Peer p : peers.keySet()) {
+ for (Thread t : peers.values()) {
// interrupt each and every DataXceiver thread.
- peers.get(p).interrupt();
+ t.interrupt();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6dc8812a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
index 197b8ae..a576c57 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotDiffInfo.java
@@ -187,12 +187,13 @@ class SnapshotDiffInfo {
*/
public SnapshotDiffReport generateReport() {
List<DiffReportEntry> diffReportList = new ArrayList<DiffReportEntry>();
- for (INode node : diffMap.keySet()) {
- diffReportList.add(new DiffReportEntry(DiffType.MODIFY, diffMap
- .get(node), null));
+ for (Map.Entry<INode,byte[][]> drEntry : diffMap.entrySet()) {
+ INode node = drEntry.getKey();
+ byte[][] path = drEntry.getValue();
+ diffReportList.add(new DiffReportEntry(DiffType.MODIFY, path, null));
if (node.isDirectory()) {
List<DiffReportEntry> subList = generateReport(dirDiffMap.get(node),
- diffMap.get(node), isFromEarlier(), renameMap);
+ path, isFromEarlier(), renameMap);
diffReportList.addAll(subList);
}
}
[47/50] [abbrv] hadoop git commit: MAPREDUCE-6261.
NullPointerException if MapOutputBuffer.flush invoked twice. Contributed by
Tsuyoshi OZAWA
Posted by zj...@apache.org.
MAPREDUCE-6261. NullPointerException if MapOutputBuffer.flush invoked twice. Contributed by Tsuyoshi OZAWA
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4981d082
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4981d082
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4981d082
Branch: refs/heads/YARN-2928
Commit: 4981d082d4f3c82d1c2c900c7488b83bf20301cc
Parents: 1714609
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Feb 18 19:28:02 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Feb 18 19:28:02 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/mapred/MapTask.java | 4 ++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4981d082/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index e944d82..7f4c3e7 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -379,6 +379,9 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-4286. TestClientProtocolProviderImpls passes on failure
conditions. (Devaraj K via ozawa)
+ MAPREDUCE-6261. NullPointerException if MapOutputBuffer.flush invoked
+ twice (Tsuyoshi OZAWA via jlowe)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4981d082/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
index 1a4901b..8094317 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/MapTask.java
@@ -1458,6 +1458,10 @@ public class MapTask extends Task {
public void flush() throws IOException, ClassNotFoundException,
InterruptedException {
LOG.info("Starting flush of map output");
+ if (kvbuffer == null) {
+ LOG.info("kvbuffer is null. Skipping flush.");
+ return;
+ }
spillLock.lock();
try {
while (spillInProgress) {