You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zj...@apache.org on 2015/02/11 20:48:19 UTC
[01/50] [abbrv] hadoop git commit: HDFS-7756. Restore method
signature for LocatedBlock#getLocations(). (Ted Yu via yliu)
Repository: hadoop
Updated Branches:
refs/heads/YARN-2928 f26941b39 -> e2786151d
HDFS-7756. Restore method signature for LocatedBlock#getLocations(). (Ted Yu via yliu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/260b5e32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/260b5e32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/260b5e32
Branch: refs/heads/YARN-2928
Commit: 260b5e32c427d54c8c74b9f84432700317d1f282
Parents: 1b56d1c
Author: yliu <yl...@apache.org>
Authored: Tue Feb 10 01:57:51 2015 +0800
Committer: yliu <yl...@apache.org>
Committed: Tue Feb 10 01:57:51 2015 +0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/protocol/DatanodeInfoWithStorage.java | 63 ++++++++++++++++++++
.../hadoop/hdfs/protocol/LocatedBlock.java | 4 +-
.../protocol/DatanodeInfoWithStorage.java | 59 ------------------
.../blockmanagement/TestDatanodeManager.java | 8 +--
5 files changed, 72 insertions(+), 65 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/260b5e32/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 5a77829..a841c7e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -887,6 +887,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7647. DatanodeManager.sortLocatedBlocks sorts DatanodeInfos
but not StorageIDs. (Milan Desai via Arpit Agarwal)
+ HDFS-7756. Restore method signature for LocatedBlock#getLocations(). (Ted
+ Yu via yliu)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/260b5e32/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java
new file mode 100644
index 0000000..db2c2e7
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfoWithStorage.java
@@ -0,0 +1,63 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class DatanodeInfoWithStorage extends DatanodeInfo {
+ private final String storageID;
+ private final StorageType storageType;
+
+ public DatanodeInfoWithStorage(DatanodeInfo from, String storageID,
+ StorageType storageType) {
+ super(from);
+ this.storageID = storageID;
+ this.storageType = storageType;
+ }
+
+ public String getStorageID() {
+ return storageID;
+ }
+
+ public StorageType getStorageType() {
+ return storageType;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ // allows this class to be used interchangeably with DatanodeInfo
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ // allows this class to be used interchangeably with DatanodeInfo
+ return super.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "DatanodeInfoWithStorage[" + super.toString() + "," + storageID +
+ "," + storageType + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/260b5e32/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
index 7fb2e30..4ff24b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
@@ -25,7 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeInfoWithStorage;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
import org.apache.hadoop.security.token.Token;
import com.google.common.collect.Lists;
@@ -140,7 +140,7 @@ public class LocatedBlock {
* {@link org.apache.hadoop.hdfs.protocol.LocatedBlock#invalidateCachedStorageInfo}
* to invalidate the cached Storage ID/Type arrays.
*/
- public DatanodeInfoWithStorage[] getLocations() {
+ public DatanodeInfo[] getLocations() {
return locs;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/260b5e32/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java
deleted file mode 100644
index ec8c346..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.protocol;
-
-import org.apache.hadoop.hdfs.StorageType;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-public class DatanodeInfoWithStorage extends DatanodeInfo {
- private final String storageID;
- private final StorageType storageType;
-
- public DatanodeInfoWithStorage(DatanodeInfo from, String storageID,
- StorageType storageType) {
- super(from);
- this.storageID = storageID;
- this.storageType = storageType;
- }
-
- public String getStorageID() {
- return storageID;
- }
-
- public StorageType getStorageType() {
- return storageType;
- }
-
- @Override
- public boolean equals(Object o) {
- // allows this class to be used interchangeably with DatanodeInfo
- return super.equals(o);
- }
-
- @Override
- public int hashCode() {
- // allows this class to be used interchangeably with DatanodeInfo
- return super.hashCode();
- }
-
- @Override
- public String toString() {
- return "DatanodeInfoWithStorage[" + super.toString() + "," + storageID +
- "," + storageType + "]";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/260b5e32/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index adf31a0..059eac0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.hdfs.server.protocol.DatanodeInfoWithStorage;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfoWithStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.junit.Assert;
@@ -274,15 +274,15 @@ public class TestDatanodeManager {
dm.sortLocatedBlocks(targetIp, blocks);
// check that storage IDs/types are aligned with datanode locs
- DatanodeInfoWithStorage[] sortedLocs = block.getLocations();
+ DatanodeInfo[] sortedLocs = block.getLocations();
storageIDs = block.getStorageIDs();
storageTypes = block.getStorageTypes();
assertThat(sortedLocs.length, is(5));
assertThat(storageIDs.length, is(5));
assertThat(storageTypes.length, is(5));
for(int i = 0; i < sortedLocs.length; i++) {
- assertThat(sortedLocs[i].getStorageID(), is(storageIDs[i]));
- assertThat(sortedLocs[i].getStorageType(), is(storageTypes[i]));
+ assertThat(((DatanodeInfoWithStorage)sortedLocs[i]).getStorageID(), is(storageIDs[i]));
+ assertThat(((DatanodeInfoWithStorage)sortedLocs[i]).getStorageType(), is(storageTypes[i]));
}
// Ensure the local node is first.
[31/50] [abbrv] hadoop git commit: HADOOP-11559. Add links to
RackAwareness and InterfaceClassification to site index (Masatake Iwasaki via
aw)
Posted by zj...@apache.org.
HADOOP-11559. Add links to RackAwareness and InterfaceClassification to site index (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7eeca90d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7eeca90d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7eeca90d
Branch: refs/heads/YARN-2928
Commit: 7eeca90daabd74934d4c94af6f07fd598abdb4ed
Parents: d5855c0
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 10 17:06:03 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 10 17:06:03 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../site/markdown/InterfaceClassification.md | 204 +++++++++++++++----
.../src/site/markdown/RackAwareness.md | 54 ++++-
hadoop-project/src/site/site.xml | 2 +
4 files changed, 208 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eeca90d/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1ba93e8..1536e44 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -377,6 +377,9 @@ Trunk (Unreleased)
HADOOP-10908. Common needs updates for shell rewrite (aw)
+ HADOOP-11559. Add links to RackAwareness and InterfaceClassification
+ to site index (Masatake Iwasaki via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eeca90d/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md b/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
index 0392610..9863611 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
@@ -26,80 +26,196 @@ Hadoop Interface Taxonomy: Audience and Stability Classification
Motivation
----------
-The interface taxonomy classification provided here is for guidance to developers and users of interfaces. The classification guides a developer to declare the targeted audience or users of an interface and also its stability.
+The interface taxonomy classification provided here is for guidance to
+developers and users of interfaces. The classification guides a developer to
+declare the targeted audience or users of an interface and also its stability.
* Benefits to the user of an interface: Knows which interfaces to use or not use and their stability.
-* Benefits to the developer: to prevent accidental changes of interfaces and hence accidental impact on users or other components or system. This is particularly useful in large systems with many developers who may not all have a shared state/history of the project.
+
+* Benefits to the developer: to prevent accidental changes of interfaces and
+ hence accidental impact on users or other components or system. This is
+ particularly useful in large systems with many developers who may not all have
+ a shared state/history of the project.
Interface Classification
------------------------
-Hadoop adopts the following interface classification, this classification was derived from the [OpenSolaris taxonomy](http://www.opensolaris.org/os/community/arc/policies/interface-taxonomy/#Advice) and, to some extent, from taxonomy used inside Yahoo. Interfaces have two main attributes: Audience and Stability
+Hadoop adopts the following interface classification,
+this classification was derived from the
+[OpenSolaris taxonomy](http://www.opensolaris.org/os/community/arc/policies/interface-taxonomy/#Advice)
+and, to some extent, from taxonomy used inside Yahoo.
+Interfaces have two main attributes: Audience and Stability
### Audience
-Audience denotes the potential consumers of the interface. While many interfaces are internal/private to the implementation, other are public/external interfaces are meant for wider consumption by applications and/or clients. For example, in posix, libc is an external or public interface, while large parts of the kernel are internal or private interfaces. Also, some interfaces are targeted towards other specific subsystems.
+Audience denotes the potential consumers of the interface. While many interfaces
+are internal/private to the implementation, other are public/external interfaces
+are meant for wider consumption by applications and/or clients. For example, in
+posix, libc is an external or public interface, while large parts of the kernel
+are internal or private interfaces. Also, some interfaces are targeted towards
+other specific subsystems.
-Identifying the audience of an interface helps define the impact of breaking it. For instance, it might be okay to break the compatibility of an interface whose audience is a small number of specific subsystems. On the other hand, it is probably not okay to break a protocol interfaces that millions of Internet users depend on.
+Identifying the audience of an interface helps define the impact of breaking
+it. For instance, it might be okay to break the compatibility of an interface
+whose audience is a small number of specific subsystems. On the other hand, it
+is probably not okay to break a protocol interfaces that millions of Internet
+users depend on.
Hadoop uses the following kinds of audience in order of increasing/wider visibility:
-* Private:
- * The interface is for internal use within the project (such as HDFS or MapReduce) and should not be used by applications or by other projects. It is subject to change at anytime without notice. Most interfaces of a project are Private (also referred to as project-private).
-* Limited-Private:
- * The interface is used by a specified set of projects or systems (typically closely related projects). Other projects or systems should not use the interface. Changes to the interface will be communicated/ negotiated with the specified projects. For example, in the Hadoop project, some interfaces are LimitedPrivate{HDFS, MapReduce} in that they are private to the HDFS and MapReduce projects.
-* Public
- * The interface is for general use by any application.
+> Hadoop doesn't have a Company-Private classification, which is meant for APIs
+> which are intended to be used by other projects within the company, since it
+> doesn't apply to opensource projects. Also, certain APIs are annotated as
+> @VisibleForTesting (from com.google.common .annotations.VisibleForTesting) -
+> these are meant to be used strictly for unit tests and should be treated as
+> "Private" APIs.
+
+#### Private
+
+The interface is for internal use within the project (such as HDFS or MapReduce)
+and should not be used by applications or by other projects. It is subject to
+change at anytime without notice. Most interfaces of a project are Private (also
+referred to as project-private).
+
+#### Limited-Private
+
+The interface is used by a specified set of projects or systems (typically
+closely related projects). Other projects or systems should not use the
+interface. Changes to the interface will be communicated/ negotiated with the
+specified projects. For example, in the Hadoop project, some interfaces are
+LimitedPrivate{HDFS, MapReduce} in that they are private to the HDFS and
+MapReduce projects.
-Hadoop doesn't have a Company-Private classification, which is meant for APIs which are intended to be used by other projects within the company, since it doesn't apply to opensource projects. Also, certain APIs are annotated as @VisibleForTesting (from com.google.common .annotations.VisibleForTesting) - these are meant to be used strictly for unit tests and should be treated as "Private" APIs.
+#### Public
+
+The interface is for general use by any application.
### Stability
-Stability denotes how stable an interface is, as in when incompatible changes to the interface are allowed. Hadoop APIs have the following levels of stability.
+Stability denotes how stable an interface is, as in when incompatible changes to
+the interface are allowed. Hadoop APIs have the following levels of stability.
+
+#### Stable
+
+Can evolve while retaining compatibility for minor release boundaries; in other
+words, incompatible changes to APIs marked Stable are allowed only at major
+releases (i.e. at m.0).
+
+#### Evolving
+
+Evolving, but incompatible changes are allowed at minor release (i.e. m .x)
-* Stable
- * Can evolve while retaining compatibility for minor release boundaries; in other words, incompatible changes to APIs marked Stable are allowed only at major releases (i.e. at m.0).
-* Evolving
- * Evolving, but incompatible changes are allowed at minor release (i.e. m .x)
-* Unstable
- * Incompatible changes to Unstable APIs are allowed any time. This usually makes sense for only private interfaces.
- * However one may call this out for a supposedly public interface to highlight that it should not be used as an interface; for public interfaces, labeling it as Not-an-interface is probably more appropriate than "Unstable".
- * Examples of publicly visible interfaces that are unstable (i.e. not-an-interface): GUI, CLIs whose output format will change
-* Deprecated
- * APIs that could potentially removed in the future and should not be used.
+#### Unstable
+
+Incompatible changes to Unstable APIs are allowed any time. This usually makes
+sense for only private interfaces.
+
+However one may call this out for a supposedly public interface to highlight
+that it should not be used as an interface; for public interfaces, labeling it
+as Not-an-interface is probably more appropriate than "Unstable".
+
+Examples of publicly visible interfaces that are unstable
+(i.e. not-an-interface): GUI, CLIs whose output format will change
+
+#### Deprecated
+
+APIs that could potentially removed in the future and should not be used.
How are the Classifications Recorded?
-------------------------------------
How will the classification be recorded for Hadoop APIs?
-* Each interface or class will have the audience and stability recorded using annotations in org.apache.hadoop.classification package.
+* Each interface or class will have the audience and stability recorded using
+ annotations in org.apache.hadoop.classification package.
+
* The javadoc generated by the maven target javadoc:javadoc lists only the public API.
-* One can derive the audience of java classes and java interfaces by the audience of the package in which they are contained. Hence it is useful to declare the audience of each java package as public or private (along with the private audience variations).
+
+* One can derive the audience of java classes and java interfaces by the
+ audience of the package in which they are contained. Hence it is useful to
+ declare the audience of each java package as public or private (along with the
+ private audience variations).
FAQ
---
* Why aren’t the java scopes (private, package private and public) good enough?
- * Java’s scoping is not very complete. One is often forced to make a class public in order for other internal components to use it. It does not have friends or sub-package-private like C++.
-* But I can easily access a private implementation interface if it is Java public. Where is the protection and control?
- * The purpose of this is not providing absolute access control. Its purpose is to communicate to users and developers. One can access private implementation functions in libc; however if they change the internal implementation details, your application will break and you will have little sympathy from the folks who are supplying libc. If you use a non-public interface you understand the risks.
-* Why bother declaring the stability of a private interface? Aren’t private interfaces always unstable?
- * Private interfaces are not always unstable. In the cases where they are stable they capture internal properties of the system and can communicate these properties to its internal users and to developers of the interface.
- * e.g. In HDFS, NN-DN protocol is private but stable and can help implement rolling upgrades. It communicates that this interface should not be changed in incompatible ways even though it is private.
+ * Java’s scoping is not very complete. One is often forced to make a class
+ public in order for other internal components to use it. It does not have
+ friends or sub-package-private like C++.
+
+* But I can easily access a private implementation interface if it is Java public.
+ Where is the protection and control?
+ * The purpose of this is not providing absolute access control. Its purpose
+ is to communicate to users and developers. One can access private
+ implementation functions in libc; however if they change the internal
+ implementation details, your application will break and you will have
+ little sympathy from the folks who are supplying libc. If you use a
+ non-public interface you understand the risks.
+
+* Why bother declaring the stability of a private interface?
+ Aren’t private interfaces always unstable?
+ * Private interfaces are not always unstable. In the cases where they are
+ stable they capture internal properties of the system and can communicate
+ these properties to its internal users and to developers of the interface.
+ * e.g. In HDFS, NN-DN protocol is private but stable and can help
+ implement rolling upgrades. It communicates that this interface should
+ not be changed in incompatible ways even though it is private.
* e.g. In HDFS, FSImage stability can help provide more flexible roll backs.
-* What is the harm in applications using a private interface that is stable? How is it different than a public stable interface?
- * While a private interface marked as stable is targeted to change only at major releases, it may break at other times if the providers of that interface are willing to changes the internal users of that interface. Further, a public stable interface is less likely to break even at major releases (even though it is allowed to break compatibility) because the impact of the change is larger. If you use a private interface (regardless of its stability) you run the risk of incompatibility.
-* Why bother with Limited-private? Isn’t it giving special treatment to some projects? That is not fair.
- * First, most interfaces should be public or private; actually let us state it even stronger: make it private unless you really want to expose it to public for general use.
- * Limited-private is for interfaces that are not intended for general use. They are exposed to related projects that need special hooks. Such a classification has a cost to both the supplier and consumer of the limited interface. Both will have to work together if ever there is a need to break the interface in the future; for example the supplier and the consumers will have to work together to get coordinated releases of their respective projects. This should not be taken lightly – if you can get away with private then do so; if the interface is really for general use for all applications then do so. But remember that making an interface public has huge responsibility. Sometimes Limited-private is just right.
- * A good example of a limited-private interface is BlockLocations, This is fairly low-level interface that we are willing to expose to MR and perhaps HBase. We are likely to change it down the road and at that time we will have get a coordinated effort with the MR team to release matching releases. While MR and HDFS are always released in sync today, they may change down the road.
- * If you have a limited-private interface with many projects listed then you are fooling yourself. It is practically public.
- * It might be worth declaring a special audience classification called Hadoop-Private for the Hadoop family.
-* Lets treat all private interfaces as Hadoop-private. What is the harm in projects in the Hadoop family have access to private classes?
- * Do we want MR accessing class files that are implementation details inside HDFS. There used to be many such layer violations in the code that we have been cleaning up over the last few years. We don’t want such layer violations to creep back in by no separating between the major components like HDFS and MR.
-* Aren't all public interfaces stable?
- * One may mark a public interface as evolving in its early days. Here one is promising to make an effort to make compatible changes but may need to break it at minor releases.
- * One example of a public interface that is unstable is where one is providing an implementation of a standards-body based interface that is still under development. For example, many companies, in an attampt to be first to market, have provided implementations of a new NFS protocol even when the protocol was not fully completed by IETF. The implementor cannot evolve the interface in a fashion that causes least distruption because the stability is controlled by the standards body. Hence it is appropriate to label the interface as unstable.
+* What is the harm in applications using a private interface that is stable? How
+ is it different than a public stable interface?
+ * While a private interface marked as stable is targeted to change only at
+ major releases, it may break at other times if the providers of that
+ interface are willing to changes the internal users of that
+ interface. Further, a public stable interface is less likely to break even
+ at major releases (even though it is allowed to break compatibility)
+ because the impact of the change is larger. If you use a private interface
+ (regardless of its stability) you run the risk of incompatibility.
+
+* Why bother with Limited-private? Isn’t it giving special treatment to some projects?
+ That is not fair.
+ * First, most interfaces should be public or private; actually let us state
+ it even stronger: make it private unless you really want to expose it to
+ public for general use.
+ * Limited-private is for interfaces that are not intended for general
+ use. They are exposed to related projects that need special hooks. Such a
+ classification has a cost to both the supplier and consumer of the limited
+ interface. Both will have to work together if ever there is a need to
+ break the interface in the future; for example the supplier and the
+ consumers will have to work together to get coordinated releases of their
+ respective projects. This should not be taken lightly – if you can get
+ away with private then do so; if the interface is really for general use
+ for all applications then do so. But remember that making an interface
+ public has huge responsibility. Sometimes Limited-private is just right.
+ * A good example of a limited-private interface is BlockLocations, This is
+ fairly low-level interface that we are willing to expose to MR and perhaps
+ HBase. We are likely to change it down the road and at that time we will
+ have get a coordinated effort with the MR team to release matching
+ releases. While MR and HDFS are always released in sync today, they may
+ change down the road.
+ * If you have a limited-private interface with many projects listed then you
+ are fooling yourself. It is practically public.
+ * It might be worth declaring a special audience classification called
+ Hadoop-Private for the Hadoop family.
+
+* Lets treat all private interfaces as Hadoop-private. What is the harm in
+ projects in the Hadoop family have access to private classes?
+ * Do we want MR accessing class files that are implementation details inside
+ HDFS. There used to be many such layer violations in the code that we have
+ been cleaning up over the last few years. We don’t want such layer
+ violations to creep back in by no separating between the major components
+ like HDFS and MR.
+* Aren't all public interfaces stable?
+ * One may mark a public interface as evolving in its early days. Here one is
+ promising to make an effort to make compatible changes but may need to
+ break it at minor releases.
+ * One example of a public interface that is unstable is where one is
+ providing an implementation of a standards-body based interface that is
+ still under development. For example, many companies, in an attampt to be
+ first to market, have provided implementations of a new NFS protocol even
+ when the protocol was not fully completed by IETF. The implementor cannot
+ evolve the interface in a fashion that causes least distruption because
+ the stability is controlled by the standards body. Hence it is appropriate
+ to label the interface as unstable.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eeca90d/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
index 41fcb37..c5ab19a 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -19,17 +19,49 @@
Rack Awareness
==============
-Hadoop components are rack-aware. For example, HDFS block placement will use rack awareness for fault tolerance by placing one block replica on a different rack. This provides data availability in the event of a network switch failure or partition within the cluster.
-
-Hadoop master daemons obtain the rack id of the cluster slaves by invoking either an external script or java class as specified by configuration files. Using either the java class or external script for topology, output must adhere to the java **org.apache.hadoop.net.DNSToSwitchMapping** interface. The interface expects a one-to-one correspondence to be maintained and the topology information in the format of '/myrack/myhost', where '/' is the topology delimiter, 'myrack' is the rack identifier, and 'myhost' is the individual host. Assuming a single /24 subnet per rack, one could use the format of '/192.168.100.0/192.168.100.5' as a unique rack-host topology mapping.
-
-To use the java class for topology mapping, the class name is specified by the **topology.node.switch.mapping.impl** parameter in the configuration file. An example, NetworkTopology.java, is included with the hadoop distribution and can be customized by the Hadoop administrator. Using a Java class instead of an external script has a performance benefit in that Hadoop doesn't need to fork an external process when a new slave node registers itself.
-
-If implementing an external script, it will be specified with the **topology.script.file.name** parameter in the configuration files. Unlike the java class, the external topology script is not included with the Hadoop distribution and is provided by the administrator. Hadoop will send multiple IP addresses to ARGV when forking the topology script. The number of IP addresses sent to the topology script is controlled with **net.topology.script.number.args** and defaults to 100. If **net.topology.script.number.args** was changed to 1, a topology script would get forked for each IP submitted by DataNodes and/or NodeManagers.
-
-If **topology.script.file.name** or **topology.node.switch.mapping.impl** is not set, the rack id '/default-rack' is returned for any passed IP address. While this behavior appears desirable, it can cause issues with HDFS block replication as default behavior is to write one replicated block off rack and is unable to do so as there is only a single rack named '/default-rack'.
-
-An additional configuration setting is **mapreduce.jobtracker.taskcache.levels** which determines the number of levels (in the network topology) of caches MapReduce will use. So, for example, if it is the default value of 2, two levels of caches will be constructed - one for hosts (host -\> task mapping) and another for racks (rack -\> task mapping). Giving us our one-to-one mapping of '/myrack/myhost'.
+Hadoop components are rack-aware. For example, HDFS block placement
+will use rack awareness for fault tolerance by placing one block
+replica on a different rack. This provides data availability in the
+event of a network switch failure or partition within the cluster.
+
+Hadoop master daemons obtain the rack id of the cluster slaves by
+invoking either an external script or java class as specified by
+configuration files. Using either the java class or external script
+for topology, output must adhere to the java
+**org.apache.hadoop.net.DNSToSwitchMapping** interface. The interface
+expects a one-to-one correspondence to be maintained and the topology
+information in the format of '/myrack/myhost', where '/' is the
+topology delimiter, 'myrack' is the rack identifier, and 'myhost' is
+the individual host. Assuming a single /24 subnet per rack, one could
+use the format of '/192.168.100.0/192.168.100.5' as a unique rack-host
+topology mapping.
+
+To use the java class for topology mapping, the class name is
+specified by the **net.topology.node.switch.mapping.impl** parameter
+in the configuration file. An example, NetworkTopology.java, is
+included with the hadoop distribution and can be customized by the
+Hadoop administrator. Using a Java class instead of an external script
+has a performance benefit in that Hadoop doesn't need to fork an
+external process when a new slave node registers itself.
+
+If implementing an external script, it will be specified with the
+**net.topology.script.file.name** parameter in the configuration
+files. Unlike the java class, the external topology script is not
+included with the Hadoop distribution and is provided by the
+administrator. Hadoop will send multiple IP addresses to ARGV when
+forking the topology script. The number of IP addresses sent to the
+topology script is controlled with **net.topology.script.number.args**
+and defaults to 100. If **net.topology.script.number.args** was
+changed to 1, a topology script would get forked for each IP submitted
+by DataNodes and/or NodeManagers.
+
+If **net.topology.script.file.name** or
+**net.topology.node.switch.mapping.impl** is not set, the rack id
+'/default-rack' is returned for any passed IP address. While this
+behavior appears desirable, it can cause issues with HDFS block
+replication as default behavior is to write one replicated block off
+rack and is unable to do so as there is only a single rack named
+'/default-rack'.
python Example
--------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7eeca90d/hadoop-project/src/site/site.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/src/site/site.xml b/hadoop-project/src/site/site.xml
index 4077738..1deba81 100644
--- a/hadoop-project/src/site/site.xml
+++ b/hadoop-project/src/site/site.xml
@@ -53,6 +53,7 @@
<item name="Hadoop Commands Reference" href="hadoop-project-dist/hadoop-common/CommandsManual.html"/>
<item name="FileSystem Shell" href="hadoop-project-dist/hadoop-common/FileSystemShell.html"/>
<item name="Hadoop Compatibility" href="hadoop-project-dist/hadoop-common/Compatibility.html"/>
+ <item name="Interface Classification" href="hadoop-project-dist/hadoop-common/InterfaceClassification.html"/>
<item name="FileSystem Specification"
href="hadoop-project-dist/hadoop-common/filesystem/index.html"/>
</menu>
@@ -61,6 +62,7 @@
<item name="CLI Mini Cluster" href="hadoop-project-dist/hadoop-common/CLIMiniCluster.html"/>
<item name="Native Libraries" href="hadoop-project-dist/hadoop-common/NativeLibraries.html"/>
<item name="Proxy User" href="hadoop-project-dist/hadoop-common/Superusers.html"/>
+ <item name="Rack Awareness" href="hadoop-project-dist/hadoop-common/RackAwareness.html"/>
<item name="Secure Mode" href="hadoop-project-dist/hadoop-common/SecureMode.html"/>
<item name="Service Level Authorization" href="hadoop-project-dist/hadoop-common/ServiceLevelAuth.html"/>
<item name="HTTP Authentication" href="hadoop-project-dist/hadoop-common/HttpAuthentication.html"/>
[45/50] [abbrv] hadoop git commit: HDFS-7723. Quota By Storage Type
namenode implemenation. (Contributed by Xiaoyu Yao)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
index ebe4603..f9fc91c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FSImageFormatPBSnapshot.java
@@ -34,9 +34,11 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import com.google.common.collect.ImmutableList;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -66,12 +68,15 @@ import org.apache.hadoop.hdfs.server.namenode.INodeReference.DstReference;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
import org.apache.hadoop.hdfs.server.namenode.INodeWithAdditionalFields;
+import org.apache.hadoop.hdfs.server.namenode.QuotaByStorageTypeEntry;
import org.apache.hadoop.hdfs.server.namenode.SaveNamespaceContext;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot.Root;
import org.apache.hadoop.hdfs.server.namenode.XAttrFeature;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.util.Diff.ListType;
+import org.apache.hadoop.hdfs.util.EnumCounters;
import com.google.common.base.Preconditions;
import com.google.protobuf.ByteString;
@@ -343,13 +348,31 @@ public class FSImageFormatPBSnapshot {
long modTime = dirCopyInPb.getModificationTime();
boolean noQuota = dirCopyInPb.getNsQuota() == -1
- && dirCopyInPb.getDsQuota() == -1;
-
- copy = noQuota ? new INodeDirectoryAttributes.SnapshotCopy(name,
- permission, acl, modTime, xAttrs)
- : new INodeDirectoryAttributes.CopyWithQuota(name, permission,
- acl, modTime, dirCopyInPb.getNsQuota(),
- dirCopyInPb.getDsQuota(), xAttrs);
+ && dirCopyInPb.getDsQuota() == -1
+ && (!dirCopyInPb.hasTypeQuotas());
+
+ if (noQuota) {
+ copy = new INodeDirectoryAttributes.SnapshotCopy(name,
+ permission, acl, modTime, xAttrs);
+ } else {
+ EnumCounters<StorageType> typeQuotas = null;
+ if (dirCopyInPb.hasTypeQuotas()) {
+ ImmutableList<QuotaByStorageTypeEntry> qes =
+ FSImageFormatPBINode.Loader.loadQuotaByStorageTypeEntries(
+ dirCopyInPb.getTypeQuotas());
+ typeQuotas = new EnumCounters<StorageType>(StorageType.class,
+ HdfsConstants.QUOTA_RESET);
+ for (QuotaByStorageTypeEntry qe : qes) {
+ if (qe.getQuota() >= 0 && qe.getStorageType() != null &&
+ qe.getStorageType().supportTypeQuota()) {
+ typeQuotas.set(qe.getStorageType(), qe.getQuota());
+ }
+ }
+ }
+ copy = new INodeDirectoryAttributes.CopyWithQuota(name, permission,
+ acl, modTime, dirCopyInPb.getNsQuota(),
+ dirCopyInPb.getDsQuota(), typeQuotas, xAttrs);
+ }
}
// load created list
List<INode> clist = loadCreatedList(in, dir,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
index 10bba67..931f7f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiff.java
@@ -23,12 +23,13 @@ import java.util.Arrays;
import java.util.List;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
/**
@@ -80,13 +81,14 @@ public class FileDiff extends
}
@Override
- Quota.Counts combinePosteriorAndCollectBlocks(INodeFile currentINode,
+ QuotaCounts combinePosteriorAndCollectBlocks(
+ BlockStoragePolicySuite bsps, INodeFile currentINode,
FileDiff posterior, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
FileWithSnapshotFeature sf = currentINode.getFileWithSnapshotFeature();
assert sf != null : "FileWithSnapshotFeature is null";
return sf.updateQuotaAndCollectBlocks(
- currentINode, posterior, collectedBlocks, removedINodes);
+ bsps, currentINode, posterior, collectedBlocks, removedINodes);
}
@Override
@@ -110,10 +112,10 @@ public class FileDiff extends
}
@Override
- Quota.Counts destroyDiffAndCollectBlocks(INodeFile currentINode,
+ QuotaCounts destroyDiffAndCollectBlocks(BlockStoragePolicySuite bsps, INodeFile currentINode,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
return currentINode.getFileWithSnapshotFeature()
- .updateQuotaAndCollectBlocks(currentINode, this, collectedBlocks,
+ .updateQuotaAndCollectBlocks(bsps, currentINode, this, collectedBlocks,
removedINodes);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
index 5ea23dc..f9f5056 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileDiffList.java
@@ -21,6 +21,7 @@ import java.util.Collections;
import java.util.List;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
@@ -93,7 +94,7 @@ public class FileDiffList extends
* up to the file length of the latter.
* Collect unused blocks of the removed snapshot.
*/
- void combineAndCollectSnapshotBlocks(INodeFile file,
+ void combineAndCollectSnapshotBlocks(BlockStoragePolicySuite bsps, INodeFile file,
FileDiff removed,
BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes) {
@@ -102,7 +103,7 @@ public class FileDiffList extends
FileWithSnapshotFeature sf = file.getFileWithSnapshotFeature();
assert sf != null : "FileWithSnapshotFeature is null";
if(sf.isCurrentFileDeleted())
- sf.collectBlocksAndClear(file, collectedBlocks, removedINodes);
+ sf.collectBlocksAndClear(bsps, file, collectedBlocks, removedINodes);
return;
}
int p = getPrior(removed.getSnapshotId(), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
index 419557b..6bcdbd7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/FileWithSnapshotFeature.java
@@ -21,13 +21,17 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.AclFeature;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.AclStorage;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeFileAttributes;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.util.EnumCounters;
/**
* Feature for file with snapshot-related information.
@@ -114,7 +118,8 @@ public class FileWithSnapshotFeature implements INode.Feature {
return (isCurrentFileDeleted()? "(DELETED), ": ", ") + diffs;
}
- public Quota.Counts cleanFile(final INodeFile file, final int snapshotId,
+ public QuotaCounts cleanFile(final BlockStoragePolicySuite bsps,
+ final INodeFile file, final int snapshotId,
int priorSnapshotId, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
if (snapshotId == Snapshot.CURRENT_STATE_ID) {
@@ -123,11 +128,11 @@ public class FileWithSnapshotFeature implements INode.Feature {
file.recordModification(priorSnapshotId);
deleteCurrentFile();
}
- collectBlocksAndClear(file, collectedBlocks, removedINodes);
- return Quota.Counts.newInstance();
+ collectBlocksAndClear(bsps, file, collectedBlocks, removedINodes);
+ return new QuotaCounts.Builder().build();
} else { // delete the snapshot
priorSnapshotId = getDiffs().updatePrior(snapshotId, priorSnapshotId);
- return diffs.deleteSnapshotDiff(snapshotId, priorSnapshotId, file,
+ return diffs.deleteSnapshotDiff(bsps, snapshotId, priorSnapshotId, file,
collectedBlocks, removedINodes);
}
}
@@ -136,17 +141,52 @@ public class FileWithSnapshotFeature implements INode.Feature {
this.diffs.clear();
}
- public Quota.Counts updateQuotaAndCollectBlocks(INodeFile file,
+ public QuotaCounts updateQuotaAndCollectBlocks(BlockStoragePolicySuite bsps, INodeFile file,
FileDiff removed, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
long oldDiskspace = file.diskspaceConsumed();
+
+ byte storagePolicyID = file.getStoragePolicyID();
+ BlockStoragePolicy bsp = null;
+ EnumCounters<StorageType> typeSpaces =
+ new EnumCounters<StorageType>(StorageType.class);
+ if (storagePolicyID != BlockStoragePolicySuite.ID_UNSPECIFIED) {
+ bsp = bsps.getPolicy(file.getStoragePolicyID());
+ }
+
if (removed.snapshotINode != null) {
short replication = removed.snapshotINode.getFileReplication();
short currentRepl = file.getBlockReplication();
if (currentRepl == 0) {
- oldDiskspace = file.computeFileSize(true, true) * replication;
- } else if (replication > currentRepl) {
- oldDiskspace = oldDiskspace / file.getBlockReplication() * replication;
+ long oldFileSizeNoRep = file.computeFileSize(true, true);
+ oldDiskspace = oldFileSizeNoRep * replication;
+
+ if (bsp != null) {
+ List<StorageType> oldTypeChosen = bsp.chooseStorageTypes(replication);
+ for (StorageType t : oldTypeChosen) {
+ if (t.supportTypeQuota()) {
+ typeSpaces.add(t, -oldFileSizeNoRep);
+ }
+ }
+ }
+ } else if (replication > currentRepl) {
+ long oldFileSizeNoRep = file.diskspaceConsumedNoReplication();
+ oldDiskspace = oldFileSizeNoRep * replication;
+
+ if (bsp != null) {
+ List<StorageType> oldTypeChosen = bsp.chooseStorageTypes(replication);
+ for (StorageType t : oldTypeChosen) {
+ if (t.supportTypeQuota()) {
+ typeSpaces.add(t, -oldFileSizeNoRep);
+ }
+ }
+ List<StorageType> newTypeChosen = bsp.chooseStorageTypes(currentRepl);
+ for (StorageType t: newTypeChosen) {
+ if (t.supportTypeQuota()) {
+ typeSpaces.add(t, oldFileSizeNoRep);
+ }
+ }
+ }
}
AclFeature aclFeature = removed.getSnapshotINode().getAclFeature();
if (aclFeature != null) {
@@ -155,21 +195,24 @@ public class FileWithSnapshotFeature implements INode.Feature {
}
getDiffs().combineAndCollectSnapshotBlocks(
- file, removed, collectedBlocks, removedINodes);
+ bsps, file, removed, collectedBlocks, removedINodes);
long dsDelta = oldDiskspace - file.diskspaceConsumed();
- return Quota.Counts.newInstance(0, dsDelta);
+ return new QuotaCounts.Builder().
+ spaceCount(dsDelta).
+ typeCounts(typeSpaces).
+ build();
}
/**
* If some blocks at the end of the block list no longer belongs to
* any inode, collect them and update the block list.
*/
- public void collectBlocksAndClear(final INodeFile file,
+ public void collectBlocksAndClear(final BlockStoragePolicySuite bsps, final INodeFile file,
final BlocksMapUpdateInfo info, final List<INode> removedINodes) {
// check if everything is deleted.
if (isCurrentFileDeleted() && getDiffs().asList().isEmpty()) {
- file.destroyAndCollectBlocks(info, removedINodes);
+ file.destroyAndCollectBlocks(bsps, info, removedINodes);
return;
}
// find max file size.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
index ba7c43c..c3b7523 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/SnapshotManager.java
@@ -234,7 +234,8 @@ public class SnapshotManager implements SnapshotStatsMXBean {
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
throws IOException {
INodeDirectory srcRoot = getSnapshottableRoot(iip);
- srcRoot.removeSnapshot(snapshotName, collectedBlocks, removedINodes);
+ srcRoot.removeSnapshot(fsdir.getBlockStoragePolicySuite(), snapshotName,
+ collectedBlocks, removedINodes);
numSnapshots.getAndDecrement();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
index 8a8e61f..18f4bd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/util/EnumCounters.java
@@ -50,6 +50,14 @@ public class EnumCounters<E extends Enum<E>> {
this.enumClass = enumClass;
this.counters = new long[enumConstants.length];
}
+
+ public EnumCounters(final Class<E> enumClass, long defaultVal) {
+ final E[] enumConstants = enumClass.getEnumConstants();
+ Preconditions.checkNotNull(enumConstants);
+ this.enumClass = enumClass;
+ this.counters = new long[enumConstants.length];
+ reset(defaultVal);
+ }
/** @return the value of counter e. */
public final long get(final E e) {
@@ -77,9 +85,7 @@ public class EnumCounters<E extends Enum<E>> {
/** Reset all counters to zero. */
public final void reset() {
- for(int i = 0; i < counters.length; i++) {
- this.counters[i] = 0L;
- }
+ reset(0L);
}
/** Add the given value to counter e. */
@@ -143,6 +149,30 @@ public class EnumCounters<E extends Enum<E>> {
return b.substring(0, b.length() - 2);
}
+ public final void reset(long val) {
+ for(int i = 0; i < counters.length; i++) {
+ this.counters[i] = val;
+ }
+ }
+
+ public boolean allLessOrEqual(long val) {
+ for (long c : counters) {
+ if (c > val) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean anyGreaterOrEqual(long val) {
+ for (long c: counters) {
+ if (c >= val) {
+ return true;
+ }
+ }
+ return false;
+ }
+
/**
* A factory for creating counters.
*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
index 643a034..3bd1d91 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/fsimage.proto
@@ -141,6 +141,15 @@ message INodeSection {
optional uint32 storagePolicyID = 10;
}
+ message QuotaByStorageTypeEntryProto {
+ required StorageTypeProto storageType = 1;
+ required uint64 quota = 2;
+ }
+
+ message QuotaByStorageTypeFeatureProto {
+ repeated QuotaByStorageTypeEntryProto quotas = 1;
+ }
+
message INodeDirectory {
optional uint64 modificationTime = 1;
// namespace quota
@@ -150,6 +159,7 @@ message INodeSection {
optional fixed64 permission = 4;
optional AclFeatureProto acl = 5;
optional XAttrFeatureProto xAttrs = 6;
+ optional QuotaByStorageTypeFeatureProto typeQuotas = 7;
}
message INodeSymlink {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index e8fc662..f47bd24 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1184,6 +1184,8 @@ public class DFSTestUtil {
// OP_SET_QUOTA 14
filesystem.setQuota(pathDirectoryMkdir, 1000L,
HdfsConstants.QUOTA_DONT_SET);
+ // OP_SET_QUOTA_BY_STORAGETYPE
+ filesystem.setQuotaByStorageType(pathDirectoryMkdir, StorageType.SSD, 888L);
// OP_RENAME 15
fc.rename(pathFileCreate, pathFileMoved, Rename.NONE);
// OP_CONCAT_DELETE 16
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
index 4f449d1..6e91e06 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSInotifyEventInputStream.java
@@ -72,7 +72,7 @@ public class TestDFSInotifyEventInputStream {
*/
@Test
public void testOpcodeCount() {
- Assert.assertEquals(49, FSEditLogOpCodes.values().length);
+ Assert.assertEquals(50, FSEditLogOpCodes.values().length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
index d3f122f..445c1e1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSImageTestUtil.java
@@ -554,7 +554,7 @@ public abstract class FSImageTestUtil {
* get NameSpace quota.
*/
public static long getNSQuota(FSNamesystem ns) {
- return ns.dir.rootDir.getQuotaCounts().get(Quota.NAMESPACE);
+ return ns.dir.rootDir.getQuotaCounts().getNameSpace();
}
public static void assertNNFilesMatch(MiniDFSCluster cluster) throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
index fad8807..ede2f89 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDiskspaceQuotaUpdate.java
@@ -81,10 +81,10 @@ public class TestDiskspaceQuotaUpdate {
INode fnode = fsdir.getINode4Write(foo.toString());
assertTrue(fnode.isDirectory());
assertTrue(fnode.isQuotaSet());
- Quota.Counts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
.getSpaceConsumed();
- assertEquals(2, cnt.get(Quota.NAMESPACE));
- assertEquals(fileLen * REPLICATION, cnt.get(Quota.DISKSPACE));
+ assertEquals(2, cnt.getNameSpace());
+ assertEquals(fileLen * REPLICATION, cnt.getDiskSpace());
}
/**
@@ -105,10 +105,10 @@ public class TestDiskspaceQuotaUpdate {
INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
assertTrue(fooNode.isQuotaSet());
- Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature()
+ QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed();
- long ns = quota.get(Quota.NAMESPACE);
- long ds = quota.get(Quota.DISKSPACE);
+ long ns = quota.getNameSpace();
+ long ds = quota.getDiskSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
ContentSummary c = dfs.getContentSummary(foo);
@@ -119,8 +119,8 @@ public class TestDiskspaceQuotaUpdate {
currentFileLen += BLOCKSIZE;
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
- ns = quota.get(Quota.NAMESPACE);
- ds = quota.get(Quota.DISKSPACE);
+ ns = quota.getNameSpace();
+ ds = quota.getDiskSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
c = dfs.getContentSummary(foo);
@@ -131,8 +131,8 @@ public class TestDiskspaceQuotaUpdate {
currentFileLen += (BLOCKSIZE * 3 + BLOCKSIZE / 8);
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
- ns = quota.get(Quota.NAMESPACE);
- ds = quota.get(Quota.DISKSPACE);
+ ns = quota.getNameSpace();
+ ds = quota.getDiskSpace();
assertEquals(2, ns); // foo and bar
assertEquals(currentFileLen * REPLICATION, ds);
c = dfs.getContentSummary(foo);
@@ -156,10 +156,10 @@ public class TestDiskspaceQuotaUpdate {
.of(HdfsDataOutputStream.SyncFlag.UPDATE_LENGTH));
INodeDirectory fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
- Quota.Counts quota = fooNode.getDirectoryWithQuotaFeature()
+ QuotaCounts quota = fooNode.getDirectoryWithQuotaFeature()
.getSpaceConsumed();
- long ns = quota.get(Quota.NAMESPACE);
- long ds = quota.get(Quota.DISKSPACE);
+ long ns = quota.getNameSpace();
+ long ds = quota.getDiskSpace();
assertEquals(2, ns); // foo and bar
assertEquals(BLOCKSIZE * 2 * REPLICATION, ds); // file is under construction
@@ -168,8 +168,8 @@ public class TestDiskspaceQuotaUpdate {
fooNode = fsdir.getINode4Write(foo.toString()).asDirectory();
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
- ns = quota.get(Quota.NAMESPACE);
- ds = quota.get(Quota.DISKSPACE);
+ ns = quota.getNameSpace();
+ ds = quota.getDiskSpace();
assertEquals(2, ns);
assertEquals((BLOCKSIZE + BLOCKSIZE / 2) * REPLICATION, ds);
@@ -177,8 +177,8 @@ public class TestDiskspaceQuotaUpdate {
DFSTestUtil.appendFile(dfs, bar, BLOCKSIZE);
quota = fooNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
- ns = quota.get(Quota.NAMESPACE);
- ds = quota.get(Quota.DISKSPACE);
+ ns = quota.getNameSpace();
+ ds = quota.getDiskSpace();
assertEquals(2, ns); // foo and bar
assertEquals((BLOCKSIZE * 2 + BLOCKSIZE / 2) * REPLICATION, ds);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
index 8b829e4..741dd10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImageWithSnapshot.java
@@ -158,7 +158,7 @@ public class TestFSImageWithSnapshot {
fsn.getFSDirectory().writeLock();
try {
loader.load(imageFile, false);
- FSImage.updateCountForQuota(
+ FSImage.updateCountForQuota(fsn.getBlockManager().getStoragePolicySuite(),
INodeDirectory.valueOf(fsn.getFSDirectory().getINode("/"), "/"));
} finally {
fsn.getFSDirectory().writeUnlock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
new file mode 100644
index 0000000..b0d5d87
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestQuotaByStorageType.java
@@ -0,0 +1,524 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+ import static org.apache.hadoop.hdfs.StorageType.DEFAULT;
+ import static org.junit.Assert.assertEquals;
+ import static org.junit.Assert.assertFalse;
+ import static org.junit.Assert.assertTrue;
+ import static org.junit.Assert.fail;
+
+ import org.apache.commons.logging.Log;
+ import org.apache.commons.logging.LogFactory;
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.Path;
+ import org.apache.hadoop.hdfs.*;
+ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+ import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotTestHelper;
+ import org.junit.After;
+ import org.junit.Before;
+ import org.junit.Test;
+
+public class TestQuotaByStorageType {
+
+ private static final int BLOCKSIZE = 1024;
+ private static final short REPLICATION = 3;
+ static final long seed = 0L;
+ private static final Path dir = new Path("/TestQuotaByStorageType");
+
+ private Configuration conf;
+ private MiniDFSCluster cluster;
+ private FSDirectory fsdir;
+ private DistributedFileSystem dfs;
+ private FSNamesystem fsn;
+
+ protected static final Log LOG = LogFactory.getLog(TestQuotaByStorageType.class);
+
+ @Before
+ public void setUp() throws Exception {
+ conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCKSIZE);
+
+ // Setup a 3-node cluster and configure
+ // each node with 1 SSD and 1 DISK without capacity limitation
+ cluster = new MiniDFSCluster
+ .Builder(conf)
+ .numDataNodes(REPLICATION)
+ .storageTypes(new StorageType[]{StorageType.SSD, DEFAULT})
+ .build();
+ cluster.waitActive();
+
+ fsdir = cluster.getNamesystem().getFSDirectory();
+ dfs = cluster.getFileSystem();
+ fsn = cluster.getNamesystem();
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithFileCreateOneSSD() throws Exception {
+ testQuotaByStorageTypeWithFileCreateCase(
+ HdfsConstants.ONESSD_STORAGE_POLICY_NAME,
+ StorageType.SSD,
+ (short)1);
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithFileCreateAllSSD() throws Exception {
+ testQuotaByStorageTypeWithFileCreateCase(
+ HdfsConstants.ALLSSD_STORAGE_POLICY_NAME,
+ StorageType.SSD,
+ (short)3);
+ }
+
+ void testQuotaByStorageTypeWithFileCreateCase(
+ String storagePolicy, StorageType storageType, short replication) throws Exception {
+ final Path foo = new Path(dir, "foo");
+ Path createdFile1 = new Path(foo, "created_file1.data");
+ dfs.mkdirs(foo);
+
+ // set storage policy on directory "foo" to storagePolicy
+ dfs.setStoragePolicy(foo, storagePolicy);
+
+ // set quota by storage type on directory "foo"
+ dfs.setQuotaByStorageType(foo, storageType, BLOCKSIZE * 10);
+
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ // Create file of size 2 * BLOCKSIZE under directory "foo"
+ long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+ // Verify space consumed and remaining quota
+ long storageTypeConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(storageType);
+ assertEquals(file1Len * replication, storageTypeConsumed);
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithFileCreateAppend() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ Path createdFile1 = new Path(foo, "created_file1.data");
+ dfs.mkdirs(foo);
+
+ // set storage policy on directory "foo" to ONESSD
+ dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+ // set quota by storage type on directory "foo"
+ dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 4);
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ // Create file of size 2 * BLOCKSIZE under directory "foo"
+ long file1Len = BLOCKSIZE * 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+ // Verify space consumed and remaining quota
+ long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, ssdConsumed);
+
+ // append several blocks
+ int appendLen = BLOCKSIZE * 2;
+ DFSTestUtil.appendFile(dfs, createdFile1, appendLen);
+ file1Len += appendLen;
+
+ ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, ssdConsumed);
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithFileCreateDelete() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ Path createdFile1 = new Path(foo, "created_file1.data");
+ dfs.mkdirs(foo);
+
+ dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+ // set quota by storage type on directory "foo"
+ dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 10);
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ // Create file of size 2.5 * BLOCKSIZE under directory "foo"
+ long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+ // Verify space consumed and remaining quota
+ long storageTypeConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, storageTypeConsumed);
+
+ // Delete file and verify the consumed space of the storage type is updated
+ dfs.delete(createdFile1, false);
+ storageTypeConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(0, storageTypeConsumed);
+
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ fnode.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), counts, true);
+ assertEquals(fnode.dumpTreeRecursively().toString(), 0,
+ counts.getTypeSpaces().get(StorageType.SSD));
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithFileCreateRename() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ dfs.mkdirs(foo);
+ Path createdFile1foo = new Path(foo, "created_file1.data");
+
+ final Path bar = new Path(dir, "bar");
+ dfs.mkdirs(bar);
+ Path createdFile1bar = new Path(bar, "created_file1.data");
+
+ // set storage policy on directory "foo" and "bar" to ONESSD
+ dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ dfs.setStoragePolicy(bar, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+ // set quota by storage type on directory "foo"
+ dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 4);
+ dfs.setQuotaByStorageType(bar, StorageType.SSD, BLOCKSIZE * 2);
+
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ // Create file of size 3 * BLOCKSIZE under directory "foo"
+ long file1Len = BLOCKSIZE * 3;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1foo, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+ // Verify space consumed and remaining quota
+ long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ ;
+ assertEquals(file1Len, ssdConsumed);
+
+ // move file from foo to bar
+ try {
+ dfs.rename(createdFile1foo, createdFile1bar);
+ fail("Should have failed with QuotaByStorageTypeExceededException ");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+ }
+ }
+
+ /**
+ * Test if the quota can be correctly updated for create file even
+ * QuotaByStorageTypeExceededException is thrown
+ */
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeExceptionWithFileCreate() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ Path createdFile1 = new Path(foo, "created_file1.data");
+ dfs.mkdirs(foo);
+
+ dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 4);
+
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ // Create the 1st file of size 2 * BLOCKSIZE under directory "foo" and expect no exception
+ long file1Len = BLOCKSIZE * 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+ long currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, currentSSDConsumed);
+
+ // Create the 2nd file of size 1.5 * BLOCKSIZE under directory "foo" and expect no exception
+ Path createdFile2 = new Path(foo, "created_file2.data");
+ long file2Len = BLOCKSIZE + BLOCKSIZE / 2;
+ DFSTestUtil.createFile(dfs, createdFile2, bufLen, file2Len, BLOCKSIZE, REPLICATION, seed);
+ currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+
+ assertEquals(file1Len + file2Len, currentSSDConsumed);
+
+ // Create the 3rd file of size BLOCKSIZE under directory "foo" and expect quota exceeded exception
+ Path createdFile3 = new Path(foo, "created_file3.data");
+ long file3Len = BLOCKSIZE;
+
+ try {
+ DFSTestUtil.createFile(dfs, createdFile3, bufLen, file3Len, BLOCKSIZE, REPLICATION, seed);
+ fail("Should have failed with QuotaByStorageTypeExceededException ");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+
+ currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len + file2Len, currentSSDConsumed);
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeParentOffChildOff() throws Exception {
+ final Path parent = new Path(dir, "parent");
+ final Path child = new Path(parent, "child");
+ dfs.mkdirs(parent);
+ dfs.mkdirs(child);
+
+ dfs.setStoragePolicy(parent, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+ // Create file of size 2.5 * BLOCKSIZE under child directory.
+ // Since both parent and child directory do not have SSD quota set,
+ // expect succeed without exception
+ Path createdFile1 = new Path(child, "created_file1.data");
+ long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE,
+ REPLICATION, seed);
+
+ // Verify SSD usage at the root level as both parent/child don't have DirectoryWithQuotaFeature
+ INode fnode = fsdir.getINode4Write("/");
+ long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, ssdConsumed);
+
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeParentOffChildOn() throws Exception {
+ final Path parent = new Path(dir, "parent");
+ final Path child = new Path(parent, "child");
+ dfs.mkdirs(parent);
+ dfs.mkdirs(child);
+
+ dfs.setStoragePolicy(parent, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ dfs.setQuotaByStorageType(child, StorageType.SSD, 2 * BLOCKSIZE);
+
+ // Create file of size 2.5 * BLOCKSIZE under child directory
+ // Since child directory have SSD quota of 2 * BLOCKSIZE,
+ // expect an exception when creating files under child directory.
+ Path createdFile1 = new Path(child, "created_file1.data");
+ long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ int bufLen = BLOCKSIZE / 16;
+ try {
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE,
+ REPLICATION, seed);
+ fail("Should have failed with QuotaByStorageTypeExceededException ");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeParentOnChildOff() throws Exception {
+ short replication = 1;
+ final Path parent = new Path(dir, "parent");
+ final Path child = new Path(parent, "child");
+ dfs.mkdirs(parent);
+ dfs.mkdirs(child);
+
+ dfs.setStoragePolicy(parent, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ dfs.setQuotaByStorageType(parent, StorageType.SSD, 3 * BLOCKSIZE);
+
+ // Create file of size 2.5 * BLOCKSIZE under child directory
+ // Verify parent Quota applies
+ Path createdFile1 = new Path(child, "created_file1.data");
+ long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE,
+ replication, seed);
+
+ INode fnode = fsdir.getINode4Write(parent.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+ long currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ ;
+ assertEquals(file1Len, currentSSDConsumed);
+
+ // Create the 2nd file of size BLOCKSIZE under child directory and expect quota exceeded exception
+ Path createdFile2 = new Path(child, "created_file2.data");
+ long file2Len = BLOCKSIZE;
+
+ try {
+ DFSTestUtil.createFile(dfs, createdFile2, bufLen, file2Len, BLOCKSIZE, replication, seed);
+ fail("Should have failed with QuotaByStorageTypeExceededException ");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+ currentSSDConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, currentSSDConsumed);
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeParentOnChildOn() throws Exception {
+ final Path parent = new Path(dir, "parent");
+ final Path child = new Path(parent, "child");
+ dfs.mkdirs(parent);
+ dfs.mkdirs(child);
+
+ dfs.setStoragePolicy(parent, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ dfs.setQuotaByStorageType(parent, StorageType.SSD, 2 * BLOCKSIZE);
+ dfs.setQuotaByStorageType(child, StorageType.SSD, 3 * BLOCKSIZE);
+
+ // Create file of size 2.5 * BLOCKSIZE under child directory
+ // Verify parent Quota applies
+ Path createdFile1 = new Path(child, "created_file1.data");
+ long file1Len = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ int bufLen = BLOCKSIZE / 16;
+ try {
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE,
+ REPLICATION, seed);
+ fail("Should have failed with QuotaByStorageTypeExceededException ");
+ } catch (Throwable t) {
+ LOG.info("Got expected exception ", t);
+ }
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithTraditionalQuota() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ dfs.mkdirs(foo);
+ dfs.setQuota(foo, Long.MAX_VALUE - 1, REPLICATION * BLOCKSIZE * 10);
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ Path createdFile = new Path(foo, "created_file.data");
+ long fileLen = BLOCKSIZE * 2 + BLOCKSIZE / 2;
+ DFSTestUtil.createFile(dfs, createdFile, BLOCKSIZE / 16,
+ fileLen, BLOCKSIZE, REPLICATION, seed);
+
+ QuotaCounts cnt = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed();
+ assertEquals(2, cnt.getNameSpace());
+ assertEquals(fileLen * REPLICATION, cnt.getDiskSpace());
+
+ dfs.delete(createdFile, true);
+
+ QuotaCounts cntAfterDelete = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed();
+ assertEquals(1, cntAfterDelete.getNameSpace());
+ assertEquals(0, cntAfterDelete.getDiskSpace());
+
+ // Validate the computeQuotaUsage()
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ fnode.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), counts, true);
+ assertEquals(fnode.dumpTreeRecursively().toString(), 1,
+ counts.getNameSpace());
+ assertEquals(fnode.dumpTreeRecursively().toString(), 0,
+ counts.getDiskSpace());
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithSnapshot() throws Exception {
+ final Path sub1 = new Path(dir, "Sub1");
+ dfs.mkdirs(sub1);
+
+ // Setup ONE_SSD policy and SSD quota of 4 * BLOCKSIZE on sub1
+ dfs.setStoragePolicy(sub1, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+ dfs.setQuotaByStorageType(sub1, StorageType.SSD, 4 * BLOCKSIZE);
+
+ INode sub1Node = fsdir.getINode4Write(sub1.toString());
+ assertTrue(sub1Node.isDirectory());
+ assertTrue(sub1Node.isQuotaSet());
+
+ // Create file1 of size 2 * BLOCKSIZE under sub1
+ Path file1 = new Path(sub1, "file1");
+ long file1Len = 2 * BLOCKSIZE;
+ DFSTestUtil.createFile(dfs, file1, file1Len, REPLICATION, seed);
+
+ // Create snapshot on sub1 named s1
+ SnapshotTestHelper.createSnapshot(dfs, sub1, "s1");
+
+ // Verify sub1 SSD usage is unchanged after creating snapshot s1
+ long ssdConsumed = sub1Node.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, ssdConsumed);
+
+ // Delete file1
+ dfs.delete(file1, false);
+
+ // Verify sub1 SSD usage is unchanged due to the existence of snapshot s1
+ ssdConsumed = sub1Node.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, ssdConsumed);
+
+ QuotaCounts counts1 = new QuotaCounts.Builder().build();
+ sub1Node.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), counts1, true);
+ assertEquals(sub1Node.dumpTreeRecursively().toString(), file1Len,
+ counts1.getTypeSpaces().get(StorageType.SSD));
+
+ // Delete the snapshot s1
+ dfs.deleteSnapshot(sub1, "s1");
+
+ // Verify sub1 SSD usage is fully reclaimed and changed to 0
+ ssdConsumed = sub1Node.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(0, ssdConsumed);
+
+ QuotaCounts counts2 = new QuotaCounts.Builder().build();
+ sub1Node.computeQuotaUsage(fsn.getBlockManager().getStoragePolicySuite(), counts2, true);
+ assertEquals(sub1Node.dumpTreeRecursively().toString(), 0,
+ counts2.getTypeSpaces().get(StorageType.SSD));
+ }
+
+ @Test(timeout = 60000)
+ public void testQuotaByStorageTypeWithFileCreateTruncate() throws Exception {
+ final Path foo = new Path(dir, "foo");
+ Path createdFile1 = new Path(foo, "created_file1.data");
+ dfs.mkdirs(foo);
+
+ // set storage policy on directory "foo" to ONESSD
+ dfs.setStoragePolicy(foo, HdfsConstants.ONESSD_STORAGE_POLICY_NAME);
+
+ // set quota by storage type on directory "foo"
+ dfs.setQuotaByStorageType(foo, StorageType.SSD, BLOCKSIZE * 4);
+ INode fnode = fsdir.getINode4Write(foo.toString());
+ assertTrue(fnode.isDirectory());
+ assertTrue(fnode.isQuotaSet());
+
+ // Create file of size 2 * BLOCKSIZE under directory "foo"
+ long file1Len = BLOCKSIZE * 2;
+ int bufLen = BLOCKSIZE / 16;
+ DFSTestUtil.createFile(dfs, createdFile1, bufLen, file1Len, BLOCKSIZE, REPLICATION, seed);
+
+ // Verify SSD consumed before truncate
+ long ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(file1Len, ssdConsumed);
+
+ // Truncate file to 1 * BLOCKSIZE
+ int newFile1Len = BLOCKSIZE * 1;
+ dfs.truncate(createdFile1, newFile1Len);
+
+ // Verify SSD consumed after truncate
+ ssdConsumed = fnode.asDirectory().getDirectoryWithQuotaFeature()
+ .getSpaceConsumed().getTypeSpaces().get(StorageType.SSD);
+ assertEquals(newFile1Len, ssdConsumed);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index 207a4e8..9edeafd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -63,7 +63,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.INodesInPath;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.ChildrenDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiff;
import org.apache.hadoop.hdfs.util.Diff.ListType;
@@ -1198,15 +1198,15 @@ public class TestRenameWithSnapshots {
assertFalse(hdfs.exists(bar_s2));
restartClusterAndCheckImage(true);
// make sure the whole referred subtree has been destroyed
- Quota.Counts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
- assertEquals(3, q.get(Quota.NAMESPACE));
- assertEquals(0, q.get(Quota.DISKSPACE));
+ QuotaCounts q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
+ assertEquals(3, q.getNameSpace());
+ assertEquals(0, q.getDiskSpace());
hdfs.deleteSnapshot(sdir1, "s1");
restartClusterAndCheckImage(true);
- q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
- assertEquals(3, q.get(Quota.NAMESPACE));
- assertEquals(0, q.get(Quota.DISKSPACE));
+ q = fsdir.getRoot().getDirectoryWithQuotaFeature().getSpaceConsumed();
+ assertEquals(3, q.getNameSpace());
+ assertEquals(0, q.getDiskSpace());
}
/**
@@ -1600,9 +1600,9 @@ public class TestRenameWithSnapshots {
// check dir2
INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable());
- Quota.Counts counts = dir2Node.computeQuotaUsage();
- assertEquals(2, counts.get(Quota.NAMESPACE));
- assertEquals(0, counts.get(Quota.DISKSPACE));
+ QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
+ assertEquals(2, counts.getNameSpace());
+ assertEquals(0, counts.getDiskSpace());
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
@@ -1674,9 +1674,9 @@ public class TestRenameWithSnapshots {
// check dir2
INodeDirectory dir2Node = fsdir2.getINode4Write(dir2.toString()).asDirectory();
assertTrue(dir2Node.isSnapshottable());
- Quota.Counts counts = dir2Node.computeQuotaUsage();
- assertEquals(3, counts.get(Quota.NAMESPACE));
- assertEquals(0, counts.get(Quota.DISKSPACE));
+ QuotaCounts counts = dir2Node.computeQuotaUsage(fsdir.getBlockStoragePolicySuite());
+ assertEquals(3, counts.getNameSpace());
+ assertEquals(0, counts.getDiskSpace());
childrenList = ReadOnlyList.Util.asList(dir2Node.asDirectory()
.getChildrenList(Snapshot.CURRENT_STATE_ID));
assertEquals(1, childrenList.size());
@@ -1790,9 +1790,10 @@ public class TestRenameWithSnapshots {
// check dir2
INode dir2Node = fsdir.getINode4Write(dir2.toString());
assertTrue(dir2Node.asDirectory().isSnapshottable());
- Quota.Counts counts = dir2Node.computeQuotaUsage();
- assertEquals(4, counts.get(Quota.NAMESPACE));
- assertEquals(BLOCKSIZE * REPL * 2, counts.get(Quota.DISKSPACE));
+ QuotaCounts counts = dir2Node.computeQuotaUsage(
+ fsdir.getBlockStoragePolicySuite());
+ assertEquals(4, counts.getNameSpace());
+ assertEquals(BLOCKSIZE * REPL * 2, counts.getDiskSpace());
}
@Test
@@ -1958,12 +1959,12 @@ public class TestRenameWithSnapshots {
// check
final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString())
.asDirectory();
- Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
- assertEquals(3, q1.get(Quota.NAMESPACE));
+ QuotaCounts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
+ assertEquals(3, q1.getNameSpace());
final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString())
.asDirectory();
- Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
- assertEquals(1, q2.get(Quota.NAMESPACE));
+ QuotaCounts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
+ assertEquals(1, q2.getNameSpace());
final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
foo.getName());
@@ -2028,12 +2029,12 @@ public class TestRenameWithSnapshots {
final INodeDirectory dir1Node = fsdir.getINode4Write(sdir1.toString())
.asDirectory();
// sdir1 + s1 + foo_s1 (foo) + foo (foo + s1 + bar~bar3)
- Quota.Counts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
- assertEquals(7, q1.get(Quota.NAMESPACE));
+ QuotaCounts q1 = dir1Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
+ assertEquals(7, q1.getNameSpace());
final INodeDirectory dir2Node = fsdir.getINode4Write(sdir2.toString())
.asDirectory();
- Quota.Counts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
- assertEquals(1, q2.get(Quota.NAMESPACE));
+ QuotaCounts q2 = dir2Node.getDirectoryWithQuotaFeature().getSpaceConsumed();
+ assertEquals(1, q2.getNameSpace());
final Path foo_s1 = SnapshotTestHelper.getSnapshotPath(sdir1, "s1",
foo.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
index bc4ec90..c494322 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestSnapshotDeletion.java
@@ -51,7 +51,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -203,17 +203,17 @@ public class TestSnapshotDeletion {
final long expectedNs, final long expectedDs) throws IOException {
INodeDirectory dirNode = getDir(fsdir, dirPath);
assertTrue(dirNode.isQuotaSet());
- Quota.Counts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
+ QuotaCounts q = dirNode.getDirectoryWithQuotaFeature().getSpaceConsumed();
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
- q.get(Quota.NAMESPACE));
+ q.getNameSpace());
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
- q.get(Quota.DISKSPACE));
- Quota.Counts counts = Quota.Counts.newInstance();
- dirNode.computeQuotaUsage(counts, false);
+ q.getDiskSpace());
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ dirNode.computeQuotaUsage(fsdir.getBlockStoragePolicySuite(), counts, false);
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedNs,
- counts.get(Quota.NAMESPACE));
+ counts.getNameSpace());
assertEquals(dirNode.dumpTreeRecursively().toString(), expectedDs,
- counts.get(Quota.DISKSPACE));
+ counts.getDiskSpace());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored
index da8c190..a32cd98 100644
Binary files a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored and b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored differ
[35/50] [abbrv] hadoop git commit: YARN-2616 [YARN-913] Add CLI
client to the registry to list,
view and manipulate entries. (Akshay Radia via stevel)
Posted by zj...@apache.org.
YARN-2616 [YARN-913] Add CLI client to the registry to list, view and manipulate entries. (Akshay Radia via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/362565cf
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/362565cf
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/362565cf
Branch: refs/heads/YARN-2928
Commit: 362565cf5a8cbc1e7e66847649c29666d79f6938
Parents: c541a37
Author: Steve Loughran <st...@apache.org>
Authored: Wed Feb 11 14:18:46 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Feb 11 14:19:01 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/registry/cli/RegistryCli.java | 425 +++++++++++--------
.../hadoop/registry/cli/TestRegistryCli.java | 197 +++++++++
3 files changed, 439 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/362565cf/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 41927d7..14b3462 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -78,6 +78,9 @@ Release 2.7.0 - UNRELEASED
YARN-2217. [YARN-1492] Shared cache client side changes.
(Chris Trezzo via kasha)
+ YARN-2616 [YARN-913] Add CLI client to the registry to list, view
+ and manipulate entries. (Akshay Radia via stevel)
+
IMPROVEMENTS
YARN-3005. [JDK7] Use switch statement for String instead of if-else
http://git-wip-us.apache.org/repos/asf/hadoop/blob/362565cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java
index bf2b5e5..480ce0e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/main/java/org/apache/hadoop/registry/cli/RegistryCli.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.registry.cli;
import static org.apache.hadoop.registry.client.binding.RegistryTypeUtils.*;
+import java.io.Closeable;
import java.io.IOException;
import java.io.PrintStream;
import java.net.URI;
@@ -34,12 +35,14 @@ import org.apache.commons.cli.Option;
import org.apache.commons.cli.OptionBuilder;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.PathNotFoundException;
import org.apache.hadoop.security.AccessControlException;
+import org.apache.hadoop.service.ServiceOperations;
+import org.apache.hadoop.util.ExitUtil;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.registry.client.api.BindFlags;
import org.apache.hadoop.registry.client.api.RegistryOperations;
import org.apache.hadoop.registry.client.api.RegistryOperationsFactory;
@@ -54,41 +57,76 @@ import org.apache.hadoop.registry.client.types.ServiceRecord;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-public class RegistryCli extends Configured implements Tool {
+/**
+ * Command line for registry operations.
+ */
+public class RegistryCli extends Configured implements Tool, Closeable {
private static final Logger LOG =
LoggerFactory.getLogger(RegistryCli.class);
- protected final PrintStream sysout;
- protected final PrintStream syserr;
+ protected final PrintStream sysout;
+ protected final PrintStream syserr;
- private RegistryOperations registry;
+ private RegistryOperations registry;
- static final String LS_USAGE = "ls pathName";
- static final String RESOLVE_USAGE = "resolve pathName";
- static final String BIND_USAGE =
+ private static final String LS_USAGE = "ls pathName";
+ private static final String RESOLVE_USAGE = "resolve pathName";
+ private static final String BIND_USAGE =
"bind -inet -api apiName -p portNumber -h hostName pathName" + "\n"
+ "bind -webui uriString -api apiName pathName" + "\n"
+ "bind -rest uriString -api apiName pathName";
- static final String MKNODE_USAGE = "mknode directoryName";
- static final String RM_USAGE = "rm pathName";
- static final String USAGE =
+ private static final String MKNODE_USAGE = "mknode directoryName";
+ private static final String RM_USAGE = "rm pathName";
+ private static final String USAGE =
"\n" + LS_USAGE + "\n" + RESOLVE_USAGE + "\n" + BIND_USAGE + "\n" +
MKNODE_USAGE + "\n" + RM_USAGE;
-
- public RegistryCli(PrintStream sysout, PrintStream syserr) {
- super(new YarnConfiguration());
+ public RegistryCli(PrintStream sysout, PrintStream syserr) {
+ Configuration conf = new Configuration();
+ super.setConf(conf);
+ registry = RegistryOperationsFactory.createInstance(conf);
+ registry.start();
this.sysout = sysout;
this.syserr = syserr;
- }
+ }
+ public RegistryCli(RegistryOperations reg,
+ Configuration conf,
+ PrintStream sysout,
+ PrintStream syserr) {
+ super(conf);
+ Preconditions.checkArgument(reg != null, "Null registry");
+ registry = reg;
+ this.sysout = sysout;
+ this.syserr = syserr;
+ }
@SuppressWarnings("UseOfSystemOutOrSystemErr")
public static void main(String[] args) throws Exception {
- RegistryCli cli = new RegistryCli(System.out, System.err);
- int res = ToolRunner.run(cli, args);
- System.exit(res);
+ int res = -1;
+ try (RegistryCli cli = new RegistryCli(System.out, System.err)) {
+ res = ToolRunner.run(cli, args);
+ } catch (Exception e) {
+ ExitUtil.terminate(res, e);
+ }
+ ExitUtil.terminate(res);
+ }
+
+ /**
+ * Close the object by stopping the registry.
+ * <p>
+ * <i>Important:</i>
+ * <p>
+ * After this call is made, no operations may be made of this
+ * object, <i>or of a YARN registry instance used when constructing
+ * this object. </i>
+ * @throws IOException
+ */
+ @Override
+ public void close() throws IOException {
+ ServiceOperations.stopQuietly(registry);
+ registry = null;
}
private int usageError(String err, String usage) {
@@ -104,146 +142,151 @@ public class RegistryCli extends Configured implements Tool {
}
return true;
}
+
@Override
public int run(String[] args) throws Exception {
Preconditions.checkArgument(getConf() != null, "null configuration");
- registry = RegistryOperationsFactory.createInstance(
- new YarnConfiguration(getConf()));
- registry.start();
if (args.length > 0) {
- if (args[0].equals("ls")) {
- return ls(args);
- } else if (args[0].equals("resolve")) {
- return resolve(args);
- } else if (args[0].equals("bind")) {
- return bind(args);
- } else if (args[0].equals("mknode")) {
- return mknode(args);
- } else if (args[0].equals("rm")) {
- return rm(args);
+ switch (args[0]) {
+ case "ls":
+ return ls(args);
+ case "resolve":
+ return resolve(args);
+ case "bind":
+ return bind(args);
+ case "mknode":
+ return mknode(args);
+ case "rm":
+ return rm(args);
+ default:
+ return usageError("Invalid command: " + args[0], USAGE);
}
}
- return usageError("Invalid command: " + args[0], USAGE);
+ return usageError("No command arg passed.", USAGE);
}
@SuppressWarnings("unchecked")
- public int ls(String [] args) {
-
- Options lsOption = new Options();
- CommandLineParser parser = new GnuParser();
- try {
- CommandLine line = parser.parse(lsOption, args);
-
- List<String> argsList = line.getArgList();
- if (argsList.size() != 2) {
- return usageError("ls requires exactly one path argument", LS_USAGE);
- }
- if (!validatePath(argsList.get(1)))
- return -1;
-
- try {
- List<String> children = registry.list(argsList.get(1));
+ public int ls(String[] args) {
+
+ Options lsOption = new Options();
+ CommandLineParser parser = new GnuParser();
+ try {
+ CommandLine line = parser.parse(lsOption, args);
+
+ List<String> argsList = line.getArgList();
+ if (argsList.size() != 2) {
+ return usageError("ls requires exactly one path argument", LS_USAGE);
+ }
+ if (!validatePath(argsList.get(1))) {
+ return -1;
+ }
+
+ try {
+ List<String> children = registry.list(argsList.get(1));
for (String child : children) {
sysout.println(child);
}
- return 0;
+ return 0;
} catch (Exception e) {
syserr.println(analyzeException("ls", e, argsList));
}
- return -1;
- } catch (ParseException exp) {
- return usageError("Invalid syntax " + exp, LS_USAGE);
- }
- }
+ return -1;
+ } catch (ParseException exp) {
+ return usageError("Invalid syntax " + exp, LS_USAGE);
+ }
+ }
@SuppressWarnings("unchecked")
- public int resolve(String [] args) {
- Options resolveOption = new Options();
- CommandLineParser parser = new GnuParser();
- try {
- CommandLine line = parser.parse(resolveOption, args);
-
- List<String> argsList = line.getArgList();
- if (argsList.size() != 2) {
- return usageError("resolve requires exactly one path argument", RESOLVE_USAGE);
- }
- if (!validatePath(argsList.get(1)))
- return -1;
-
- try {
- ServiceRecord record = registry.resolve(argsList.get(1));
-
- for (Endpoint endpoint : record.external) {
- sysout.println(" Endpoint(ProtocolType="
- + endpoint.protocolType + ", Api="
- + endpoint.api + ");"
- + " Addresses(AddressType="
- + endpoint.addressType + ") are: ");
+ public int resolve(String[] args) {
+ Options resolveOption = new Options();
+ CommandLineParser parser = new GnuParser();
+ try {
+ CommandLine line = parser.parse(resolveOption, args);
+
+ List<String> argsList = line.getArgList();
+ if (argsList.size() != 2) {
+ return usageError("resolve requires exactly one path argument",
+ RESOLVE_USAGE);
+ }
+ if (!validatePath(argsList.get(1))) {
+ return -1;
+ }
+
+ try {
+ ServiceRecord record = registry.resolve(argsList.get(1));
+
+ for (Endpoint endpoint : record.external) {
+ sysout.println(" Endpoint(ProtocolType="
+ + endpoint.protocolType + ", Api="
+ + endpoint.api + ");"
+ + " Addresses(AddressType="
+ + endpoint.addressType + ") are: ");
for (Map<String, String> address : endpoint.addresses) {
- sysout.println(" [ ");
+ sysout.println("[ ");
for (Map.Entry<String, String> entry : address.entrySet()) {
- sysout.println(" " + entry.getKey()
- + ": \"" + entry.getValue() + "\"");
+ sysout.print("\t" + entry.getKey()
+ + ":" + entry.getValue());
}
- sysout.println(" ]");
+
+ sysout.println("\n]");
}
sysout.println();
}
- return 0;
+ return 0;
} catch (Exception e) {
syserr.println(analyzeException("resolve", e, argsList));
}
- return -1;
- } catch (org.apache.commons.cli.ParseException exp) {
- return usageError("Invalid syntax " + exp, RESOLVE_USAGE);
- }
-
- }
-
- public int bind(String [] args) {
- Option rest = OptionBuilder.withArgName("rest")
- .hasArg()
- .withDescription("rest Option")
- .create("rest");
- Option webui = OptionBuilder.withArgName("webui")
- .hasArg()
- .withDescription("webui Option")
- .create("webui");
- Option inet = OptionBuilder.withArgName("inet")
- .withDescription("inet Option")
- .create("inet");
- Option port = OptionBuilder.withArgName("port")
- .hasArg()
- .withDescription("port to listen on [9999]")
- .create("p");
- Option host = OptionBuilder.withArgName("host")
- .hasArg()
- .withDescription("host name")
- .create("h");
- Option apiOpt = OptionBuilder.withArgName("api")
- .hasArg()
- .withDescription("api")
- .create("api");
- Options inetOption = new Options();
- inetOption.addOption(inet);
- inetOption.addOption(port);
- inetOption.addOption(host);
- inetOption.addOption(apiOpt);
-
- Options webuiOpt = new Options();
- webuiOpt.addOption(webui);
- webuiOpt.addOption(apiOpt);
-
- Options restOpt = new Options();
- restOpt.addOption(rest);
- restOpt.addOption(apiOpt);
+ return -1;
+ } catch (ParseException exp) {
+ return usageError("Invalid syntax " + exp, RESOLVE_USAGE);
+ }
+
+ }
+
+ public int bind(String[] args) {
+ Option rest = OptionBuilder.withArgName("rest")
+ .hasArg()
+ .withDescription("rest Option")
+ .create("rest");
+ Option webui = OptionBuilder.withArgName("webui")
+ .hasArg()
+ .withDescription("webui Option")
+ .create("webui");
+ Option inet = OptionBuilder.withArgName("inet")
+ .withDescription("inet Option")
+ .create("inet");
+ Option port = OptionBuilder.withArgName("port")
+ .hasArg()
+ .withDescription("port to listen on [9999]")
+ .create("p");
+ Option host = OptionBuilder.withArgName("host")
+ .hasArg()
+ .withDescription("host name")
+ .create("h");
+ Option apiOpt = OptionBuilder.withArgName("api")
+ .hasArg()
+ .withDescription("api")
+ .create("api");
+ Options inetOption = new Options();
+ inetOption.addOption(inet);
+ inetOption.addOption(port);
+ inetOption.addOption(host);
+ inetOption.addOption(apiOpt);
+
+ Options webuiOpt = new Options();
+ webuiOpt.addOption(webui);
+ webuiOpt.addOption(apiOpt);
+
+ Options restOpt = new Options();
+ restOpt.addOption(rest);
+ restOpt.addOption(apiOpt);
CommandLineParser parser = new GnuParser();
ServiceRecord sr = new ServiceRecord();
- CommandLine line = null;
+ CommandLine line;
if (args.length <= 1) {
return usageError("Invalid syntax ", BIND_USAGE);
}
@@ -259,7 +302,12 @@ public class RegistryCli extends Configured implements Tool {
}
if (line.hasOption("inet") && line.hasOption("p") &&
line.hasOption("h") && line.hasOption("api")) {
- portNum = Integer.parseInt(line.getOptionValue("p"));
+ try {
+ portNum = Integer.parseInt(line.getOptionValue("p"));
+ } catch (NumberFormatException exp) {
+ return usageError("Invalid Port - int required" + exp.getMessage(),
+ BIND_USAGE);
+ }
hostName = line.getOptionValue("h");
api = line.getOptionValue("api");
sr.addExternalEndpoint(
@@ -278,7 +326,7 @@ public class RegistryCli extends Configured implements Tool {
return usageError("Invalid syntax " + exp.getMessage(), BIND_USAGE);
}
if (line.hasOption("webui") && line.hasOption("api")) {
- URI theUri = null;
+ URI theUri;
try {
theUri = new URI(line.getOptionValue("webui"));
} catch (URISyntaxException e) {
@@ -315,86 +363,91 @@ public class RegistryCli extends Configured implements Tool {
return usageError("Invalid syntax", BIND_USAGE);
}
@SuppressWarnings("unchecked")
- List<String> argsList = line.getArgList();
- if (argsList.size() != 2) {
- return usageError("bind requires exactly one path argument", BIND_USAGE);
- }
- if (!validatePath(argsList.get(1)))
- return -1;
-
- try {
- registry.bind(argsList.get(1), sr, BindFlags.OVERWRITE);
- return 0;
+ List<String> argsList = line.getArgList();
+ if (argsList.size() != 2) {
+ return usageError("bind requires exactly one path argument", BIND_USAGE);
+ }
+ if (!validatePath(argsList.get(1))) {
+ return -1;
+ }
+
+ try {
+ registry.bind(argsList.get(1), sr, BindFlags.OVERWRITE);
+ return 0;
} catch (Exception e) {
syserr.println(analyzeException("bind", e, argsList));
}
return -1;
- }
+ }
@SuppressWarnings("unchecked")
- public int mknode(String [] args) {
- Options mknodeOption = new Options();
- CommandLineParser parser = new GnuParser();
- try {
- CommandLine line = parser.parse(mknodeOption, args);
-
- List<String> argsList = line.getArgList();
- if (argsList.size() != 2) {
- return usageError("mknode requires exactly one path argument", MKNODE_USAGE);
- }
- if (!validatePath(argsList.get(1)))
- return -1;
-
- try {
- registry.mknode(args[1], false);
- return 0;
- } catch (Exception e) {
+ public int mknode(String[] args) {
+ Options mknodeOption = new Options();
+ CommandLineParser parser = new GnuParser();
+ try {
+ CommandLine line = parser.parse(mknodeOption, args);
+
+ List<String> argsList = line.getArgList();
+ if (argsList.size() != 2) {
+ return usageError("mknode requires exactly one path argument",
+ MKNODE_USAGE);
+ }
+ if (!validatePath(argsList.get(1))) {
+ return -1;
+ }
+
+ try {
+ registry.mknode(args[1], false);
+ return 0;
+ } catch (Exception e) {
syserr.println(analyzeException("mknode", e, argsList));
- }
- return -1;
- } catch (ParseException exp) {
- return usageError("Invalid syntax " + exp.toString(), MKNODE_USAGE);
- }
- }
+ }
+ return -1;
+ } catch (ParseException exp) {
+ return usageError("Invalid syntax " + exp.toString(), MKNODE_USAGE);
+ }
+ }
@SuppressWarnings("unchecked")
public int rm(String[] args) {
- Option recursive = OptionBuilder.withArgName("recursive")
- .withDescription("delete recursively").create("r");
+ Option recursive = OptionBuilder.withArgName("recursive")
+ .withDescription("delete recursively")
+ .create("r");
- Options rmOption = new Options();
- rmOption.addOption(recursive);
+ Options rmOption = new Options();
+ rmOption.addOption(recursive);
- boolean recursiveOpt = false;
+ boolean recursiveOpt = false;
- CommandLineParser parser = new GnuParser();
- try {
- CommandLine line = parser.parse(rmOption, args);
+ CommandLineParser parser = new GnuParser();
+ try {
+ CommandLine line = parser.parse(rmOption, args);
- List<String> argsList = line.getArgList();
- if (argsList.size() != 2) {
- return usageError("RM requires exactly one path argument", RM_USAGE);
- }
- if (!validatePath(argsList.get(1)))
- return -1;
+ List<String> argsList = line.getArgList();
+ if (argsList.size() != 2) {
+ return usageError("RM requires exactly one path argument", RM_USAGE);
+ }
+ if (!validatePath(argsList.get(1))) {
+ return -1;
+ }
- try {
- if (line.hasOption("r")) {
- recursiveOpt = true;
- }
+ try {
+ if (line.hasOption("r")) {
+ recursiveOpt = true;
+ }
- registry.delete(argsList.get(1), recursiveOpt);
- return 0;
+ registry.delete(argsList.get(1), recursiveOpt);
+ return 0;
} catch (Exception e) {
syserr.println(analyzeException("rm", e, argsList));
}
return -1;
- } catch (ParseException exp) {
- return usageError("Invalid syntax " + exp.toString(), RM_USAGE);
- }
- }
+ } catch (ParseException exp) {
+ return usageError("Invalid syntax " + exp.toString(), RM_USAGE);
+ }
+ }
/**
* Given an exception and a possibly empty argument list, generate
http://git-wip-us.apache.org/repos/asf/hadoop/blob/362565cf/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
new file mode 100644
index 0000000..bd8a38d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-registry/src/test/java/org/apache/hadoop/registry/cli/TestRegistryCli.java
@@ -0,0 +1,197 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.registry.cli;
+
+import java.io.ByteArrayOutputStream;
+import java.io.PrintStream;
+
+import org.apache.hadoop.registry.AbstractRegistryTest;
+import org.apache.hadoop.registry.operations.TestRegistryOperations;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class TestRegistryCli extends AbstractRegistryTest {
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestRegistryOperations.class);
+
+ private ByteArrayOutputStream sysOutStream;
+ private PrintStream sysOut;
+ private ByteArrayOutputStream sysErrStream;
+ private PrintStream sysErr;
+ private RegistryCli cli;
+
+ @Before
+ public void setUp() throws Exception {
+ sysOutStream = new ByteArrayOutputStream();
+ sysOut = new PrintStream(sysOutStream);
+ sysErrStream = new ByteArrayOutputStream();
+ sysErr = new PrintStream(sysErrStream);
+ System.setOut(sysOut);
+ cli = new RegistryCli(operations, createRegistryConfiguration(), sysOut, sysErr);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ cli.close();
+ }
+
+ private void assertResult(RegistryCli cli, int code, String...args) throws Exception {
+ int result = cli.run(args);
+ assertEquals(code, result);
+ }
+
+ @Test
+ public void testBadCommands() throws Exception {
+ assertResult(cli, -1, new String[] { });
+ assertResult(cli, -1, "foo");
+ }
+
+ @Test
+ public void testInvalidNumArgs() throws Exception {
+ assertResult(cli, -1, "ls");
+ assertResult(cli, -1, "ls", "/path", "/extraPath");
+ assertResult(cli, -1, "resolve");
+ assertResult(cli, -1, "resolve", "/path", "/extraPath");
+ assertResult(cli, -1, "mknode");
+ assertResult(cli, -1, "mknode", "/path", "/extraPath");
+ assertResult(cli, -1, "rm");
+ assertResult(cli, -1, "rm", "/path", "/extraPath");
+ assertResult(cli, -1, "bind");
+ assertResult(cli, -1, "bind", "foo");
+ assertResult(cli, -1, "bind", "-inet", "foo");
+ assertResult(cli, -1, "bind", "-inet", "-api", "-p", "378", "-h", "host", "/foo");
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "-h", "host", "/foo");
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "/foo");
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host");
+ assertResult(cli, -1, "bind", "-api", "Api", "-p", "378", "-h", "host", "/foo");
+ assertResult(cli, -1, "bind", "-webui", "foo");
+ assertResult(cli, -1, "bind", "-webui", "-api", "Api", "/foo");
+ assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "/foo");
+ assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api");
+ assertResult(cli, -1, "bind", "-rest", "foo");
+ assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api");
+ assertResult(cli, -1, "bind", "-rest", "-api", "Api", "/foo");
+ assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "/foo");
+ assertResult(cli, -1, "bind", "uriString", "-api", "Api", "/foo");
+ }
+
+ @Test
+ public void testBadArgType() throws Exception {
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "fooPort", "-h",
+ "host", "/dir");
+ }
+
+ @Test
+ public void testBadPath() throws Exception {
+ assertResult(cli, -1, "ls", "NonSlashPath");
+ assertResult(cli, -1, "ls", "//");
+ assertResult(cli, -1, "resolve", "NonSlashPath");
+ assertResult(cli, -1, "resolve", "//");
+ assertResult(cli, -1, "mknode", "NonSlashPath");
+ assertResult(cli, -1, "mknode", "//");
+ assertResult(cli, -1, "rm", "NonSlashPath");
+ assertResult(cli, -1, "rm", "//");
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "NonSlashPath");
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "//");
+ assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api", "NonSlashPath");
+ assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api", "//");
+ assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api", "NonSlashPath");
+ assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api", "//");
+ }
+
+ @Test
+ public void testNotExistingPaths() throws Exception {
+ assertResult(cli, -1, "ls", "/nonexisting_path");
+ assertResult(cli, -1, "ls", "/NonExistingDir/nonexisting_path");
+ assertResult(cli, -1, "resolve", "/nonexisting_path");
+ assertResult(cli, -1, "resolve", "/NonExistingDir/nonexisting_path");
+ assertResult(cli, -1, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/NonExistingDir/nonexisting_path");
+ assertResult(cli, -1, "bind", "-webui", "uriString", "-api", "Api", "/NonExistingDir/nonexisting_path");
+ assertResult(cli, -1, "bind", "-rest", "uriString", "-api", "Api", "/NonExistingDir/nonexisting_path");
+ }
+
+ @Test
+ public void testValidCommands() throws Exception {
+ assertResult(cli, 0, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/foo");
+ assertResult(cli, 0, "resolve", "/foo");
+ assertResult(cli, 0, "rm", "/foo");
+ assertResult(cli, -1, "resolve", "/foo");
+
+ assertResult(cli, 0, "bind", "-webui", "uriString", "-api", "Api", "/foo");
+ assertResult(cli, 0, "resolve", "/foo");
+ assertResult(cli, 0, "rm", "/foo");
+ assertResult(cli, -1, "resolve", "/foo");
+
+ assertResult(cli, 0, "bind", "-rest", "uriString", "-api", "Api", "/foo");
+ assertResult(cli, 0, "resolve", "/foo");
+ assertResult(cli, 0, "rm", "/foo");
+ assertResult(cli, -1, "resolve", "/foo");
+
+ //Test Sub Directories Binds
+ assertResult(cli, 0, "mknode", "/subdir");
+ assertResult(cli, -1, "resolve", "/subdir");
+
+ assertResult(cli, 0, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/subdir/foo");
+ assertResult(cli, 0, "resolve", "/subdir/foo");
+ assertResult(cli, 0, "rm", "/subdir/foo");
+ assertResult(cli, -1, "resolve", "/subdir/foo");
+
+ assertResult(cli, 0, "bind", "-webui", "uriString", "-api", "Api", "/subdir/foo");
+ assertResult(cli, 0, "resolve", "/subdir/foo");
+ assertResult(cli, 0, "rm", "/subdir/foo");
+ assertResult(cli, -1, "resolve", "/subdir/foo");
+
+ assertResult(cli, 0, "bind", "-rest", "uriString", "-api", "Api", "/subdir/foo");
+ assertResult(cli, 0, "resolve", "/subdir/foo");
+ assertResult(cli, 0, "rm", "/subdir/foo");
+ assertResult(cli, -1, "resolve", "/subdir/foo");
+
+ assertResult(cli, 0, "rm", "/subdir");
+ assertResult(cli, -1, "resolve", "/subdir");
+
+ //Test Bind that the dir itself
+ assertResult(cli, 0, "mknode", "/dir");
+ assertResult(cli, -1, "resolve", "/dir");
+
+ assertResult(cli, 0, "bind", "-inet", "-api", "Api", "-p", "378", "-h", "host", "/dir");
+ assertResult(cli, 0, "resolve", "/dir");
+ assertResult(cli, 0, "rm", "/dir");
+ assertResult(cli, -1, "resolve", "/dir");
+
+ assertResult(cli, 0, "mknode", "/dir");
+ assertResult(cli, -1, "resolve", "/dir");
+
+ assertResult(cli, 0, "bind", "-webui", "uriString", "-api", "Api", "/dir");
+ assertResult(cli, 0, "resolve", "/dir");
+ assertResult(cli, 0, "rm", "/dir");
+ assertResult(cli, -1, "resolve", "/dir");
+
+ assertResult(cli, 0, "mknode", "/dir");
+ assertResult(cli, -1, "resolve", "/dir");
+
+ assertResult(cli, 0, "bind", "-rest", "uriString", "-api", "Api", "/dir");
+ assertResult(cli, 0, "resolve", "/dir");
+ assertResult(cli, 0, "rm", "/dir");
+ assertResult(cli, -1, "resolve", "/dir");
+
+ assertResult(cli, 0, "rm", "/Nonexitent");
+ }
+}
[04/50] [abbrv] hadoop git commit: HADOOP-8934. Shell command ls
should include sort options (Jonathan Allen via aw)
Posted by zj...@apache.org.
HADOOP-8934. Shell command ls should include sort options (Jonathan Allen via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/30b797ee
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/30b797ee
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/30b797ee
Branch: refs/heads/YARN-2928
Commit: 30b797ee9df30260314eeadffc7d51492871b352
Parents: ab934e8
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 12:50:44 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 12:50:44 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/shell/Ls.java | 187 ++++++++++++++++---
.../src/site/apt/FileSystemShell.apt.vm | 22 ++-
.../src/test/resources/testConf.xml | 34 +++-
.../src/test/resources/testHDFSConf.xml | 148 +++++++++++++++
5 files changed, 361 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b797ee/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index b02e695..55baf8a 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -24,6 +24,9 @@ Trunk (Unreleased)
HADOOP-11485. Pluggable shell integration (aw)
+ HADOOP-8934. Shell command ls should include sort options (Jonathan Allen
+ via aw)
+
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b797ee/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index c7e80b6..0e46700 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.fs.shell;
import java.io.IOException;
import java.text.SimpleDateFormat;
+import java.util.Arrays;
+import java.util.Comparator;
import java.util.Date;
import java.util.LinkedList;
import org.apache.hadoop.util.StringUtils;
@@ -40,29 +42,59 @@ class Ls extends FsCommand {
factory.addClass(Ls.class, "-ls");
factory.addClass(Lsr.class, "-lsr");
}
-
+
+ private static final String OPTION_DIRECTORY = "d";
+ private static final String OPTION_HUMAN = "h";
+ private static final String OPTION_RECURSIVE = "R";
+ private static final String OPTION_REVERSE = "r";
+ private static final String OPTION_MTIME = "t";
+ private static final String OPTION_ATIME = "u";
+ private static final String OPTION_SIZE = "S";
+
public static final String NAME = "ls";
- public static final String USAGE = "[-d] [-h] [-R] [<path> ...]";
+ public static final String USAGE = "[-" + OPTION_DIRECTORY + "] [-"
+ + OPTION_HUMAN + "] " + "[-" + OPTION_RECURSIVE + "] [-" + OPTION_MTIME
+ + "] [-" + OPTION_SIZE + "] [-" + OPTION_REVERSE + "] " + "[-"
+ + OPTION_ATIME + "] [<path> ...]";
+
public static final String DESCRIPTION =
- "List the contents that match the specified file pattern. If " +
- "path is not specified, the contents of /user/<currentUser> " +
- "will be listed. Directory entries are of the form:\n" +
- "\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
- "and file entries are of the form:\n" +
- "\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n" +
- "-d: Directories are listed as plain files.\n" +
- "-h: Formats the sizes of files in a human-readable fashion " +
- "rather than a number of bytes.\n" +
- "-R: Recursively list the contents of directories.";
-
-
-
- protected final SimpleDateFormat dateFormat =
+ "List the contents that match the specified file pattern. If " +
+ "path is not specified, the contents of /user/<currentUser> " +
+ "will be listed. For a directory a list of its direct children " +
+ "is returned (unless -" + OPTION_DIRECTORY +
+ " option is specified).\n\n" +
+ "Directory entries are of the form:\n" +
+ "\tpermissions - userId groupId sizeOfDirectory(in bytes) modificationDate(yyyy-MM-dd HH:mm) directoryName\n\n" +
+ "and file entries are of the form:\n" +
+ "\tpermissions numberOfReplicas userId groupId sizeOfFile(in bytes) modificationDate(yyyy-MM-dd HH:mm) fileName\n\n" +
+ " -" + OPTION_DIRECTORY +
+ " Directories are listed as plain files.\n" +
+ " -" + OPTION_HUMAN +
+ " Formats the sizes of files in a human-readable fashion\n" +
+ " rather than a number of bytes.\n" +
+ " -" + OPTION_RECURSIVE +
+ " Recursively list the contents of directories.\n" +
+ " -" + OPTION_MTIME +
+ " Sort files by modification time (most recent first).\n" +
+ " -" + OPTION_SIZE +
+ " Sort files by size.\n" +
+ " -" + OPTION_REVERSE +
+ " Reverse the order of the sort.\n" +
+ " -" + OPTION_ATIME +
+ " Use time of last access instead of modification for\n" +
+ " display and sorting.";
+
+ protected static final SimpleDateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
protected String lineFormat;
protected boolean dirRecurse;
+ private boolean orderReverse;
+ private boolean orderTime;
+ private boolean orderSize;
+ private boolean useAtime;
+ private Comparator<PathData> orderComparator;
protected boolean humanReadable = false;
@@ -75,12 +107,74 @@ class Ls extends FsCommand {
@Override
protected void processOptions(LinkedList<String> args)
throws IOException {
- CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE, "d", "h", "R");
+ CommandFormat cf = new CommandFormat(0, Integer.MAX_VALUE,
+ OPTION_DIRECTORY, OPTION_HUMAN, OPTION_RECURSIVE, OPTION_REVERSE,
+ OPTION_MTIME, OPTION_SIZE, OPTION_ATIME);
cf.parse(args);
- dirRecurse = !cf.getOpt("d");
- setRecursive(cf.getOpt("R") && dirRecurse);
- humanReadable = cf.getOpt("h");
+ dirRecurse = !cf.getOpt(OPTION_DIRECTORY);
+ setRecursive(cf.getOpt(OPTION_RECURSIVE) && dirRecurse);
+ humanReadable = cf.getOpt(OPTION_HUMAN);
+ orderReverse = cf.getOpt(OPTION_REVERSE);
+ orderTime = cf.getOpt(OPTION_MTIME);
+ orderSize = !orderTime && cf.getOpt(OPTION_SIZE);
+ useAtime = cf.getOpt(OPTION_ATIME);
if (args.isEmpty()) args.add(Path.CUR_DIR);
+
+ initialiseOrderComparator();
+ }
+
+ /**
+ * Should the contents of the directory be shown or just the directory?
+ * @return true if directory contents, false if just directory
+ */
+ @InterfaceAudience.Private
+ boolean isDirRecurse() {
+ return this.dirRecurse;
+ }
+
+ /**
+ * Should file sizes be returned in human readable format rather than bytes?
+ * @return true is human readable, false if bytes
+ */
+ @InterfaceAudience.Private
+ boolean isHumanReadable() {
+ return this.humanReadable;
+ }
+
+ /**
+ * Should directory contents be displayed in reverse order
+ * @return true reverse order, false default order
+ */
+ @InterfaceAudience.Private
+ boolean isOrderReverse() {
+ return this.orderReverse;
+ }
+
+ /**
+ * Should directory contents be displayed in mtime order.
+ * @return true mtime order, false default order
+ */
+ @InterfaceAudience.Private
+ boolean isOrderTime() {
+ return this.orderTime;
+ }
+
+ /**
+ * Should directory contents be displayed in size order.
+ * @return true size order, false default order
+ */
+ @InterfaceAudience.Private
+ boolean isOrderSize() {
+ return this.orderSize;
+ }
+
+ /**
+ * Should access time be used rather than modification time.
+ * @return true use access time, false use modification time
+ */
+ @InterfaceAudience.Private
+ boolean isUseAtime() {
+ return this.useAtime;
}
@Override
@@ -98,6 +192,7 @@ class Ls extends FsCommand {
throws IOException {
if (parent != null && !isRecursive() && items.length != 0) {
out.println("Found " + items.length + " items");
+ Arrays.sort(items, getOrderComparator());
}
adjustColumnWidths(items);
super.processPaths(parent, items);
@@ -113,9 +208,10 @@ class Ls extends FsCommand {
stat.getOwner(),
stat.getGroup(),
formatSize(stat.getLen()),
- dateFormat.format(new Date(stat.getModificationTime())),
- item
- );
+ dateFormat.format(new Date(isUseAtime()
+ ? stat.getAccessTime()
+ : stat.getModificationTime())),
+ item);
out.println(line);
}
@@ -150,6 +246,49 @@ class Ls extends FsCommand {
}
/**
+ * Get the comparator to be used for sorting files.
+ * @return comparator
+ */
+ private Comparator<PathData> getOrderComparator() {
+ return this.orderComparator;
+ }
+
+ /**
+ * Initialise the comparator to be used for sorting files. If multiple options
+ * are selected then the order is chosen in the following precedence: -
+ * Modification time (or access time if requested) - File size - File name
+ */
+ private void initialiseOrderComparator() {
+ if (isOrderTime()) {
+ // mtime is ordered latest date first in line with the unix ls -t command
+ this.orderComparator = new Comparator<PathData>() {
+ public int compare(PathData o1, PathData o2) {
+ Long o1Time = (isUseAtime() ? o1.stat.getAccessTime()
+ : o1.stat.getModificationTime());
+ Long o2Time = (isUseAtime() ? o2.stat.getAccessTime()
+ : o2.stat.getModificationTime());
+ return o2Time.compareTo(o1Time) * (isOrderReverse() ? -1 : 1);
+ }
+ };
+ } else if (isOrderSize()) {
+ // size is ordered largest first in line with the unix ls -S command
+ this.orderComparator = new Comparator<PathData>() {
+ public int compare(PathData o1, PathData o2) {
+ Long o1Length = o1.stat.getLen();
+ Long o2Length = o2.stat.getLen();
+ return o2Length.compareTo(o1Length) * (isOrderReverse() ? -1 : 1);
+ }
+ };
+ } else {
+ this.orderComparator = new Comparator<PathData>() {
+ public int compare(PathData o1, PathData o2) {
+ return o1.compareTo(o2) * (isOrderReverse() ? -1 : 1);
+ }
+ };
+ }
+ }
+
+ /**
* Get a recursive listing of all files in that match the file patterns.
* Same as "-ls -R"
*/
@@ -162,7 +301,7 @@ class Ls extends FsCommand {
args.addFirst("-R");
super.processOptions(args);
}
-
+
@Override
public String getReplacementCommand() {
return "ls -R";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b797ee/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
index 3fd56fc..6831ebf 100644
--- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
+++ b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
@@ -406,14 +406,25 @@ bin/hadoop fs <args>
* ls
- Usage: <<<hadoop fs -ls [-R] <args> >>>
+ Usage: <<<hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> >>>
Options:
- * The -R option will return stat recursively through the directory
- structure.
+ * -d: Directories are listed as plain files.
- For a file returns stat on the file with the following format:
+ * -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
+
+ * -R: Recursively list subdirectories encountered.
+
+ * -t: Sort output by modification time (most recent first).
+
+ * -S: Sort output by file size.
+
+ * -r: Reverse the sort order.
+
+ * -u: Use access time rather than modification time for display and sorting.
+
+ For a file ls returns stat on the file with the following format:
+---+
permissions number_of_replicas userid groupid filesize modification_date modification_time filename
@@ -425,6 +436,9 @@ permissions number_of_replicas userid groupid filesize modification_date modific
permissions userid groupid modification_date modification_time dirname
+---+
+ Files within a directory are order by filename by default.
+
+
Example:
* <<<hadoop fs -ls /user/hadoop/file1>>>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b797ee/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
index f962813..ac28192 100644
--- a/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
+++ b/hadoop-common-project/hadoop-common/src/test/resources/testConf.xml
@@ -54,7 +54,7 @@
<comparators>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[<path> \.\.\.\] :( |\t)*</expected-output>
+ <expected-output>^-ls \[-d\] \[-h\] \[-R\] \[-t\] \[-S\] \[-r\] \[-u\] \[<path> \.\.\.\] :( |\t)*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -62,11 +62,15 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*specified, the contents of /user/<currentUser> will be listed. Directory entries( )*</expected-output>
+ <expected-output>^\s*specified, the contents of /user/<currentUser> will be listed. For a directory a( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*are of the form:( )*</expected-output>
+ <expected-output>^( |\t)*list of its direct children is returned \(unless -d option is specified\).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*Directory entries are of the form:( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
@@ -94,16 +98,36 @@
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*-h\s+Formats the sizes of files in a human-readable fashion rather than a number( )*</expected-output>
+ <expected-output>^\s*-h\s+Formats the sizes of files in a human-readable fashion( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
- <expected-output>^\s*of bytes\.( )*</expected-output>
+ <expected-output>^\s*rather than a number of bytes\.( )*</expected-output>
</comparator>
<comparator>
<type>RegexpComparator</type>
<expected-output>^\s*-R\s+Recursively list the contents of directories\.( )*</expected-output>
</comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*-t\s+Sort files by modification time \(most recent first\)\.</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*-S\s+Sort files by size\.</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*-r\s+Reverse the order of the sort\.</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*-u\s+Use time of last access instead of modification for</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^( |\t)*display and sorting\.</expected-output>
+ </comparator>
</comparators>
</test>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/30b797ee/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 6c71b6e..8b221d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -1047,6 +1047,154 @@
</test>
<test> <!-- TESTED -->
+ <description>ls: files sorted in default order</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir</command>
+ <command>-fs NAMENODE -touchz /dir/file2 /dir/file1 /dir/file3</command>
+ <command>-fs NAMENODE -ls /dir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Found 3 items
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/file1
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/file2
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/file3</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>ls: files sorted in reverse default order</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir</command>
+ <command>-fs NAMENODE -touchz /dir/file2 /dir/file1 /dir/file3</command>
+ <command>-fs NAMENODE -ls -r /dir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Found 3 items
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/file3
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/file2
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/file1</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>ls: files sorted in size order</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data30bytes /dir/data30bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dir/data15bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /dir/data120bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /dir/data60bytes</command>
+ <command>-fs NAMENODE -ls -S /dir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Found 4 items
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*120( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data120bytes
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*60( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data60bytes
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*30( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data30bytes
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*15( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data15bytes</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>ls: files sorted in reverse size order</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data30bytes /dir/data30bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /dir/data15bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /dir/data120bytes</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data60bytes /dir/data60bytes</command>
+ <command>-fs NAMENODE -ls -S -r /dir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Found 4 items
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*15( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data15bytes
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*30( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data30bytes
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*60( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data60bytes
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*120( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/data120bytes</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>ls: files sorted in modification order</description>
+ <!-- this test assumes that the files take > 1ms each to create -->
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir</command>
+ <command>-fs NAMENODE -touchz /dir/first</command>
+ <command>-fs NAMENODE -touchz /dir/second</command>
+ <command>-fs NAMENODE -touchz /dir/third</command>
+ <command>-fs NAMENODE -touchz /dir/fourth</command>
+ <command>-fs NAMENODE -touchz /dir/fifth</command>
+ <command>-fs NAMENODE -ls -t /dir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Found 5 items
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/fifth
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/fourth
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/third
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/second
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/first</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
+ <description>ls: files sorted in reverse modification order</description>
+ <!-- this test assumes that the files take > 1ms each to create -->
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir</command>
+ <command>-fs NAMENODE -touchz /dir/first</command>
+ <command>-fs NAMENODE -touchz /dir/second</command>
+ <command>-fs NAMENODE -touchz /dir/third</command>
+ <command>-fs NAMENODE -touchz /dir/fourth</command>
+ <command>-fs NAMENODE -touchz /dir/fifth</command>
+ <command>-fs NAMENODE -ls -t -r /dir</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /dir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>Found 5 items
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/first
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/second
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/third
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/fourth
+-rw-r--r--( )*1( )*[a-zA-z0-9]*( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir/fifth</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test> <!-- TESTED -->
<description>rm: Test for quoted globbing </description>
<windows>false</windows>
<test-commands>
[39/50] [abbrv] hadoop git commit: HDFS-316. Balancer should run for
a configurable # of iterations (Xiaoyu Yao via aw)
Posted by zj...@apache.org.
HDFS-316. Balancer should run for a configurable # of iterations (Xiaoyu Yao via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b94c1117
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b94c1117
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b94c1117
Branch: refs/heads/YARN-2928
Commit: b94c1117a28e996adee68fe0e181eb6f536289f4
Parents: b015fec
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Feb 11 08:10:34 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Feb 11 08:10:34 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop/hdfs/server/balancer/Balancer.java | 28 +++++++++++++++-----
.../hdfs/server/balancer/NameNodeConnector.java | 26 ++++++++++++------
.../apache/hadoop/hdfs/server/mover/Mover.java | 3 ++-
.../src/site/apt/HDFSCommands.apt.vm | 5 +++-
.../hdfs/server/balancer/TestBalancer.java | 6 ++++-
.../hadoop/hdfs/server/mover/TestMover.java | 8 +++++-
7 files changed, 61 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1d2a76a..ed5db9b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -139,6 +139,9 @@ Trunk (Unreleased)
HDFS-7546. Document, and set an accepting default for
dfs.namenode.kerberos.principal.pattern (Harsh J via aw)
+ HDFS-316. Balancer should run for a configurable # of iterations (Xiaoyu
+ Yao via aw)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index dba1e2d..5b87cb5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -74,6 +74,10 @@ import com.google.common.base.Preconditions;
* start the balancer with a default threshold of 10%
* bin/ start-balancer.sh -threshold 5
* start the balancer with a threshold of 5%
+ * bin/ start-balancer.sh -idleiterations 20
+ * start the balancer with maximum 20 consecutive idle iterations
+ * bin/ start-balancer.sh -idleiterations -1
+ * run the balancer with default threshold infinitely
* To stop:
* bin/ stop-balancer.sh
* </pre>
@@ -136,7 +140,7 @@ import com.google.common.base.Preconditions;
* <ol>
* <li>The cluster is balanced;
* <li>No block can be moved;
- * <li>No block has been moved for five consecutive iterations;
+ * <li>No block has been moved for specified consecutive iterations (5 by default);
* <li>An IOException occurs while communicating with the namenode;
* <li>Another balancer is running.
* </ol>
@@ -147,7 +151,7 @@ import com.google.common.base.Preconditions;
* <ol>
* <li>The cluster is balanced. Exiting
* <li>No block can be moved. Exiting...
- * <li>No block has been moved for 5 iterations. Exiting...
+ * <li>No block has been moved for specified iterations (5 by default). Exiting...
* <li>Received an IO exception: failure reason. Exiting...
* <li>Another balancer is running. Exiting...
* </ol>
@@ -175,7 +179,9 @@ public class Balancer {
+ "\n\t[-exclude [-f <hosts-file> | comma-sperated list of hosts]]"
+ "\tExcludes the specified datanodes."
+ "\n\t[-include [-f <hosts-file> | comma-sperated list of hosts]]"
- + "\tIncludes only the specified datanodes.";
+ + "\tIncludes only the specified datanodes."
+ + "\n\t[-idleiterations <idleiterations>]"
+ + "\tNumber of consecutive idle iterations (-1 for Infinite) before exit.";
private final Dispatcher dispatcher;
private final BalancingPolicy policy;
@@ -572,7 +578,7 @@ public class Balancer {
List<NameNodeConnector> connectors = Collections.emptyList();
try {
connectors = NameNodeConnector.newNameNodeConnectors(namenodes,
- Balancer.class.getSimpleName(), BALANCER_ID_PATH, conf);
+ Balancer.class.getSimpleName(), BALANCER_ID_PATH, conf, p.maxIdleIteration);
boolean done = false;
for(int iteration = 0; !done; iteration++) {
@@ -628,19 +634,22 @@ public class Balancer {
static class Parameters {
static final Parameters DEFAULT = new Parameters(
BalancingPolicy.Node.INSTANCE, 10.0,
+ NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS,
Collections.<String> emptySet(), Collections.<String> emptySet());
final BalancingPolicy policy;
final double threshold;
+ final int maxIdleIteration;
// exclude the nodes in this set from balancing operations
Set<String> nodesToBeExcluded;
//include only these nodes in balancing operations
Set<String> nodesToBeIncluded;
- Parameters(BalancingPolicy policy, double threshold,
+ Parameters(BalancingPolicy policy, double threshold, int maxIdleIteration,
Set<String> nodesToBeExcluded, Set<String> nodesToBeIncluded) {
this.policy = policy;
this.threshold = threshold;
+ this.maxIdleIteration = maxIdleIteration;
this.nodesToBeExcluded = nodesToBeExcluded;
this.nodesToBeIncluded = nodesToBeIncluded;
}
@@ -649,6 +658,7 @@ public class Balancer {
public String toString() {
return Balancer.class.getSimpleName() + "." + getClass().getSimpleName()
+ "[" + policy + ", threshold=" + threshold +
+ ", max idle iteration = " + maxIdleIteration +
", number of nodes to be excluded = "+ nodesToBeExcluded.size() +
", number of nodes to be included = "+ nodesToBeIncluded.size() +"]";
}
@@ -687,6 +697,7 @@ public class Balancer {
static Parameters parse(String[] args) {
BalancingPolicy policy = Parameters.DEFAULT.policy;
double threshold = Parameters.DEFAULT.threshold;
+ int maxIdleIteration = Parameters.DEFAULT.maxIdleIteration;
Set<String> nodesTobeExcluded = Parameters.DEFAULT.nodesToBeExcluded;
Set<String> nodesTobeIncluded = Parameters.DEFAULT.nodesToBeIncluded;
@@ -742,6 +753,11 @@ public class Balancer {
} else {
nodesTobeIncluded = Util.parseHostList(args[i]);
}
+ } else if ("-idleiterations".equalsIgnoreCase(args[i])) {
+ checkArgument(++i < args.length,
+ "idleiterations value is missing: args = " + Arrays.toString(args));
+ maxIdleIteration = Integer.parseInt(args[i]);
+ LOG.info("Using a idleiterations of " + maxIdleIteration);
} else {
throw new IllegalArgumentException("args = "
+ Arrays.toString(args));
@@ -755,7 +771,7 @@ public class Balancer {
}
}
- return new Parameters(policy, threshold, nodesTobeExcluded, nodesTobeIncluded);
+ return new Parameters(policy, threshold, maxIdleIteration, nodesTobeExcluded, nodesTobeIncluded);
}
private static void printUsage(PrintStream out) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
index e01d57d..cf5f36f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/NameNodeConnector.java
@@ -60,18 +60,18 @@ import com.google.common.annotations.VisibleForTesting;
public class NameNodeConnector implements Closeable {
private static final Log LOG = LogFactory.getLog(NameNodeConnector.class);
- private static final int MAX_NOT_CHANGED_ITERATIONS = 5;
+ public static final int DEFAULT_MAX_IDLE_ITERATIONS = 5;
private static boolean write2IdFile = true;
/** Create {@link NameNodeConnector} for the given namenodes. */
public static List<NameNodeConnector> newNameNodeConnectors(
- Collection<URI> namenodes, String name, Path idPath, Configuration conf)
- throws IOException {
+ Collection<URI> namenodes, String name, Path idPath, Configuration conf,
+ int maxIdleIterations) throws IOException {
final List<NameNodeConnector> connectors = new ArrayList<NameNodeConnector>(
namenodes.size());
for (URI uri : namenodes) {
NameNodeConnector nnc = new NameNodeConnector(name, uri, idPath,
- null, conf);
+ null, conf, maxIdleIterations);
nnc.getKeyManager().startBlockKeyUpdater();
connectors.add(nnc);
}
@@ -80,12 +80,12 @@ public class NameNodeConnector implements Closeable {
public static List<NameNodeConnector> newNameNodeConnectors(
Map<URI, List<Path>> namenodes, String name, Path idPath,
- Configuration conf) throws IOException {
+ Configuration conf, int maxIdleIterations) throws IOException {
final List<NameNodeConnector> connectors = new ArrayList<NameNodeConnector>(
namenodes.size());
for (Map.Entry<URI, List<Path>> entry : namenodes.entrySet()) {
NameNodeConnector nnc = new NameNodeConnector(name, entry.getKey(),
- idPath, entry.getValue(), conf);
+ idPath, entry.getValue(), conf, maxIdleIterations);
nnc.getKeyManager().startBlockKeyUpdater();
connectors.add(nnc);
}
@@ -111,15 +111,18 @@ public class NameNodeConnector implements Closeable {
private final List<Path> targetPaths;
private final AtomicLong bytesMoved = new AtomicLong();
+ private final int maxNotChangedIterations;
private int notChangedIterations = 0;
public NameNodeConnector(String name, URI nameNodeUri, Path idPath,
- List<Path> targetPaths, Configuration conf)
+ List<Path> targetPaths, Configuration conf,
+ int maxNotChangedIterations)
throws IOException {
this.nameNodeUri = nameNodeUri;
this.idPath = idPath;
this.targetPaths = targetPaths == null || targetPaths.isEmpty() ? Arrays
.asList(new Path("/")) : targetPaths;
+ this.maxNotChangedIterations = maxNotChangedIterations;
this.namenode = NameNodeProxies.createProxy(conf, nameNodeUri,
NamenodeProtocol.class).getProxy();
@@ -182,7 +185,14 @@ public class NameNodeConnector implements Closeable {
notChangedIterations = 0;
} else {
notChangedIterations++;
- if (notChangedIterations >= MAX_NOT_CHANGED_ITERATIONS) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No block has been moved for " +
+ notChangedIterations + " iterations, " +
+ "maximum notChangedIterations before exit is: " +
+ ((maxNotChangedIterations >= 0) ? maxNotChangedIterations : "Infinite"));
+ }
+ if ((maxNotChangedIterations >= 0) &&
+ (notChangedIterations >= maxNotChangedIterations)) {
System.out.println("No block has been moved for "
+ notChangedIterations + " iterations. Exiting...");
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index a22f920..6fa6963 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -530,7 +530,8 @@ public class Mover {
List<NameNodeConnector> connectors = Collections.emptyList();
try {
connectors = NameNodeConnector.newNameNodeConnectors(namenodes,
- Mover.class.getSimpleName(), MOVER_ID_PATH, conf);
+ Mover.class.getSimpleName(), MOVER_ID_PATH, conf,
+ NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS);
while (connectors.size() > 0) {
Collections.shuffle(connectors);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
index 941a8ee..846b0b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HDFSCommands.apt.vm
@@ -324,7 +324,7 @@ HDFS Commands Guide
** <<<balancer>>>
- Usage: <<<hdfs balancer [-threshold <threshold>] [-policy <policy>]>>>
+ Usage: <<<hdfs balancer [-threshold <threshold>] [-policy <policy>] [-idleiterations <idleiterations>]>>>
*------------------------+----------------------------------------------------+
|| COMMAND_OPTION | Description
@@ -337,6 +337,9 @@ HDFS Commands Guide
| -threshold <threshold> | Percentage of disk capacity. This overwrites the
| | default threshold.
*------------------------+----------------------------------------------------+
+| -idleiterations <iterations> | Maximum number of idle iterations before exit.
+| | This overwrites the default idleiterations(5).
+*------------------------+----------------------------------------------------+
Runs a cluster balancing utility. An administrator can simply press Ctrl-C
to stop the rebalancing process. See
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
index 6955fcd..153baeb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/balancer/TestBalancer.java
@@ -564,6 +564,7 @@ public class TestBalancer {
p = new Balancer.Parameters(
Balancer.Parameters.DEFAULT.policy,
Balancer.Parameters.DEFAULT.threshold,
+ Balancer.Parameters.DEFAULT.maxIdleIteration,
nodes.getNodesToBeExcluded(), nodes.getNodesToBeIncluded());
}
@@ -629,7 +630,8 @@ public class TestBalancer {
List<NameNodeConnector> connectors = Collections.emptyList();
try {
connectors = NameNodeConnector.newNameNodeConnectors(namenodes,
- Balancer.class.getSimpleName(), Balancer.BALANCER_ID_PATH, conf);
+ Balancer.class.getSimpleName(), Balancer.BALANCER_ID_PATH, conf,
+ Balancer.Parameters.DEFAULT.maxIdleIteration);
boolean done = false;
for(int iteration = 0; !done; iteration++) {
@@ -801,6 +803,7 @@ public class TestBalancer {
Balancer.Parameters p = new Balancer.Parameters(
Balancer.Parameters.DEFAULT.policy,
Balancer.Parameters.DEFAULT.threshold,
+ Balancer.Parameters.DEFAULT.maxIdleIteration,
datanodes, Balancer.Parameters.DEFAULT.nodesToBeIncluded);
final int r = Balancer.run(namenodes, p, conf);
assertEquals(ExitStatus.SUCCESS.getExitCode(), r);
@@ -1233,6 +1236,7 @@ public class TestBalancer {
Balancer.Parameters p = new Balancer.Parameters(
Parameters.DEFAULT.policy,
Parameters.DEFAULT.threshold,
+ Balancer.Parameters.DEFAULT.maxIdleIteration,
Parameters.DEFAULT.nodesToBeExcluded,
Parameters.DEFAULT.nodesToBeIncluded);
final int r = Balancer.run(namenodes, p, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94c1117/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
index c9fc5ba..f35e1c8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/mover/TestMover.java
@@ -21,6 +21,7 @@ import java.io.IOException;
import java.net.URI;
import java.util.*;
+import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -39,9 +40,14 @@ public class TestMover {
static Mover newMover(Configuration conf) throws IOException {
final Collection<URI> namenodes = DFSUtil.getNsServiceRpcUris(conf);
Assert.assertEquals(1, namenodes.size());
+ Map<URI, List<Path>> nnMap = Maps.newHashMap();
+ for (URI nn : namenodes) {
+ nnMap.put(nn, null);
+ }
final List<NameNodeConnector> nncs = NameNodeConnector.newNameNodeConnectors(
- namenodes, Mover.class.getSimpleName(), Mover.MOVER_ID_PATH, conf);
+ nnMap, Mover.class.getSimpleName(), Mover.MOVER_ID_PATH, conf,
+ NameNodeConnector.DEFAULT_MAX_IDLE_ITERATIONS);
return new Mover(nncs.get(0), conf);
}
[36/50] [abbrv] hadoop git commit: YARN-2683. [YARN-913] registry
config options: document and move to core-default. (stevel)
Posted by zj...@apache.org.
YARN-2683. [YARN-913] registry config options: document and move to core-default. (stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3da2db4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3da2db4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3da2db4
Branch: refs/heads/YARN-2928
Commit: c3da2db48fd18c41096fe5d6d4650978fb31ae24
Parents: 362565c
Author: Steve Loughran <st...@apache.org>
Authored: Wed Feb 11 14:26:22 2015 +0000
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Feb 11 14:26:37 2015 +0000
----------------------------------------------------------------------
.../src/main/resources/core-default.xml | 132 ++++++
hadoop-yarn-project/CHANGES.txt | 3 +
.../src/main/resources/yarn-default.xml | 125 -----
.../src/site/markdown/registry/index.md | 1 +
.../markdown/registry/registry-configuration.md | 453 +++++++++++++++++++
.../site/markdown/registry/registry-security.md | 4 +-
.../registry/using-the-yarn-service-registry.md | 235 +++++++---
.../src/site/markdown/registry/yarn-registry.md | 73 +--
8 files changed, 813 insertions(+), 213 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
index f31db31..7135ab8 100644
--- a/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
+++ b/hadoop-common-project/hadoop-common/src/main/resources/core-default.xml
@@ -1676,4 +1676,136 @@ for ldap providers in the same way as above does.
</description>
</property>
+
+ <!-- YARN registry -->
+
+ <property>
+ <description>
+ Is the registry enabled in the YARN Resource Manager?
+
+ If true, the YARN RM will, as needed.
+ create the user and system paths, and purge
+ service records when containers, application attempts
+ and applications complete.
+
+ If false, the paths must be created by other means,
+ and no automatic cleanup of service records will take place.
+ </description>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>
+ </description>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff.
+
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>
+ List of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+
+ <property>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+
+ These are given full access to all entries.
+
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:hdfs@</value>
+ </property>
+
+ <property>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+
+ If empty, the default realm of the running process
+ is used.
+
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
+ Key to define the JAAS context. Used in secure
+ mode
+ </description>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ </property>
+
</configuration>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 14b3462..a1c3407 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -263,6 +263,9 @@ Release 2.7.0 - UNRELEASED
YARN-3100. Made YARN authorization pluggable. (Jian He via zjshen)
+ YARN-2683. [YARN-913] registry config options: document and move to
+ core-default. (stevel)
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 971ce0b..fb5fbe7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -1568,131 +1568,6 @@
<value>false</value>
</property>
- <!-- YARN registry -->
-
- <property>
- <description>
- Is the registry enabled: does the RM start it up,
- create the user and system paths, and purge
- service records when containers, application attempts
- and applications complete
- </description>
- <name>hadoop.registry.rm.enabled</name>
- <value>false</value>
- </property>
-
- <property>
- <description>
- </description>
- <name>hadoop.registry.zk.root</name>
- <value>/registry</value>
- </property>
-
- <property>
- <description>
- Zookeeper session timeout in milliseconds
- </description>
- <name>hadoop.registry.zk.session.timeout.ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <description>
- Zookeeper session timeout in milliseconds
- </description>
- <name>hadoop.registry.zk.connection.timeout.ms</name>
- <value>15000</value>
- </property>
-
- <property>
- <description>
- Zookeeper connection retry count before failing
- </description>
- <name>hadoop.registry.zk.retry.times</name>
- <value>5</value>
- </property>
-
- <property>
- <description>
- </description>
- <name>hadoop.registry.zk.retry.interval.ms</name>
- <value>1000</value>
- </property>
-
- <property>
- <description>
- Zookeeper retry limit in milliseconds, during
- exponential backoff: {@value}
-
- This places a limit even
- if the retry times and interval limit, combined
- with the backoff policy, result in a long retry
- period
- </description>
- <name>hadoop.registry.zk.retry.ceiling.ms</name>
- <value>60000</value>
- </property>
-
- <property>
- <description>
- List of hostname:port pairs defining the
- zookeeper quorum binding for the registry
- </description>
- <name>hadoop.registry.zk.quorum</name>
- <value>localhost:2181</value>
- </property>
-
- <property>
- <description>
- Key to set if the registry is secure. Turning it on
- changes the permissions policy from "open access"
- to restrictions on kerberos with the option of
- a user adding one or more auth key pairs down their
- own tree.
- </description>
- <name>hadoop.registry.secure</name>
- <value>false</value>
- </property>
-
- <property>
- <description>
- A comma separated list of Zookeeper ACL identifiers with
- system access to the registry in a secure cluster.
-
- These are given full access to all entries.
-
- If there is an "@" at the end of a SASL entry it
- instructs the registry client to append the default kerberos domain.
- </description>
- <name>hadoop.registry.system.acls</name>
- <value>sasl:yarn@, sasl:mapred@, sasl:mapred@hdfs@</value>
- </property>
-
- <property>
- <description>
- The kerberos realm: used to set the realm of
- system principals which do not declare their realm,
- and any other accounts that need the value.
-
- If empty, the default realm of the running process
- is used.
-
- If neither are known and the realm is needed, then the registry
- service/client will fail.
- </description>
- <name>hadoop.registry.kerberos.realm</name>
- <value></value>
- </property>
-
- <property>
- <description>
- Key to define the JAAS context. Used in secure
- mode
- </description>
- <name>hadoop.registry.jaas.context</name>
- <value>Client</value>
- </property>
-
<property>
<description>Defines how often NMs wake up to upload log files.
The default value is -1. By default, the logs will be uploaded when
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
index a9ea24f..3a648b6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/index.md
@@ -24,5 +24,6 @@ and use the binding information to connect with the services's network-accessibl
endpoints, be they REST, IPC, Web UI, Zookeeper quorum+path or some other protocol.
* [Architecture](yarn-registry.html)
+* [Configuration](registry-configuration.html)
* [Using the YARN Service registry](using-the-yarn-service-registry.html)
* [Security](registry-security.html)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
new file mode 100644
index 0000000..adca451
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-configuration.md
@@ -0,0 +1,453 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+# Registry Configuration
+
+
+The YARN service registry is built on top of Apache Zookeeper.
+It is configured by way of a Hadoop `Configuration` class:
+the instance used to create the service controls the behavior of the client.
+
+This document lists the configuration parameters which control the
+registry client and its deployment in the YARN Resource Manager.
+
+The default values of all these settings are defined in `core-default.xml`.
+The values in this file may not match those listed in this document.
+If this is the case, the values in `core-default.xml` MUST be considered normative.
+
+## Changing the configuration values
+
+Changes to the configuration values SHOULD be done in `core-site.xml`.
+This will ensure that client and non-YARN applications will pick up
+the values, so enabling them to read from and potentially write to the registry.
+
+
+## Core Settings
+
+
+### Enabling the Registry in the Resource Manager
+
+The Resource Manager manages user directory creation and record cleanup
+on YARN container/application attempt/application completion.
+
+```
+ <property>
+ <description>
+ Is the registry enabled in the YARN Resource Manager?
+
+ If true, the YARN RM will, as needed.
+ create the user and system paths, and purge
+ service records when containers, application attempts
+ and applications complete.
+
+ If false, the paths must be created by other means,
+ and no automatic cleanup of service records will take place.
+ </description>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ </property>
+```
+
+If the property is set in `core-site.xml` or `yarn-site.xml`,
+the YARN Resource Manager will behave as follows:
+1. On startup: create the initial root paths of `/`, `/services` and `/users`.
+ On a secure cluster, access will be restricted to the system accounts (see below).
+2. When a user submits a job: create the user path under `/users`.
+3. When a container is completed: delete from the registry all service records
+ with a `yarn:persistence` field of value `container`, and a `yarn:id` field
+ whose value matches the ID of the completed container.
+4. When an application attempt is completed: remove all service records with
+ `yarn:persistence` set to `application-attempt` and `yarn:id` set to the
+ pplication attempt ID.
+5. When an application finishes: remove all service records with
+ `yarn:persistence` set to `application` and `yarn:id` set to the
+ application ID.
+
+All these operations are asynchronous, so that zookeeper connectivity problems
+do not delay RM operations or work scheduling.
+
+If the property `hadoop.registry.rm.enabled` is set to `false`, the RM will
+not interact with the registry —and the listed operations will not take place.
+The root paths may be created by other means, but service record cleanup will not take place.
+
+### Setting the Zookeeper Quorum: `hadoop.registry.zk.quorum`
+
+This is an essential setting: it identifies the lists of zookeeper hosts
+and the ports on which the ZK services are listening.
+
+
+```
+ <property>
+ <description>
+ List of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+```
+
+It takes a comma-separated list, such as `zk1:2181 ,zk2:2181, zk3:2181`
+
+### Setting the Zookeeper Registry Base path: `hadoop.registry.zk.root`
+
+This path sets the base zookeeper node for the registry
+
+```
+ <property>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ </property>
+```
+
+The default value of `/registry` is normally sufficient. A different value
+may be needed for security reasons or because the `/registry` path is in use.
+
+The root value is prepended to all registry paths so as to create the absolute
+path. For example:
+
+* `/` maps to `/registry`
+* `/services` maps to `/registry/services`
+* `/users/yarn` maps to `/registry/users/yarn`
+
+A different value of `hadoop.registry.zk.root` would result in a different
+mapping to absolute zookeeper paths.
+
+
+## Security Options
+
+Registry security is enabled when the property `hadoop.registry.secure`
+is set to `true`. Once set, nodes are created with permissions, so that
+only a specific user *and the configured cluster "superuser" accounts*
+can write under their home path of `${hadoop.registry.zk.root}/users`.
+Only the superuser accounts
+will be able to manipulate the root path, including `${hadoop.registry.zk.root}/services`
+and `${hadoop.registry.zk.root}/users`.
+
+All write operations on the registry (including deleting entries and paths)
+must be authenticated. Read operations are still permitted by unauthenticated
+callers.
+
+The key settings for secure registry support are:
+
+* enabling the secure mode: `hadoop.registry.secure`
+* listing the superuser zookeeper ACLs: `hadoop.registry.system.acls`
+* listing the kerberos realm for the principals: `hadoop.registry.kerberos.realm`
+* identifying the JAAS context within the JAAS configuration which defines
+the user: `hadoop.registry.jaas.context`
+
+
+### Enabling security
+
+```
+ <property>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ </property>
+```
+
+### Identifying the client JAAS context
+
+The registry clients must identify the JAAS context which they use
+to authenticate to the registry.
+
+```
+ <property>
+ <description>
+ Key to define the JAAS context. Used in secure mode
+ </description>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ </property>
+```
+
+*Note* as the Resource Manager is simply another client of the registry, it
+too must have this context defined.
+
+
+### Identifying the system accounts `hadoop.registry.system.acls`
+
+These are the the accounts which are given full access to the base of the
+registry. The Resource Manager needs this option to create the root paths.
+
+Client applications writing to the registry access to the nodes it creates.
+
+1. The property `hadoop.registry.system.acls` takes a comma-separated list
+of zookeeper `ACLs` which are given full access to created nodes; the permissions
+`READ | WRITE | CREATE | DELETE | ADMIN`.
+2. Any zookeeper ACL scheme may be added to this, such as the `digest:` scheme.
+3. The SASL scheme, `sasl:`, is used to identify which callers identified
+by sasl have full access. These are the superuser accounts.
+4. They may be identified by elements such as `sasl:yarn@REALM.COM`.
+5. To aid portability of SASL settings, especially the default value,
+any `sasl:` entry without the realm value —that
+is, any entry that terminates in the `@` symbol— has the current realm appended
+to it.
+6. This realm is set to that of the current user.
+7. It may be overridden by the property `hadoop.registry.kerberos.realm`.
+
+
+```
+ <property>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+ These are given full access to all entries.
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:mapred@, sasl:hdfs@</value>
+ </property>
+
+ <property>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+ If empty, the default realm of the running process
+ is used.
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ </property>
+```
+
+Example: an `hadoop.registry.system.acls` entry of
+ `sasl:yarn@, sasl:admin@EXAMPLE.COM, sasl:system@REALM2`,
+would, in a YARN cluster with the realm `EXAMPLE.COM`, add the following
+admin accounts to every node
+
+* `sasl:yarn@EXAMPLE.COM`
+* `sasl:admin@EXAMPLE.COM`
+* `sasl:system@REALM2`
+
+The identity of a client application creating registry entries will be
+automatically included in the permissions of all entries created.
+If, for example, the account creating an entry was `hbase`, another
+entry would be created
+
+* `sasl:hbase@EXAMPLE.COM`
+
+
+**Important**: when setting the system ACLS, *it is critical to include the
+identity of the YARN Resource Manager*.
+
+The RM needs to be able to create the root and user paths, and delete service
+records during application and container cleanup.
+
+
+## Zookeeper connection management options
+
+Some low level options manage the ZK connection —more specifically, its failure
+handling.
+
+The Zookeeper registry clients use Apache Curator to connect to Zookeeper,
+a library which detects timeouts and attempts to reconnect to one of the
+servers which forms the zookeeper quorum. It is only after a timeout is detected
+that a retry is triggered.
+
+```
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>
+ </description>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff.
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ </property>
+```
+
+The retry strategy used in the registry client is
+[`BoundedExponentialBackoffRetry`](https://curator.apache.org/apidocs/org/apache/curator/retry/BoundedExponentialBackoffRetry.html):
+This backs off exponentially on connection failures before eventually
+concluding that the quorum is unreachable and failing.
+
+## Complete Set of Configuration Options
+
+```
+ <!-- YARN registry -->
+
+ <property>
+ <description>
+ Is the registry enabled: does the RM start it up,
+ create the user and system paths, and purge
+ service records when containers, application attempts
+ and applications complete
+ </description>
+ <name>hadoop.registry.rm.enabled</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ List of hostname:port pairs defining the
+ zookeeper quorum binding for the registry
+ </description>
+ <name>hadoop.registry.zk.quorum</name>
+ <value>localhost:2181</value>
+ </property>
+
+ <property>
+ <description>
+ The root zookeeper node for the registry
+ </description>
+ <name>hadoop.registry.zk.root</name>
+ <value>/registry</value>
+ </property>
+
+ <property>
+ <description>
+ Key to set if the registry is secure. Turning it on
+ changes the permissions policy from "open access"
+ to restrictions on kerberos with the option of
+ a user adding one or more auth key pairs down their
+ own tree.
+ </description>
+ <name>hadoop.registry.secure</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
+ A comma separated list of Zookeeper ACL identifiers with
+ system access to the registry in a secure cluster.
+
+ These are given full access to all entries.
+
+ If there is an "@" at the end of a SASL entry it
+ instructs the registry client to append the default kerberos domain.
+ </description>
+ <name>hadoop.registry.system.acls</name>
+ <value>sasl:yarn@, sasl:mapred@, sasl:mapred@, sasl:hdfs@</value>
+ </property>
+
+ <property>
+ <description>
+ The kerberos realm: used to set the realm of
+ system principals which do not declare their realm,
+ and any other accounts that need the value.
+
+ If empty, the default realm of the running process
+ is used.
+
+ If neither are known and the realm is needed, then the registry
+ service/client will fail.
+ </description>
+ <name>hadoop.registry.kerberos.realm</name>
+ <value></value>
+ </property>
+
+ <property>
+ <description>
+ Key to define the JAAS context. Used in secure
+ mode
+ </description>
+ <name>hadoop.registry.jaas.context</name>
+ <value>Client</value>
+ </property>
+
+
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.session.timeout.ms</name>
+ <value>60000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper session timeout in milliseconds
+ </description>
+ <name>hadoop.registry.zk.connection.timeout.ms</name>
+ <value>15000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper connection retry count before failing
+ </description>
+ <name>hadoop.registry.zk.retry.times</name>
+ <value>5</value>
+ </property>
+
+ <property>
+ <description>
+ </description>
+ <name>hadoop.registry.zk.retry.interval.ms</name>
+ <value>1000</value>
+ </property>
+
+ <property>
+ <description>
+ Zookeeper retry limit in milliseconds, during
+ exponential backoff: {@value}
+
+ This places a limit even
+ if the retry times and interval limit, combined
+ with the backoff policy, result in a long retry
+ period
+ </description>
+ <name>hadoop.registry.zk.retry.ceiling.ms</name>
+ <value>60000</value>
+ </property>
+```
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
index 7278534..6317681 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/registry-security.md
@@ -24,8 +24,8 @@ This document is therefore relevant only to secure clusters.
## Security Model
The security model of the registry is designed to meet the following goals
-a secur
-1. Deliver functional security on e ZK installation.
+a secure registry:
+1. Deliver functional security on a secure ZK installation.
1. Allow the RM to create per-user regions of the registration space
1. Allow applications belonging to a user to write registry entries
into their part of the space. These may be short-lived or long-lived
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
index dcae372..4df762e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/using-the-yarn-service-registry.md
@@ -19,17 +19,39 @@
The YARN service registry can be used in a numbe of ways :-
-1. To register dynamic YARN-deployed applications with entries that match the lifespan of the YARN application. Service Records can be set to be deleted on the completion of the YARN application, the application attempt, or an individual container.
-1. To look up static or dynamic applications and the mechanisms to communicate with them. Those mechanisms can incude: HTTP(S) URLs, Zookeeper paths, hostnames and ports and even paths in a Hadoop filesystem to configuration data.
-1. On a secure cluster, to verify that a service binding has been published by a specific user, or a system account. This can be done simply by looking at the path under which an entry has been placed.
-1. To register static applications. These will remain in the registry until deleted. They can be updated as required.
-
-A user of the registry may be both a publisher of entries —Service Records— and a consumer of other services located via their service records. Different parts of a distributed application may also use it for different purposes -central manager services can publish bindings for use by the worker services, which can then look up the bindings to communicate with that manager even if it was restarted on different nodes in the cluster.
+1. To register dynamic YARN-deployed applications with entries that match the
+ lifespan of the YARN application.
+ Service Records can be set to be deleted on
+ the completion of the YARN application, the application attempt,
+ or an individual container.
+1. To look up static or dynamic applications and the mechanisms to communicate
+ with them.
+ Those mechanisms can incude: HTTP(S) URLs, Zookeeper paths,
+ hostnames and ports and even paths in a Hadoop filesystem to
+ configuration data.
+1. On a secure cluster, to verify that a service binding has been published
+ by a specific user, or a system account.
+ This can be done simply by looking at the path under which an entry has
+ been placed.
+1. To register static applications.
+ These will remain in the registry until deleted.
+ They can be updated as required.
+
+A user of the registry may be both a publisher of entries —Service Records—
+and a consumer of other services located via their service records.
+Different parts of a distributed application may also use it for different
+purposes. As an example, the Application Master of a YARN application
+can publish bindings for use by its worker containers. The code running in the containers
+which can then look up the bindings to communicate with that manager even
+if it was restarted on different nodes in the cluster. Client applications can
+look up external service endpoints to interact with the AM via a public API.
The registry cannot be used:-
* To subscribe to service records or registry paths and listen for changes.
-* To directly share arbitrary data from a server for their clients. Such data must be published by some other means, a means which the registry entry can publish.
+* To directly share arbitrary data from a server for their clients.
+ Such data must be published by some other means, a means which the registry
+ entry can publish.
* To share secrets between processes. The registry is world readable.
@@ -38,13 +60,17 @@ The registry cannot be used:-
### Short-lived YARN Application Masters registering their public service endpoints.
-1. A YARN application is deployed. In a secure cluster, it is given the kerberos token to write to the registry.
+1. A YARN application is deployed. In a secure cluster, it is given the kerberos
+ token to write to the registry.
2. When launched, it creates a service record at a known path
-3. This record MAY have application attempt persistence policy of and an ID of the application attempt
+3. This record MAY have application attempt persistence policy of and an ID of
+ the application attempt
yarn:persistence = "application_attempt"
yarn:id = ${application_attemptId}
- This means that the record will be deleted when the application attempt completes, even if a new attempt is created. Every Application attempt will have to re-register the endpoint —which may be needed to locate the service anyway.
+
+ This means that the record will be deleted when the application attempt
+ completes, even if a new attempt is created. Every Application attempt will have to re-register the endpoint —which may be needed to locate the service anyway.
4. Alternatively, the record MAY have the persistence policy of "application":
yarn:persistence = "application_attempt"
@@ -52,7 +78,9 @@ The registry cannot be used:-
This means that the record will persist even between application attempts, albeit with out of date endpoint information.
5. Client applications look up the service by way of the path.
-The choice of path is an application specific one. For services with a yarn application name guaranteed to be unique, we recommend a convention of:
+The choice of path is an application specific one.
+For services with a YARN application name guaranteed to be unique,
+we recommend a convention of:
/users/${username}/applications/${service-class}/${instance-name}
@@ -68,16 +96,21 @@ Client applications may locate the service
* From a supplied service class and instance name
* If listed by application ID, from the service class and application ID.
-After locating a service record, the client may enumerate the `external` bindings and locate the entry with the desired API.
+After locating a service record, the client can enumerate the `external`
+bindings and locate the entry with the desired API.
### YARN Containers registering their public service endpoints
-Here all containers in a YARN application are publishing service endpoints for public consumption.
+Here all containers in a YARN application are publishing service endpoints
+for public consumption.
-1. The deployed containers are passed the base path under which they should register themselves.
-2. Long-lived containers must be passed an `id:password` pair which gives them the right to update these entries without the kerberos credentials of the user. This allows the containers to update their entries even after the user tokens granting the AM write access to a registry path expire.
-3. The containers instantiate a registry operations instance with the id:pass pair.
+1. The deployed containers are passed the base path under which they should
+ register themselves.
+2. Long-lived containers must be passed an `id:password` pair which gives
+ them the right to update these entries without the kerberos credentials of the user. This allows the containers to update their entries even after the user tokens granting the AM write access to a registry path expire.
+3. The containers instantiate a registry operations instance with the
+ `id:password` pair.
4. They then a register service record on a path consisting of:
${base-path} + "/" + RegistryPathUtils.encodeYarnID(containerId)
@@ -89,62 +122,152 @@ Here all containers in a YARN application are publishing service endpoints for p
When the container is terminated, the entry will be automatically deleted.
-5. The exported service endpoints of this container-deployed service should be listed in the `external` endpoint list of the service record.
-6. Clients may enumerate all containers exported by a YARN application by listing the entries under `${base-path}`.
+5. The exported service endpoints of this container-deployed service should
+ be listed in the `external` endpoint list of the service record.
+6. Clients can enumerate all containers exported by a YARN application by
+ listing the entries under `${base-path}`.
### Registering Static cluster services.
-Services which are generally fixed in a cluster, but which need to publish binding and configuration information may be published in the registry. Example: an Apache Oozie service.
-Services external to the cluster to which deployed applications may also be published. Example: An Amazon Dynamo instance.
-
-
-These services can be registered under paths which belong to the users running the service, such as `/users/oozie` or `/users/hbase`. Client applications would use this path. While this can authenticate the validity of the service record, it does rely on the client applications knowing the username a service is deployed on, or being configured with the full path.
-
-The alternative is for the services to be deployed under a static services path, under `/services`. For example, `/services/oozie` could contain the registration of the Oozie service. As the permissions for this path are restricted to pre-configured system accounts, the presence of a service registration does, on a secure cluster, confirm that it was registered by the cluster administration tools.
-
-1. The service is deployed by some management tool, or directly by the cluster operator.
-2. The deployed application can register itself under its own user name if given the binding information for the registry.
-3. If the application is to be registered under `/services` and it has been deployed by one of the system user accounts —it may register itself directly.
-4. If the application does not have the permissions to do so, the cluster administration tools must register the service instead.
-5. Client applications may locate a service by resolving its well known/configured path.
-5. If a service is stopped, the administration tools may delete the entry, or retain the entry but delete all it service endpoints. This is a proposed convention to indicate "the service is known but not currently reachable".
-6. When a service is restarted, its binding information may be updated, or its entire registry entry recreated.
+Services which are generally fixed in a cluster, but which need to publish
+binding and configuration information may be published in the registry.
+Example: an Apache Oozie service.
+Services external to the cluster to which deployed applications may also
+be published. Example: An Amazon Dynamo instance.
+
+
+These services can be registered under paths which belong to the users
+running the service, such as `/users/oozie` or `/users/hbase`.
+Client applications would use this path.
+While this can authenticate the validity of the service record,
+it does rely on the client applications knowing the username a service
+is deployed on, or being configured with the full path.
+
+The alternative is for the services to be deployed under a static services path,
+under `/services`. For example, `/services/oozie` could contain
+the registration of the Oozie service.
+As the permissions for this path are restricted to pre-configured
+system accounts, the presence of a service registration on this path on a secure
+cluster, confirms that it was registered by the cluster administration tools.
+
+1. The service is deployed by some management tool, or directly by
+ the cluster operator.
+2. The deployed application can register itself under its own user name
+ if given the binding information for the registry.
+3. If the application is to be registered under `/services` and it has been
+ deployed by one of the system user accounts —it may register itself directly.
+4. If the application does not have the permissions to do so, the cluster
+ administration tools must register the service instead.
+5. Client applications may locate a service by resolving its well
+ known/configured path.
+6. If a service is stopped, the administration tools may delete the entry,
+ or retain the entry but delete all it service endpoints.
+ This is a proposed convention to indicate
+ "the service is known but not currently reachable".
+7. When a service is restarted, its binding information may be updated,
+ or its entire registry entry recreated.
### YARN containers locating their Application Master
-Here YARN containers register with their AM to receive work, usually by some heartbeat mechanism where they report in regularly. If the AM is configured for containers to outlive the application attempt, when an AM fails the containers keep running. These containers will need to bind to any restarted AM. They may also wish to conclude that if an AM does not restart, that they should eventually time out and terminate themselves. Such a policy helps the application react to network partitions.
-
-1. The YARN AM publishes its service endpoints such as the FQDN and socket port neede for IPC communications, or an HTTP/HTTPS URL needed for a REST channel. These are published in the `internal` endpoint list, with the specific API the containers use
-2. The YARN containers are launched with the path to the service record (somehow) passed to them. Environment variables or command line parameters are two viable mechanisms. Shared secrets may also be passed this way, so allowing the containers to authenticate themselves with the AM. An alternative means of publishing secrets is saving information to the filesystem in a path specified as one of the endpoints.
-2. The YARN containers look up the service registry to identify the communications binding.
-3. If the registered service entry cannot be found, the container MAY do one of: exit. spin with some (jittered) retry period, polling for the entry, until the entry reappears. This implies that the AM has been found.
-4. If the service entry is found, the client should attempt to communicate with the AM on its channel. Shared authentication details may be used to validate the client with the server and vice versa.
-5. The client report in to the AM until the connections start failing to connect or authenticate, or when a long lived connection is broken and cannot be restarted.
-6. A this point the client may revert to step (3). Again, some backoff policy with some jitter helps stop a newly-restarted AM being overloaded. Containers may also with to have some timeout after which they conclude that the AM is not coming back and exit.
-6. We recommend that alongside the functional commands that an AM may issue to a client, a "terminate" command can be issued to a container. This allows the system to handle the specific situation of the YARN Node Manager terminating while spawned containers keep running.
+Here YARN containers register with their AM to receive work, usually by some
+heartbeat mechanism where they report in regularly.
+If the AM is configured for containers to outlive the application attempt,
+when an AM fails the containers keep running.
+These containers will need to bind to any restarted AM.
+They may also wish to conclude that if an AM does not restart,
+that they should eventually time out and terminate themselves.
+Such a policy helps the application react to network partitions.
+
+1. The YARN AM publishes its service endpoints such as the FQDN and
+ socket port needed for IPC communications, or an HTTP/HTTPS URL needed
+ for a REST channel.
+ These are published in the `internal` endpoint list, with the
+ `api` field set to a URL of the specific API the containers use.
+1. The YARN containers are launched with the path to the service record
+ (somehow) passed to them.
+ Environment variables or command line parameters are two viable mechanisms.
+ Shared secrets should also be passed that way: command line parameters are
+ visible in the unix `ps` command.
+ More secure is saving shared secrets to the cluster filesystem,
+ passing down the path to the containers. The URI to such as path MAY be one
+ of the registered internal endpoints of the application.
+1. The YARN containers look up the service registry to identify the
+ communications binding.
+1. If the registered service entry cannot be found, the container MAY do one of:
+ exit. spin with some (jittered) retry period, polling for the entry, until
+ the entry reappears. This implies that the AM has been found.
+1. If the service entry is found, the client should attempt to communicate
+ with the AM on its channel.
+ Shared authentication details may be used to validate the client with the
+ server and vice versa.
+1. The client report in to the AM until the connections start failing to
+ connect or authenticate, or when a long lived connection is broken
+ and cannot be restarted.
+1. A this point the client may revert to step (3).
+ Again, some backoff policy with some jitter helps stop a
+ newly-restarted AM being overloaded.
+ Containers may also with to have some timeout after which they conclude
+ that the AM is not coming back and exit.
+1. We recommend that alongside the functional commands that an AM may
+ issue to a client, a "terminate" command can be issued to a container.
+ This allows the system to handle the specific situation of the
+ YARN Node Manager terminating while spawned containers keep running.
### YARN Applications and containers publishing their management and metrics bindings
-Management ports and bindings are simply another endpoint to publish. These should be published as *internal* endpoints, as they are not intended for public consumption. By convention, the name of the management protocol shoudl be used as the endpoint's declared API: `JMX`, `ganglia`, etc.
-
+Management ports and bindings are simply others endpoint to publish.
+These should be published as *internal* endpoints, as they are not
+intended for public consumption.
### Client application enumerating services by endpoint APIs
-A client application wishes to locate all services implementing a specific API, such as `"org.apache.hbase"`
+A client application wishes to locate all services implementing a specific API,
+such as `"classpath://org.apache.hbase"`
1. The client starts from a path in the registry
-1. The client calls `registryOperations.list(path)` to list all nodes directly under that path, getting a relative list of child nodes.
-1. the client enumerates the child record statuses by calling `stat()` on each child.
-1. For all status entries, if the size of the entry is > the value of `ServiceRecordHeader.getLength()`, it MAY contain a service record.
-1. The contents can be retrieved using the `resolve()` operation. If successful, it does contain a service record —so the client can enumerate the `external` endpoints and locate the one with the desired API.
-1. The `children` field of each `RegistryPathStatus` status entry should be examined. If it is >= 0, the enumeration should be performed recursively on the path of that entry.
+1. The client calls `registryOperations.list(path)` to list all nodes directly
+ under that path, getting a relative list of child nodes.
+1. the client enumerates the child record statuses by calling `stat()`
+ on each child.
+1. For all status entries, if the size of the entry is greater than the
+ value of `ServiceRecordHeader.getLength()`, it MAY contain a service record.
+1. The contents can be retrieved using the `resolve()` operation.
+ If successful, it does contain a service record —so the client can enumerate
+ the `external` endpoints and locate the one with the desired API.
+1. The `children` field of each `RegistryPathStatus` status entry should
+ be examined. If it is >= 0, the enumeration should be performed recursively on the path of that entry.
1. The operation ultimately completes with a list of all entries.
-
-This algorithm describes a depth first search of the registry tree. Variations are of course possible, including breadth first search, or immediately halting the search as soon as a single entry point. There is also the option of parallel searches of different subtrees —which may reduce search time, albeit at the price of a higher client load on the registry infrastructure.
-
-A Utility class `RegistryUtils` provides static utility methods for common registry operations,
-in particular, `RegistryUtils.listServiceRecords(registryOperations, path)`
+1. One of the enumerated endpoints may be selected and used as the binding information
+ for a service
+
+This algorithm describes a depth first search of the registry tree.
+Variations are of course possible, including breadth-first search,
+or immediately halting the search as soon as a single entry point.
+There is also the option of parallel searches of different subtrees
+—this may reduce search time, albeit at the price of a higher client
+load on the registry infrastructure.
+
+A utility class `RegistryUtils` provides static utility methods for
+common registry operations,in particular,
+`RegistryUtils.listServiceRecords(registryOperations, path)`
performs the listing and collection of all immediate child record entries of
a specified path.
+
+Client applications are left with the problem of "what to do when the endpoint
+is not valid", specifically, when a service is not running —what should be done?
+
+Some transports assume that the outage is transient, and that spinning retries
+against the original binding is the correct strategy. This is the default
+policy of the Hadoop IPC client.
+
+Other transports fail fast, immediately reporting the failure via an
+exception or other mechanism. This is directly visible to the client —but
+does allow the client to rescan the registry and rebind to the application.
+
+Finally, some application have been designed for dynamic failover from the
+outset: their published binding information is actually a zookeeper path.
+Apache HBase and Apache Accumulo are examples of this. The registry is used
+for the initial lookup of the binding, after which the clients are inherently
+resilient to failure.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3da2db4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
index b38d9fb..f5055d9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/registry/yarn-registry.md
@@ -19,9 +19,13 @@
# Introduction and concepts
-This document describes a YARN service registry built to address a problem:
-*how can clients talk to YARN-deployed services and the components which form
-such services?*
+This document describes a YARN service registry built to address two problems:
+
+1. How can clients talk to YARN-deployed services and the components which form
+such services?
+1. Allow Hadoop core services to be registered and discovered thereby
+reducing configuration parameters and to allow core services to be more
+easily moved.
Service registration and discovery is a long-standing problem in distributed
computing, dating back to Xerox's Grapevine Service. This proposal is for a
@@ -346,31 +350,32 @@ A Service Record contains some basic informations and two lists of endpoints:
one list for users of a service, one list for internal use within the
application.
+<table>
+ <tr>
+ <td>Name</td>
+ <td>Description</td>
+ </tr>
+ <tr>
+ <td>type: String</td>
+ <td>Always: "JSONServiceRecord"</td>
+ </tr>
+ <tr>
+ <td>description: String</td>
+ <td>Human-readable description.</td>
+ </tr>
+ <tr>
+ <td>external: List[Endpoint]</td>
+ <td>A list of service endpoints for external callers.</td>
+ </tr>
+ <tr>
+ <td>internal: List[Endpoint]</td>
+ <td>A list of service endpoints for internal use within the service instance.</td>
+ </tr>
+</table>
- <table>
- <tr>
- <td>Name</td>
- <td>Description</td>
- </tr>
- <tr>
- <td>type: String</td>
- <td>Always: "JSONServiceRecord"</td>
- </tr>
- <tr>
- <td>description: String</td>
- <td>Human-readable description.</td>
- </tr>
- <tr>
- <td>external: List[Endpoint]</td>
- <td>A list of service endpoints for external callers.</td>
- </tr>
- <tr>
- <td>internal: List[Endpoint]</td>
- <td>A list of service endpoints for internal use within the service instance.</td>
- </tr>
- </table>
-
-The type field MUST be `"JSONServiceRecord"`. Mandating this string allows future record types *and* permits rapid rejection of byte arrays that lack this string before attempting JSON parsing.
+The type field MUST be `"JSONServiceRecord"`. Mandating this string allows
+future record types *and* permits rapid rejection of byte arrays that
+lack this string before attempting to parse the data with a JSON parser.
### YARN Persistence policies
@@ -379,8 +384,15 @@ as an application, attempt or container is completed.
This allows service to register entries which have a lifespan bound to one of
these aspects of YARN applications' lifecycles. This is a feature which is only
-supported when the RM has enabled its support, and would not apply to
-any use of the registry without the RM's participation.
+supported when the RM has had its registry integration enabled via the
+configuration option `hadoop.registry.rm.enabled`.
+
+If this option is enabled, and the YARN resource manager is running,
+it will clean up service records as defined
+below.
+
+If the option is disabled, the RM does not provide any registry support at all.
+
The attributes, `yarn:id` and `yarn:persistence` specify which records
*and any child entries* may be deleted as the associated YARN components complete.
@@ -1006,7 +1018,8 @@ The details are irrelevant —note that they use an application-specific API
value to ensure uniqueness.
Internal:
-1. Two URLS to REST APIs offered by the AM for containers deployed by
+
+1. Two URLs to REST APIs offered by the AM for containers deployed by
the application itself.
Python agents running in the containers retrieve the internal endpoint
[44/50] [abbrv] hadoop git commit: HDFS-7723. Quota By Storage Type
namenode implemenation. (Contributed by Xiaoyu Yao)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
index 2a1b549..201db9e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/editsStored.xml
@@ -1,6 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<EDITS>
- <EDITS_VERSION>-62</EDITS_VERSION>
+ <EDITS_VERSION>-63</EDITS_VERSION>
<RECORD>
<OPCODE>OP_START_LOG_SEGMENT</OPCODE>
<DATA>
@@ -13,8 +13,8 @@
<TXID>2</TXID>
<DELEGATION_KEY>
<KEY_ID>1</KEY_ID>
- <EXPIRY_DATE>1422569009939</EXPIRY_DATE>
- <KEY>907cb34000041937</KEY>
+ <EXPIRY_DATE>1423097579620</EXPIRY_DATE>
+ <KEY>ef3f2032e2797e8e</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -24,8 +24,8 @@
<TXID>3</TXID>
<DELEGATION_KEY>
<KEY_ID>2</KEY_ID>
- <EXPIRY_DATE>1422569009941</EXPIRY_DATE>
- <KEY>178fa1bd83474b43</KEY>
+ <EXPIRY_DATE>1423097579622</EXPIRY_DATE>
+ <KEY>b978ed731a0b4a65</KEY>
</DELEGATION_KEY>
</DATA>
</RECORD>
@@ -37,18 +37,18 @@
<INODEID>16386</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877810832</MTIME>
- <ATIME>1421877810832</ATIME>
+ <MTIME>1422406380345</MTIME>
+ <ATIME>1422406380345</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>6</RPC_CALLID>
</DATA>
</RECORD>
@@ -60,14 +60,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877810888</MTIME>
- <ATIME>1421877810832</ATIME>
+ <MTIME>1422406380369</MTIME>
+ <ATIME>1422406380345</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -78,10 +78,10 @@
<DATA>
<TXID>6</TXID>
<PATH>/file_create</PATH>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<NEWBLOCK>false</NEWBLOCK>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>8</RPC_CALLID>
</DATA>
</RECORD>
@@ -93,14 +93,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877810899</MTIME>
- <ATIME>1421877810832</ATIME>
+ <MTIME>1422406380376</MTIME>
+ <ATIME>1422406380345</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -121,8 +121,8 @@
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1421877810907</TIMESTAMP>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <TIMESTAMP>1422406380383</TIMESTAMP>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>11</RPC_CALLID>
</DATA>
</RECORD>
@@ -132,8 +132,8 @@
<TXID>10</TXID>
<LENGTH>0</LENGTH>
<PATH>/file_moved</PATH>
- <TIMESTAMP>1421877810915</TIMESTAMP>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <TIMESTAMP>1422406380392</TIMESTAMP>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>12</RPC_CALLID>
</DATA>
</RECORD>
@@ -144,9 +144,9 @@
<LENGTH>0</LENGTH>
<INODEID>16387</INODEID>
<PATH>/directory_mkdir</PATH>
- <TIMESTAMP>1421877810923</TIMESTAMP>
+ <TIMESTAMP>1422406380399</TIMESTAMP>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>493</MODE>
</PERMISSION_STATUS>
@@ -179,7 +179,7 @@
<TXID>15</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot1</SNAPSHOTNAME>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>17</RPC_CALLID>
</DATA>
</RECORD>
@@ -190,7 +190,7 @@
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTOLDNAME>snapshot1</SNAPSHOTOLDNAME>
<SNAPSHOTNEWNAME>snapshot2</SNAPSHOTNEWNAME>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>18</RPC_CALLID>
</DATA>
</RECORD>
@@ -200,7 +200,7 @@
<TXID>17</TXID>
<SNAPSHOTROOT>/directory_mkdir</SNAPSHOTROOT>
<SNAPSHOTNAME>snapshot2</SNAPSHOTNAME>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>19</RPC_CALLID>
</DATA>
</RECORD>
@@ -212,18 +212,18 @@
<INODEID>16388</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877810946</MTIME>
- <ATIME>1421877810946</ATIME>
+ <MTIME>1422406380423</MTIME>
+ <ATIME>1422406380423</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
<RPC_CALLID>20</RPC_CALLID>
</DATA>
</RECORD>
@@ -235,14 +235,14 @@
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877810948</MTIME>
- <ATIME>1421877810946</ATIME>
+ <MTIME>1422406380425</MTIME>
+ <ATIME>1422406380423</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
<OVERWRITE>false</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -292,59 +292,68 @@
</DATA>
</RECORD>
<RECORD>
- <OPCODE>OP_RENAME</OPCODE>
+ <OPCODE>OP_SET_QUOTA_BY_STORAGETYPE</OPCODE>
<DATA>
<TXID>25</TXID>
+ <SRC>/directory_mkdir</SRC>
+ <STORAGETYPE>1</STORAGETYPE>
+ <DSQUOTA>888</DSQUOTA>
+ </DATA>
+ </RECORD>
+ <RECORD>
+ <OPCODE>OP_RENAME</OPCODE>
+ <DATA>
+ <TXID>26</TXID>
<LENGTH>0</LENGTH>
<SRC>/file_create</SRC>
<DST>/file_moved</DST>
- <TIMESTAMP>1421877810968</TIMESTAMP>
+ <TIMESTAMP>1422406380442</TIMESTAMP>
<OPTIONS>NONE</OPTIONS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>27</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>28</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>26</TXID>
+ <TXID>27</TXID>
<LENGTH>0</LENGTH>
<INODEID>16389</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877810972</MTIME>
- <ATIME>1421877810972</ATIME>
+ <MTIME>1422406380446</MTIME>
+ <ATIME>1422406380446</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>29</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>30</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>27</TXID>
+ <TXID>28</TXID>
<BLOCK_ID>1073741825</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>28</TXID>
+ <TXID>29</TXID>
<GENSTAMPV2>1001</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>29</TXID>
+ <TXID>30</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
<BLOCK_ID>1073741825</BLOCK_ID>
@@ -358,21 +367,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>30</TXID>
+ <TXID>31</TXID>
<BLOCK_ID>1073741826</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>31</TXID>
+ <TXID>32</TXID>
<GENSTAMPV2>1002</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>32</TXID>
+ <TXID>33</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
<BLOCK_ID>1073741825</BLOCK_ID>
@@ -391,21 +400,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>33</TXID>
+ <TXID>34</TXID>
<BLOCK_ID>1073741827</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>34</TXID>
+ <TXID>35</TXID>
<GENSTAMPV2>1003</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>35</TXID>
+ <TXID>36</TXID>
<PATH>/file_concat_target</PATH>
<BLOCK>
<BLOCK_ID>1073741826</BLOCK_ID>
@@ -424,13 +433,13 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>36</TXID>
+ <TXID>37</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_target</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811083</MTIME>
- <ATIME>1421877810972</ATIME>
+ <MTIME>1422406380534</MTIME>
+ <ATIME>1422406380446</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -451,7 +460,7 @@
<GENSTAMP>1003</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -460,44 +469,44 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>37</TXID>
+ <TXID>38</TXID>
<LENGTH>0</LENGTH>
<INODEID>16390</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811086</MTIME>
- <ATIME>1421877811086</ATIME>
+ <MTIME>1422406380537</MTIME>
+ <ATIME>1422406380537</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>39</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>40</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>38</TXID>
+ <TXID>39</TXID>
<BLOCK_ID>1073741828</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>39</TXID>
+ <TXID>40</TXID>
<GENSTAMPV2>1004</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>40</TXID>
+ <TXID>41</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
<BLOCK_ID>1073741828</BLOCK_ID>
@@ -511,21 +520,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>41</TXID>
+ <TXID>42</TXID>
<BLOCK_ID>1073741829</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>42</TXID>
+ <TXID>43</TXID>
<GENSTAMPV2>1005</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>43</TXID>
+ <TXID>44</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
<BLOCK_ID>1073741828</BLOCK_ID>
@@ -544,21 +553,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>44</TXID>
+ <TXID>45</TXID>
<BLOCK_ID>1073741830</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>45</TXID>
+ <TXID>46</TXID>
<GENSTAMPV2>1006</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>46</TXID>
+ <TXID>47</TXID>
<PATH>/file_concat_0</PATH>
<BLOCK>
<BLOCK_ID>1073741829</BLOCK_ID>
@@ -577,13 +586,13 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>47</TXID>
+ <TXID>48</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_0</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811108</MTIME>
- <ATIME>1421877811086</ATIME>
+ <MTIME>1422406380558</MTIME>
+ <ATIME>1422406380537</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -604,7 +613,7 @@
<GENSTAMP>1006</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -613,44 +622,44 @@
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>48</TXID>
+ <TXID>49</TXID>
<LENGTH>0</LENGTH>
<INODEID>16391</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811110</MTIME>
- <ATIME>1421877811110</ATIME>
+ <MTIME>1422406380560</MTIME>
+ <ATIME>1422406380560</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>48</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>49</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>49</TXID>
+ <TXID>50</TXID>
<BLOCK_ID>1073741831</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>50</TXID>
+ <TXID>51</TXID>
<GENSTAMPV2>1007</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>51</TXID>
+ <TXID>52</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
<BLOCK_ID>1073741831</BLOCK_ID>
@@ -664,21 +673,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>52</TXID>
+ <TXID>53</TXID>
<BLOCK_ID>1073741832</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>53</TXID>
+ <TXID>54</TXID>
<GENSTAMPV2>1008</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>54</TXID>
+ <TXID>55</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
<BLOCK_ID>1073741831</BLOCK_ID>
@@ -697,21 +706,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>55</TXID>
+ <TXID>56</TXID>
<BLOCK_ID>1073741833</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>56</TXID>
+ <TXID>57</TXID>
<GENSTAMPV2>1009</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>57</TXID>
+ <TXID>58</TXID>
<PATH>/file_concat_1</PATH>
<BLOCK>
<BLOCK_ID>1073741832</BLOCK_ID>
@@ -730,13 +739,13 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>58</TXID>
+ <TXID>59</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_concat_1</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811131</MTIME>
- <ATIME>1421877811110</ATIME>
+ <MTIME>1422406380579</MTIME>
+ <ATIME>1422406380560</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -757,7 +766,7 @@
<GENSTAMP>1009</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -766,59 +775,59 @@
<RECORD>
<OPCODE>OP_CONCAT_DELETE</OPCODE>
<DATA>
- <TXID>59</TXID>
+ <TXID>60</TXID>
<LENGTH>0</LENGTH>
<TRG>/file_concat_target</TRG>
- <TIMESTAMP>1421877811134</TIMESTAMP>
+ <TIMESTAMP>1422406380582</TIMESTAMP>
<SOURCES>
<SOURCE1>/file_concat_0</SOURCE1>
<SOURCE2>/file_concat_1</SOURCE2>
</SOURCES>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>56</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>57</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>60</TXID>
+ <TXID>61</TXID>
<LENGTH>0</LENGTH>
<INODEID>16392</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811137</MTIME>
- <ATIME>1421877811137</ATIME>
+ <MTIME>1422406380586</MTIME>
+ <ATIME>1422406380586</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>58</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>59</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>61</TXID>
+ <TXID>62</TXID>
<BLOCK_ID>1073741834</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>62</TXID>
+ <TXID>63</TXID>
<GENSTAMPV2>1010</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>63</TXID>
+ <TXID>64</TXID>
<PATH>/file_create</PATH>
<BLOCK>
<BLOCK_ID>1073741834</BLOCK_ID>
@@ -832,21 +841,21 @@
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>64</TXID>
+ <TXID>65</TXID>
<BLOCK_ID>1073741835</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>65</TXID>
+ <TXID>66</TXID>
<GENSTAMPV2>1011</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>66</TXID>
+ <TXID>67</TXID>
<PATH>/file_create</PATH>
<BLOCK>
<BLOCK_ID>1073741834</BLOCK_ID>
@@ -865,13 +874,13 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>67</TXID>
+ <TXID>68</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/file_create</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811152</MTIME>
- <ATIME>1421877811137</ATIME>
+ <MTIME>1422406380599</MTIME>
+ <ATIME>1422406380586</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -887,7 +896,7 @@
<GENSTAMP>1011</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -896,74 +905,74 @@
<RECORD>
<OPCODE>OP_TRUNCATE</OPCODE>
<DATA>
- <TXID>68</TXID>
+ <TXID>69</TXID>
<SRC>/file_create</SRC>
- <CLIENTNAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENTNAME>
+ <CLIENTNAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENTNAME>
<CLIENTMACHINE>127.0.0.1</CLIENTMACHINE>
<NEWLENGTH>512</NEWLENGTH>
- <TIMESTAMP>1421877811154</TIMESTAMP>
+ <TIMESTAMP>1422406380601</TIMESTAMP>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SYMLINK</OPCODE>
<DATA>
- <TXID>69</TXID>
+ <TXID>70</TXID>
<LENGTH>0</LENGTH>
<INODEID>16393</INODEID>
<PATH>/file_symlink</PATH>
<VALUE>/file_concat_target</VALUE>
- <MTIME>1421877811160</MTIME>
- <ATIME>1421877811160</ATIME>
+ <MTIME>1422406380606</MTIME>
+ <ATIME>1422406380606</ATIME>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>511</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>65</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>66</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD</OPCODE>
<DATA>
- <TXID>70</TXID>
+ <TXID>71</TXID>
<LENGTH>0</LENGTH>
<INODEID>16394</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877811163</MTIME>
- <ATIME>1421877811163</ATIME>
+ <MTIME>1422406380608</MTIME>
+ <ATIME>1422406380608</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
- <CLIENT_NAME>DFSClient_NONMAPREDUCE_-986598042_1</CLIENT_NAME>
+ <CLIENT_NAME>DFSClient_NONMAPREDUCE_-156773767_1</CLIENT_NAME>
<CLIENT_MACHINE>127.0.0.1</CLIENT_MACHINE>
<OVERWRITE>true</OVERWRITE>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>66</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>67</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ALLOCATE_BLOCK_ID</OPCODE>
<DATA>
- <TXID>71</TXID>
+ <TXID>72</TXID>
<BLOCK_ID>1073741836</BLOCK_ID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>72</TXID>
+ <TXID>73</TXID>
<GENSTAMPV2>1012</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_BLOCK</OPCODE>
<DATA>
- <TXID>73</TXID>
+ <TXID>74</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
<BLOCK_ID>1073741836</BLOCK_ID>
@@ -977,7 +986,7 @@
<RECORD>
<OPCODE>OP_UPDATE_BLOCKS</OPCODE>
<DATA>
- <TXID>74</TXID>
+ <TXID>75</TXID>
<PATH>/hard-lease-recovery-test</PATH>
<BLOCK>
<BLOCK_ID>1073741836</BLOCK_ID>
@@ -991,15 +1000,15 @@
<RECORD>
<OPCODE>OP_SET_GENSTAMP_V2</OPCODE>
<DATA>
- <TXID>75</TXID>
+ <TXID>76</TXID>
<GENSTAMPV2>1013</GENSTAMPV2>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REASSIGN_LEASE</OPCODE>
<DATA>
- <TXID>76</TXID>
- <LEASEHOLDER>DFSClient_NONMAPREDUCE_-986598042_1</LEASEHOLDER>
+ <TXID>77</TXID>
+ <LEASEHOLDER>DFSClient_NONMAPREDUCE_-156773767_1</LEASEHOLDER>
<PATH>/hard-lease-recovery-test</PATH>
<NEWHOLDER>HDFS_NameNode</NEWHOLDER>
</DATA>
@@ -1007,13 +1016,13 @@
<RECORD>
<OPCODE>OP_CLOSE</OPCODE>
<DATA>
- <TXID>77</TXID>
+ <TXID>78</TXID>
<LENGTH>0</LENGTH>
<INODEID>0</INODEID>
<PATH>/hard-lease-recovery-test</PATH>
<REPLICATION>1</REPLICATION>
- <MTIME>1421877813736</MTIME>
- <ATIME>1421877811163</ATIME>
+ <MTIME>1422406383261</MTIME>
+ <ATIME>1422406380608</ATIME>
<BLOCKSIZE>512</BLOCKSIZE>
<CLIENT_NAME></CLIENT_NAME>
<CLIENT_MACHINE></CLIENT_MACHINE>
@@ -1024,7 +1033,7 @@
<GENSTAMP>1013</GENSTAMP>
</BLOCK>
<PERMISSION_STATUS>
- <USERNAME>jing</USERNAME>
+ <USERNAME>xyao</USERNAME>
<GROUPNAME>supergroup</GROUPNAME>
<MODE>420</MODE>
</PERMISSION_STATUS>
@@ -1033,72 +1042,72 @@
<RECORD>
<OPCODE>OP_ADD_CACHE_POOL</OPCODE>
<DATA>
- <TXID>78</TXID>
+ <TXID>79</TXID>
<POOLNAME>pool1</POOLNAME>
- <OWNERNAME>jing</OWNERNAME>
+ <OWNERNAME>xyao</OWNERNAME>
<GROUPNAME>staff</GROUPNAME>
<MODE>493</MODE>
<LIMIT>9223372036854775807</LIMIT>
<MAXRELATIVEEXPIRY>2305843009213693951</MAXRELATIVEEXPIRY>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>73</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>74</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MODIFY_CACHE_POOL</OPCODE>
<DATA>
- <TXID>79</TXID>
+ <TXID>80</TXID>
<POOLNAME>pool1</POOLNAME>
<LIMIT>99</LIMIT>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>74</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>75</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ADD_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>80</TXID>
+ <TXID>81</TXID>
<ID>1</ID>
<PATH>/path</PATH>
<REPLICATION>1</REPLICATION>
<POOL>pool1</POOL>
- <EXPIRATION>2305844431091508160</EXPIRATION>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>75</RPC_CALLID>
+ <EXPIRATION>2305844431620077592</EXPIRATION>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>76</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_MODIFY_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>81</TXID>
+ <TXID>82</TXID>
<ID>1</ID>
<REPLICATION>2</REPLICATION>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>76</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>77</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_CACHE_DIRECTIVE</OPCODE>
<DATA>
- <TXID>82</TXID>
+ <TXID>83</TXID>
<ID>1</ID>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>77</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>78</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_CACHE_POOL</OPCODE>
<DATA>
- <TXID>83</TXID>
+ <TXID>84</TXID>
<POOLNAME>pool1</POOLNAME>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>78</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>79</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_ACL</OPCODE>
<DATA>
- <TXID>84</TXID>
+ <TXID>85</TXID>
<SRC>/file_concat_target</SRC>
<ENTRY>
<SCOPE>ACCESS</SCOPE>
@@ -1131,62 +1140,62 @@
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>85</TXID>
+ <TXID>86</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a1</NAME>
<VALUE>0x313233</VALUE>
</XATTR>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>80</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>81</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_SET_XATTR</OPCODE>
<DATA>
- <TXID>86</TXID>
+ <TXID>87</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
<VALUE>0x373839</VALUE>
</XATTR>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>81</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>82</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_REMOVE_XATTR</OPCODE>
<DATA>
- <TXID>87</TXID>
+ <TXID>88</TXID>
<SRC>/file_concat_target</SRC>
<XATTR>
<NAMESPACE>USER</NAMESPACE>
<NAME>a2</NAME>
</XATTR>
- <RPC_CLIENTID>1730855b-1f27-4f17-9f72-b9f92eb3a8bd</RPC_CLIENTID>
- <RPC_CALLID>82</RPC_CALLID>
+ <RPC_CLIENTID>7334ec24-dd6b-4efd-807d-ed0d18625534</RPC_CLIENTID>
+ <RPC_CALLID>83</RPC_CALLID>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_START</OPCODE>
<DATA>
- <TXID>88</TXID>
- <STARTTIME>1421877814254</STARTTIME>
+ <TXID>89</TXID>
+ <STARTTIME>1422406383706</STARTTIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_ROLLING_UPGRADE_FINALIZE</OPCODE>
<DATA>
- <TXID>89</TXID>
- <FINALIZETIME>1421877814254</FINALIZETIME>
+ <TXID>90</TXID>
+ <FINALIZETIME>1422406383706</FINALIZETIME>
</DATA>
</RECORD>
<RECORD>
<OPCODE>OP_END_LOG_SEGMENT</OPCODE>
<DATA>
- <TXID>90</TXID>
+ <TXID>91</TXID>
</DATA>
</RECORD>
</EDITS>
[26/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
deleted file mode 100644
index 0235219..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/SecureMode.apt.vm
+++ /dev/null
@@ -1,689 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop in Secure Mode
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc|section=0|fromDepth=0|toDepth=3}
-
-Hadoop in Secure Mode
-
-* Introduction
-
- This document describes how to configure authentication for Hadoop in
- secure mode.
-
- By default Hadoop runs in non-secure mode in which no actual
- authentication is required.
- By configuring Hadoop runs in secure mode,
- each user and service needs to be authenticated by Kerberos
- in order to use Hadoop services.
-
- Security features of Hadoop consist of
- {{{Authentication}authentication}},
- {{{./ServiceLevelAuth.html}service level authorization}},
- {{{./HttpAuthentication.html}authentication for Web consoles}}
- and {{{Data confidentiality}data confidenciality}}.
-
-
-* Authentication
-
-** End User Accounts
-
- When service level authentication is turned on,
- end users using Hadoop in secure mode needs to be authenticated by Kerberos.
- The simplest way to do authentication is using <<<kinit>>> command of Kerberos.
-
-** User Accounts for Hadoop Daemons
-
- Ensure that HDFS and YARN daemons run as different Unix users,
- e.g. <<<hdfs>>> and <<<yarn>>>.
- Also, ensure that the MapReduce JobHistory server runs as
- different user such as <<<mapred>>>.
-
- It's recommended to have them share a Unix group, for e.g. <<<hadoop>>>.
- See also "{{Mapping from user to group}}" for group management.
-
-*---------------+----------------------------------------------------------------------+
-|| User:Group || Daemons |
-*---------------+----------------------------------------------------------------------+
-| hdfs:hadoop | NameNode, Secondary NameNode, JournalNode, DataNode |
-*---------------+----------------------------------------------------------------------+
-| yarn:hadoop | ResourceManager, NodeManager |
-*---------------+----------------------------------------------------------------------+
-| mapred:hadoop | MapReduce JobHistory Server |
-*---------------+----------------------------------------------------------------------+
-
-** Kerberos principals for Hadoop Daemons and Users
-
- For running hadoop service daemons in Hadoop in secure mode,
- Kerberos principals are required.
- Each service reads auhenticate information saved in keytab file with appropriate permission.
-
- HTTP web-consoles should be served by principal different from RPC's one.
-
- Subsections below shows the examples of credentials for Hadoop services.
-
-*** HDFS
-
- The NameNode keytab file, on the NameNode host, should look like the
- following:
-
-----
-$ klist -e -k -t /etc/security/keytab/nn.service.keytab
-Keytab name: FILE:/etc/security/keytab/nn.service.keytab
-KVNO Timestamp Principal
- 4 07/18/11 21:08:09 nn/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 nn/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 nn/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
-----
-
- The Secondary NameNode keytab file, on that host, should look like the
- following:
-
-----
-$ klist -e -k -t /etc/security/keytab/sn.service.keytab
-Keytab name: FILE:/etc/security/keytab/sn.service.keytab
-KVNO Timestamp Principal
- 4 07/18/11 21:08:09 sn/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 sn/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 sn/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
-----
-
- The DataNode keytab file, on each host, should look like the following:
-
-----
-$ klist -e -k -t /etc/security/keytab/dn.service.keytab
-Keytab name: FILE:/etc/security/keytab/dn.service.keytab
-KVNO Timestamp Principal
- 4 07/18/11 21:08:09 dn/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 dn/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 dn/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
-----
-
-*** YARN
-
- The ResourceManager keytab file, on the ResourceManager host, should look
- like the following:
-
-----
-$ klist -e -k -t /etc/security/keytab/rm.service.keytab
-Keytab name: FILE:/etc/security/keytab/rm.service.keytab
-KVNO Timestamp Principal
- 4 07/18/11 21:08:09 rm/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 rm/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 rm/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
-----
-
- The NodeManager keytab file, on each host, should look like the following:
-
-----
-$ klist -e -k -t /etc/security/keytab/nm.service.keytab
-Keytab name: FILE:/etc/security/keytab/nm.service.keytab
-KVNO Timestamp Principal
- 4 07/18/11 21:08:09 nm/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 nm/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 nm/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
-----
-
-*** MapReduce JobHistory Server
-
- The MapReduce JobHistory Server keytab file, on that host, should look
- like the following:
-
-----
-$ klist -e -k -t /etc/security/keytab/jhs.service.keytab
-Keytab name: FILE:/etc/security/keytab/jhs.service.keytab
-KVNO Timestamp Principal
- 4 07/18/11 21:08:09 jhs/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 jhs/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 jhs/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
- 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
-----
-
-** Mapping from Kerberos principal to OS user account
-
- Hadoop maps Kerberos principal to OS user account using
- the rule specified by <<<hadoop.security.auth_to_local>>>
- which works in the same way as the <<<auth_to_local>>> in
- {{{http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html}Kerberos configuration file (krb5.conf)}}.
- In addition, Hadoop <<<auth_to_local>>> mapping supports the <</L>> flag that
- lowercases the returned name.
-
- By default, it picks the first component of principal name as a user name
- if the realms matches to the <<<default_realm>>> (usually defined in /etc/krb5.conf).
- For example, <<<...@REALM.TLD>>> is mapped to <<<host>>>
- by default rule.
-
-** Mapping from user to group
-
- Though files on HDFS are associated to owner and group,
- Hadoop does not have the definition of group by itself.
- Mapping from user to group is done by OS or LDAP.
-
- You can change a way of mapping by
- specifying the name of mapping provider as a value of
- <<<hadoop.security.group.mapping>>>
- See {{{../hadoop-hdfs/HdfsPermissionsGuide.html}HDFS Permissions Guide}} for details.
-
- Practically you need to manage SSO environment using Kerberos with LDAP
- for Hadoop in secure mode.
-
-** Proxy user
-
- Some products such as Apache Oozie which access the services of Hadoop
- on behalf of end users need to be able to impersonate end users.
- See {{{./Superusers.html}the doc of proxy user}} for details.
-
-** Secure DataNode
-
- Because the data transfer protocol of DataNode
- does not use the RPC framework of Hadoop,
- DataNode must authenticate itself by
- using privileged ports which are specified by
- <<<dfs.datanode.address>>> and <<<dfs.datanode.http.address>>>.
- This authentication is based on the assumption
- that the attacker won't be able to get root privileges.
-
- When you execute <<<hdfs datanode>>> command as root,
- server process binds privileged port at first,
- then drops privilege and runs as the user account specified by
- <<<HADOOP_SECURE_DN_USER>>>.
- This startup process uses jsvc installed to <<<JSVC_HOME>>>.
- You must specify <<<HADOOP_SECURE_DN_USER>>> and <<<JSVC_HOME>>>
- as environment variables on start up (in hadoop-env.sh).
-
- As of version 2.6.0, SASL can be used to authenticate the data transfer
- protocol. In this configuration, it is no longer required for secured clusters
- to start the DataNode as root using jsvc and bind to privileged ports. To
- enable SASL on data transfer protocol, set <<<dfs.data.transfer.protection>>>
- in hdfs-site.xml, set a non-privileged port for <<<dfs.datanode.address>>>, set
- <<<dfs.http.policy>>> to <HTTPS_ONLY> and make sure the
- <<<HADOOP_SECURE_DN_USER>>> environment variable is not defined. Note that it
- is not possible to use SASL on data transfer protocol if
- <<<dfs.datanode.address>>> is set to a privileged port. This is required for
- backwards-compatibility reasons.
-
- In order to migrate an existing cluster that used root authentication to start
- using SASL instead, first ensure that version 2.6.0 or later has been deployed
- to all cluster nodes as well as any external applications that need to connect
- to the cluster. Only versions 2.6.0 and later of the HDFS client can connect
- to a DataNode that uses SASL for authentication of data transfer protocol, so
- it is vital that all callers have the correct version before migrating. After
- version 2.6.0 or later has been deployed everywhere, update configuration of
- any external applications to enable SASL. If an HDFS client is enabled for
- SASL, then it can connect successfully to a DataNode running with either root
- authentication or SASL authentication. Changing configuration for all clients
- guarantees that subsequent configuration changes on DataNodes will not disrupt
- the applications. Finally, each individual DataNode can be migrated by
- changing its configuration and restarting. It is acceptable to have a mix of
- some DataNodes running with root authentication and some DataNodes running with
- SASL authentication temporarily during this migration period, because an HDFS
- client enabled for SASL can connect to both.
-
-* Data confidentiality
-
-** Data Encryption on RPC
-
- The data transfered between hadoop services and clients.
- Setting <<<hadoop.rpc.protection>>> to <<<"privacy">>> in the core-site.xml
- activate data encryption.
-
-** Data Encryption on Block data transfer.
-
- You need to set <<<dfs.encrypt.data.transfer>>> to <<<"true">>> in the hdfs-site.xml
- in order to activate data encryption for data transfer protocol of DataNode.
-
- Optionally, you may set <<<dfs.encrypt.data.transfer.algorithm>>> to either
- "3des" or "rc4" to choose the specific encryption algorithm. If unspecified,
- then the configured JCE default on the system is used, which is usually 3DES.
-
- Setting <<<dfs.encrypt.data.transfer.cipher.suites>>> to
- <<<AES/CTR/NoPadding>>> activates AES encryption. By default, this is
- unspecified, so AES is not used. When AES is used, the algorithm specified in
- <<<dfs.encrypt.data.transfer.algorithm>>> is still used during an initial key
- exchange. The AES key bit length can be configured by setting
- <<<dfs.encrypt.data.transfer.cipher.key.bitlength>>> to 128, 192 or 256. The
- default is 128.
-
- AES offers the greatest cryptographic strength and the best performance. At
- this time, 3DES and RC4 have been used more often in Hadoop clusters.
-
-** Data Encryption on HTTP
-
- Data transfer between Web-console and clients are protected by using SSL(HTTPS).
-
-
-* Configuration
-
-** Permissions for both HDFS and local fileSystem paths
-
- The following table lists various paths on HDFS and local filesystems (on
- all nodes) and recommended permissions:
-
-*-------------------+-------------------+------------------+------------------+
-|| Filesystem || Path || User:Group || Permissions |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<dfs.namenode.name.dir>>> | hdfs:hadoop | drwx------ |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<dfs.datanode.data.dir>>> | hdfs:hadoop | drwx------ |
-*-------------------+-------------------+------------------+------------------+
-| local | $HADOOP_LOG_DIR | hdfs:hadoop | drwxrwxr-x |
-*-------------------+-------------------+------------------+------------------+
-| local | $YARN_LOG_DIR | yarn:hadoop | drwxrwxr-x |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<yarn.nodemanager.local-dirs>>> | yarn:hadoop | drwxr-xr-x |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<yarn.nodemanager.log-dirs>>> | yarn:hadoop | drwxr-xr-x |
-*-------------------+-------------------+------------------+------------------+
-| local | container-executor | root:hadoop | --Sr-s--- |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<conf/container-executor.cfg>>> | root:hadoop | r-------- |
-*-------------------+-------------------+------------------+------------------+
-| hdfs | / | hdfs:hadoop | drwxr-xr-x |
-*-------------------+-------------------+------------------+------------------+
-| hdfs | /tmp | hdfs:hadoop | drwxrwxrwxt |
-*-------------------+-------------------+------------------+------------------+
-| hdfs | /user | hdfs:hadoop | drwxr-xr-x |
-*-------------------+-------------------+------------------+------------------+
-| hdfs | <<<yarn.nodemanager.remote-app-log-dir>>> | yarn:hadoop | drwxrwxrwxt |
-*-------------------+-------------------+------------------+------------------+
-| hdfs | <<<mapreduce.jobhistory.intermediate-done-dir>>> | mapred:hadoop | |
-| | | | drwxrwxrwxt |
-*-------------------+-------------------+------------------+------------------+
-| hdfs | <<<mapreduce.jobhistory.done-dir>>> | mapred:hadoop | |
-| | | | drwxr-x--- |
-*-------------------+-------------------+------------------+------------------+
-
-** Common Configurations
-
- In order to turn on RPC authentication in hadoop,
- set the value of <<<hadoop.security.authentication>>> property to
- <<<"kerberos">>>, and set security related settings listed below appropriately.
-
- The following properties should be in the <<<core-site.xml>>> of all the
- nodes in the cluster.
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<hadoop.security.authentication>>> | <kerberos> | |
-| | | <<<simple>>> : No authentication. (default) \
-| | | <<<kerberos>>> : Enable authentication by Kerberos. |
-*-------------------------+-------------------------+------------------------+
-| <<<hadoop.security.authorization>>> | <true> | |
-| | | Enable {{{./ServiceLevelAuth.html}RPC service-level authorization}}. |
-*-------------------------+-------------------------+------------------------+
-| <<<hadoop.rpc.protection>>> | <authentication> |
-| | | <authentication> : authentication only (default) \
-| | | <integrity> : integrity check in addition to authentication \
-| | | <privacy> : data encryption in addition to integrity |
-*-------------------------+-------------------------+------------------------+
-| <<<hadoop.security.auth_to_local>>> | | |
-| | <<<RULE:>>><exp1>\
-| | <<<RULE:>>><exp2>\
-| | <...>\
-| | DEFAULT |
-| | | The value is string containing new line characters.
-| | | See
-| | | {{{http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html}Kerberos documentation}}
-| | | for format for <exp>.
-*-------------------------+-------------------------+------------------------+
-| <<<hadoop.proxyuser.>>><superuser><<<.hosts>>> | | |
-| | | comma separated hosts from which <superuser> access are allowd to impersonation. |
-| | | <<<*>>> means wildcard. |
-*-------------------------+-------------------------+------------------------+
-| <<<hadoop.proxyuser.>>><superuser><<<.groups>>> | | |
-| | | comma separated groups to which users impersonated by <superuser> belongs. |
-| | | <<<*>>> means wildcard. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/core-site.xml>>>
-
-** NameNode
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.block.access.token.enable>>> | <true> | |
-| | | Enable HDFS block access tokens for secure operations. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.https.enable>>> | <true> | |
-| | | This value is deprecated. Use dfs.http.policy |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.http.policy>>> | <HTTP_ONLY> or <HTTPS_ONLY> or <HTTP_AND_HTTPS> | |
-| | | HTTPS_ONLY turns off http access. This option takes precedence over |
-| | | the deprecated configuration dfs.https.enable and hadoop.ssl.enabled. |
-| | | If using SASL to authenticate data transfer protocol instead of |
-| | | running DataNode as root and using privileged ports, then this property |
-| | | must be set to <HTTPS_ONLY> to guarantee authentication of HTTP servers. |
-| | | (See <<<dfs.data.transfer.protection>>>.) |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.https-address>>> | <nn_host_fqdn:50470> | |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.https.port>>> | <50470> | |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.keytab.file>>> | </etc/security/keytab/nn.service.keytab> | |
-| | | Kerberos keytab file for the NameNode. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.kerberos.principal>>> | nn/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the NameNode. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.kerberos.internal.spnego.principal>>> | HTTP/_HOST@REALM.TLD | |
-| | | HTTP Kerberos principal name for the NameNode. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/hdfs-site.xml>>>
-
-** Secondary NameNode
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.secondary.http-address>>> | <c_nn_host_fqdn:50090> | |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.secondary.https-port>>> | <50470> | |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.secondary.namenode.keytab.file>>> | | |
-| | </etc/security/keytab/sn.service.keytab> | |
-| | | Kerberos keytab file for the Secondary NameNode. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.secondary.namenode.kerberos.principal>>> | sn/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the Secondary NameNode. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.secondary.namenode.kerberos.internal.spnego.principal>>> | | |
-| | HTTP/_HOST@REALM.TLD | |
-| | | HTTP Kerberos principal name for the Secondary NameNode. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/hdfs-site.xml>>>
-
-** DataNode
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.data.dir.perm>>> | 700 | |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.address>>> | <0.0.0.0:1004> | |
-| | | Secure DataNode must use privileged port |
-| | | in order to assure that the server was started securely. |
-| | | This means that the server must be started via jsvc. |
-| | | Alternatively, this must be set to a non-privileged port if using SASL |
-| | | to authenticate data transfer protocol. |
-| | | (See <<<dfs.data.transfer.protection>>>.) |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.http.address>>> | <0.0.0.0:1006> | |
-| | | Secure DataNode must use privileged port |
-| | | in order to assure that the server was started securely. |
-| | | This means that the server must be started via jsvc. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.https.address>>> | <0.0.0.0:50470> | |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.keytab.file>>> | </etc/security/keytab/dn.service.keytab> | |
-| | | Kerberos keytab file for the DataNode. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.kerberos.principal>>> | dn/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the DataNode. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.encrypt.data.transfer>>> | <false> | |
-| | | set to <<<true>>> when using data encryption |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.encrypt.data.transfer.algorithm>>> | | |
-| | | optionally set to <<<3des>>> or <<<rc4>>> when using data encryption to |
-| | | control encryption algorithm |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.encrypt.data.transfer.cipher.suites>>> | | |
-| | | optionally set to <<<AES/CTR/NoPadding>>> to activate AES encryption |
-| | | when using data encryption |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.encrypt.data.transfer.cipher.key.bitlength>>> | | |
-| | | optionally set to <<<128>>>, <<<192>>> or <<<256>>> to control key bit |
-| | | length when using AES with data encryption |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.data.transfer.protection>>> | | |
-| | | <authentication> : authentication only \
-| | | <integrity> : integrity check in addition to authentication \
-| | | <privacy> : data encryption in addition to integrity |
-| | | This property is unspecified by default. Setting this property enables |
-| | | SASL for authentication of data transfer protocol. If this is enabled, |
-| | | then <<<dfs.datanode.address>>> must use a non-privileged port, |
-| | | <<<dfs.http.policy>>> must be set to <HTTPS_ONLY> and the |
-| | | <<<HADOOP_SECURE_DN_USER>>> environment variable must be undefined when |
-| | | starting the DataNode process. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/hdfs-site.xml>>>
-
-
-** WebHDFS
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.web.authentication.kerberos.principal>>> | http/_HOST@REALM.TLD | |
-| | | Kerberos keytab file for the WebHDFS. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.web.authentication.kerberos.keytab>>> | </etc/security/keytab/http.service.keytab> | |
-| | | Kerberos principal name for WebHDFS. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/hdfs-site.xml>>>
-
-
-** ResourceManager
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.keytab>>> | | |
-| | </etc/security/keytab/rm.service.keytab> | |
-| | | Kerberos keytab file for the ResourceManager. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.principal>>> | rm/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the ResourceManager. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/yarn-site.xml>>>
-
-** NodeManager
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.keytab>>> | </etc/security/keytab/nm.service.keytab> | |
-| | | Kerberos keytab file for the NodeManager. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.principal>>> | nm/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the NodeManager. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.container-executor.class>>> | | |
-| | <<<org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor>>> |
-| | | Use LinuxContainerExecutor. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.linux-container-executor.group>>> | <hadoop> | |
-| | | Unix group of the NodeManager. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.linux-container-executor.path>>> | </path/to/bin/container-executor> | |
-| | | The path to the executable of Linux container executor. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/yarn-site.xml>>>
-
-** Configuration for WebAppProxy
-
- The <<<WebAppProxy>>> provides a proxy between the web applications
- exported by an application and an end user. If security is enabled
- it will warn users before accessing a potentially unsafe web application.
- Authentication and authorization using the proxy is handled just like
- any other privileged web application.
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.web-proxy.address>>> | | |
-| | <<<WebAppProxy>>> host:port for proxy to AM web apps. | |
-| | | <host:port> if this is the same as <<<yarn.resourcemanager.webapp.address>>>|
-| | | or it is not defined then the <<<ResourceManager>>> will run the proxy|
-| | | otherwise a standalone proxy server will need to be launched.|
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.web-proxy.keytab>>> | | |
-| | </etc/security/keytab/web-app.service.keytab> | |
-| | | Kerberos keytab file for the WebAppProxy. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.web-proxy.principal>>> | wap/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the WebAppProxy. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/yarn-site.xml>>>
-
-** LinuxContainerExecutor
-
- A <<<ContainerExecutor>>> used by YARN framework which define how any
- <container> launched and controlled.
-
- The following are the available in Hadoop YARN:
-
-*--------------------------------------+--------------------------------------+
-|| ContainerExecutor || Description |
-*--------------------------------------+--------------------------------------+
-| <<<DefaultContainerExecutor>>> | |
-| | The default executor which YARN uses to manage container execution. |
-| | The container process has the same Unix user as the NodeManager. |
-*--------------------------------------+--------------------------------------+
-| <<<LinuxContainerExecutor>>> | |
-| | Supported only on GNU/Linux, this executor runs the containers as either the |
-| | YARN user who submitted the application (when full security is enabled) or |
-| | as a dedicated user (defaults to nobody) when full security is not enabled. |
-| | When full security is enabled, this executor requires all user accounts to be |
-| | created on the cluster nodes where the containers are launched. It uses |
-| | a <setuid> executable that is included in the Hadoop distribution. |
-| | The NodeManager uses this executable to launch and kill containers. |
-| | The setuid executable switches to the user who has submitted the |
-| | application and launches or kills the containers. For maximum security, |
-| | this executor sets up restricted permissions and user/group ownership of |
-| | local files and directories used by the containers such as the shared |
-| | objects, jars, intermediate files, log files etc. Particularly note that, |
-| | because of this, except the application owner and NodeManager, no other |
-| | user can access any of the local files/directories including those |
-| | localized as part of the distributed cache. |
-*--------------------------------------+--------------------------------------+
-
- To build the LinuxContainerExecutor executable run:
-
-----
- $ mvn package -Dcontainer-executor.conf.dir=/etc/hadoop/
-----
-
- The path passed in <<<-Dcontainer-executor.conf.dir>>> should be the
- path on the cluster nodes where a configuration file for the setuid
- executable should be located. The executable should be installed in
- $HADOOP_YARN_HOME/bin.
-
- The executable must have specific permissions: 6050 or --Sr-s---
- permissions user-owned by <root> (super-user) and group-owned by a
- special group (e.g. <<<hadoop>>>) of which the NodeManager Unix user is
- the group member and no ordinary application user is. If any application
- user belongs to this special group, security will be compromised. This
- special group name should be specified for the configuration property
- <<<yarn.nodemanager.linux-container-executor.group>>> in both
- <<<conf/yarn-site.xml>>> and <<<conf/container-executor.cfg>>>.
-
- For example, let's say that the NodeManager is run as user <yarn> who is
- part of the groups users and <hadoop>, any of them being the primary group.
- Let also be that <users> has both <yarn> and another user
- (application submitter) <alice> as its members, and <alice> does not
- belong to <hadoop>. Going by the above description, the setuid/setgid
- executable should be set 6050 or --Sr-s--- with user-owner as <yarn> and
- group-owner as <hadoop> which has <yarn> as its member (and not <users>
- which has <alice> also as its member besides <yarn>).
-
- The LinuxTaskController requires that paths including and leading up to
- the directories specified in <<<yarn.nodemanager.local-dirs>>> and
- <<<yarn.nodemanager.log-dirs>>> to be set 755 permissions as described
- above in the table on permissions on directories.
-
- * <<<conf/container-executor.cfg>>>
-
- The executable requires a configuration file called
- <<<container-executor.cfg>>> to be present in the configuration
- directory passed to the mvn target mentioned above.
-
- The configuration file must be owned by the user running NodeManager
- (user <<<yarn>>> in the above example), group-owned by anyone and
- should have the permissions 0400 or r--------.
-
- The executable requires following configuration items to be present
- in the <<<conf/container-executor.cfg>>> file. The items should be
- mentioned as simple key=value pairs, one per-line:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.linux-container-executor.group>>> | <hadoop> | |
-| | | Unix group of the NodeManager. The group owner of the |
-| | |<container-executor> binary should be this group. Should be same as the |
-| | | value with which the NodeManager is configured. This configuration is |
-| | | required for validating the secure access of the <container-executor> |
-| | | binary. |
-*-------------------------+-------------------------+------------------------+
-| <<<banned.users>>> | hdfs,yarn,mapred,bin | Banned users. |
-*-------------------------+-------------------------+------------------------+
-| <<<allowed.system.users>>> | foo,bar | Allowed system users. |
-*-------------------------+-------------------------+------------------------+
-| <<<min.user.id>>> | 1000 | Prevent other super-users. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/yarn-site.xml>>>
-
- To re-cap, here are the local file-sysytem permissions required for the
- various paths related to the <<<LinuxContainerExecutor>>>:
-
-*-------------------+-------------------+------------------+------------------+
-|| Filesystem || Path || User:Group || Permissions |
-*-------------------+-------------------+------------------+------------------+
-| local | container-executor | root:hadoop | --Sr-s--- |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<conf/container-executor.cfg>>> | root:hadoop | r-------- |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<yarn.nodemanager.local-dirs>>> | yarn:hadoop | drwxr-xr-x |
-*-------------------+-------------------+------------------+------------------+
-| local | <<<yarn.nodemanager.log-dirs>>> | yarn:hadoop | drwxr-xr-x |
-*-------------------+-------------------+------------------+------------------+
-
-** MapReduce JobHistory Server
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.address>>> | | |
-| | MapReduce JobHistory Server <host:port> | Default port is 10020. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.keytab>>> | |
-| | </etc/security/keytab/jhs.service.keytab> | |
-| | | Kerberos keytab file for the MapReduce JobHistory Server. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.principal>>> | jhs/_HOST@REALM.TLD | |
-| | | Kerberos principal name for the MapReduce JobHistory Server. |
-*-------------------------+-------------------------+------------------------+
-Configuration for <<<conf/mapred-site.xml>>>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
deleted file mode 100644
index 86fb3d6..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/ServiceLevelAuth.apt.vm
+++ /dev/null
@@ -1,216 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Service Level Authorization Guide
- ---
- ---
- ${maven.build.timestamp}
-
-Service Level Authorization Guide
-
-%{toc|section=1|fromDepth=0}
-
-* Purpose
-
- This document describes how to configure and manage Service Level
- Authorization for Hadoop.
-
-* Prerequisites
-
- Make sure Hadoop is installed, configured and setup correctly. For more
- information see:
-
- * {{{./SingleCluster.html}Single Node Setup}} for first-time users.
-
- * {{{./ClusterSetup.html}Cluster Setup}} for large, distributed clusters.
-
-* Overview
-
- Service Level Authorization is the initial authorization mechanism to
- ensure clients connecting to a particular Hadoop service have the
- necessary, pre-configured, permissions and are authorized to access the
- given service. For example, a MapReduce cluster can use this mechanism
- to allow a configured list of users/groups to submit jobs.
-
- The <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>> configuration file is used to
- define the access control lists for various Hadoop services.
-
- Service Level Authorization is performed much before to other access
- control checks such as file-permission checks, access control on job
- queues etc.
-
-* Configuration
-
- This section describes how to configure service-level authorization via
- the configuration file <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>>.
-
-** Enable Service Level Authorization
-
- By default, service-level authorization is disabled for Hadoop. To
- enable it set the configuration property hadoop.security.authorization
- to true in <<<${HADOOP_CONF_DIR}/core-site.xml>>>.
-
-** Hadoop Services and Configuration Properties
-
- This section lists the various Hadoop services and their configuration
- knobs:
-
-*-------------------------------------+--------------------------------------+
-|| Property || Service
-*-------------------------------------+--------------------------------------+
-security.client.protocol.acl | ACL for ClientProtocol, which is used by user code via the DistributedFileSystem.
-*-------------------------------------+--------------------------------------+
-security.client.datanode.protocol.acl | ACL for ClientDatanodeProtocol, the client-to-datanode protocol for block recovery.
-*-------------------------------------+--------------------------------------+
-security.datanode.protocol.acl | ACL for DatanodeProtocol, which is used by datanodes to communicate with the namenode.
-*-------------------------------------+--------------------------------------+
-security.inter.datanode.protocol.acl | ACL for InterDatanodeProtocol, the inter-datanode protocol for updating generation timestamp.
-*-------------------------------------+--------------------------------------+
-security.namenode.protocol.acl | ACL for NamenodeProtocol, the protocol used by the secondary namenode to communicate with the namenode.
-*-------------------------------------+--------------------------------------+
-security.inter.tracker.protocol.acl | ACL for InterTrackerProtocol, used by the tasktrackers to communicate with the jobtracker.
-*-------------------------------------+--------------------------------------+
-security.job.submission.protocol.acl | ACL for JobSubmissionProtocol, used by job clients to communciate with the jobtracker for job submission, querying job status etc.
-*-------------------------------------+--------------------------------------+
-security.task.umbilical.protocol.acl | ACL for TaskUmbilicalProtocol, used by the map and reduce tasks to communicate with the parent tasktracker.
-*-------------------------------------+--------------------------------------+
-security.refresh.policy.protocol.acl | ACL for RefreshAuthorizationPolicyProtocol, used by the dfsadmin and mradmin commands to refresh the security policy in-effect.
-*-------------------------------------+--------------------------------------+
-security.ha.service.protocol.acl | ACL for HAService protocol used by HAAdmin to manage the active and stand-by states of namenode.
-*-------------------------------------+--------------------------------------+
-
-** Access Control Lists
-
- <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>> defines an access control list for
- each Hadoop service. Every access control list has a simple format:
-
- The list of users and groups are both comma separated list of names.
- The two lists are separated by a space.
-
- Example: <<<user1,user2 group1,group2>>>.
-
- Add a blank at the beginning of the line if only a list of groups is to
- be provided, equivalently a comma-separated list of users followed by
- a space or nothing implies only a set of given users.
-
- A special value of <<<*>>> implies that all users are allowed to access the
- service.
-
- If access control list is not defined for a service, the value of
- <<<security.service.authorization.default.acl>>> is applied. If
- <<<security.service.authorization.default.acl>>> is not defined, <<<*>>> is applied.
-
- ** Blocked Access Control Lists
-
- In some cases, it is required to specify blocked access control list for a service. This specifies
- the list of users and groups who are not authorized to access the service. The format of
- the blocked access control list is same as that of access control list. The blocked access
- control list can be specified via <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>>. The property name
- is derived by suffixing with ".blocked".
-
- Example: The property name of blocked access control list for <<<security.client.protocol.acl>>
- will be <<<security.client.protocol.acl.blocked>>>
-
- For a service, it is possible to specify both an access control list and a blocked control
- list. A user is authorized to access the service if the user is in the access control and not in
- the blocked access control list.
-
- If blocked access control list is not defined for a service, the value of
- <<<security.service.authorization.default.acl.blocked>>> is applied. If
- <<<security.service.authorization.default.acl.blocked>>> is not defined,
- empty blocked access control list is applied.
-
-
-** Refreshing Service Level Authorization Configuration
-
- The service-level authorization configuration for the NameNode and
- JobTracker can be changed without restarting either of the Hadoop
- master daemons. The cluster administrator can change
- <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>> on the master nodes and instruct
- the NameNode and JobTracker to reload their respective configurations
- via the <<<-refreshServiceAcl>>> switch to <<<dfsadmin>>> and <<<mradmin>>> commands
- respectively.
-
- Refresh the service-level authorization configuration for the NameNode:
-
-----
- $ bin/hadoop dfsadmin -refreshServiceAcl
-----
-
- Refresh the service-level authorization configuration for the
- JobTracker:
-
-----
- $ bin/hadoop mradmin -refreshServiceAcl
-----
-
- Of course, one can use the <<<security.refresh.policy.protocol.acl>>>
- property in <<<${HADOOP_CONF_DIR}/hadoop-policy.xml>>> to restrict access to
- the ability to refresh the service-level authorization configuration to
- certain users/groups.
-
- ** Access Control using list of ip addresses, host names and ip ranges
-
- Access to a service can be controlled based on the ip address of the client accessing
- the service. It is possible to restrict access to a service from a set of machines by
- specifying a list of ip addresses, host names and ip ranges. The property name for each service
- is derived from the corresponding acl's property name. If the property name of acl is
- security.client.protocol.acl, property name for the hosts list will be
- security.client.protocol.hosts.
-
- If hosts list is not defined for a service, the value of
- <<<security.service.authorization.default.hosts>>> is applied. If
- <<<security.service.authorization.default.hosts>>> is not defined, <<<*>>> is applied.
-
- It is possible to specify a blocked list of hosts. Only those machines which are in the
- hosts list, but not in the blocked hosts list will be granted access to the service. The property
- name is derived by suffixing with ".blocked".
-
- Example: The property name of blocked hosts list for <<<security.client.protocol.hosts>>
- will be <<<security.client.protocol.hosts.blocked>>>
-
- If blocked hosts list is not defined for a service, the value of
- <<<security.service.authorization.default.hosts.blocked>>> is applied. If
- <<<security.service.authorization.default.hosts.blocked>>> is not defined,
- empty blocked hosts list is applied.
-
-** Examples
-
- Allow only users <<<alice>>>, <<<bob>>> and users in the <<<mapreduce>>> group to submit
- jobs to the MapReduce cluster:
-
-----
-<property>
- <name>security.job.submission.protocol.acl</name>
- <value>alice,bob mapreduce</value>
-</property>
-----
-
- Allow only DataNodes running as the users who belong to the group
- datanodes to communicate with the NameNode:
-
-----
-<property>
- <name>security.datanode.protocol.acl</name>
- <value>datanodes</value>
-</property>
-----
-
- Allow any user to talk to the HDFS cluster as a DFSClient:
-
-----
-<property>
- <name>security.client.protocol.acl</name>
- <value>*</value>
-</property>
-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
deleted file mode 100644
index eb9c88a..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/SingleCluster.apt.vm
+++ /dev/null
@@ -1,286 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop ${project.version} - Setting up a Single Node Cluster.
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop - Setting up a Single Node Cluster.
-
-%{toc|section=1|fromDepth=0}
-
-* Purpose
-
- This document describes how to set up and configure a single-node Hadoop
- installation so that you can quickly perform simple operations using Hadoop
- MapReduce and the Hadoop Distributed File System (HDFS).
-
-* Prerequisites
-
-** Supported Platforms
-
- * GNU/Linux is supported as a development and production platform.
- Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
-
- * Windows is also a supported platform but the followings steps
- are for Linux only. To set up Hadoop on Windows, see
- {{{http://wiki.apache.org/hadoop/Hadoop2OnWindows}wiki page}}.
-
-** Required Software
-
- Required software for Linux include:
-
- [[1]] Java\u2122 must be installed. Recommended Java versions are described
- at {{{http://wiki.apache.org/hadoop/HadoopJavaVersions}
- HadoopJavaVersions}}.
-
- [[2]] ssh must be installed and sshd must be running to use the Hadoop
- scripts that manage remote Hadoop daemons if the optional start
- and stop scripts are to be used. Additionally, it is recommmended that
- pdsh also be installed for better ssh resource management.
-
-** Installing Software
-
- If your cluster doesn't have the requisite software you will need to install
- it.
-
- For example on Ubuntu Linux:
-
-----
- $ sudo apt-get install ssh
- $ sudo apt-get install pdsh
-----
-
-* Download
-
- To get a Hadoop distribution, download a recent stable release from one of
- the {{{http://www.apache.org/dyn/closer.cgi/hadoop/common/}
- Apache Download Mirrors}}.
-
-* Prepare to Start the Hadoop Cluster
-
- Unpack the downloaded Hadoop distribution. In the distribution, edit
- the file <<<etc/hadoop/hadoop-env.sh>>> to define some parameters as
- follows:
-
-----
- # set to the root of your Java installation
- export JAVA_HOME=/usr/java/latest
-----
-
- Try the following command:
-
-----
- $ bin/hadoop
-----
-
- This will display the usage documentation for the hadoop script.
-
- Now you are ready to start your Hadoop cluster in one of the three supported
- modes:
-
- * {{{Standalone Operation}Local (Standalone) Mode}}
-
- * {{{Pseudo-Distributed Operation}Pseudo-Distributed Mode}}
-
- * {{{Fully-Distributed Operation}Fully-Distributed Mode}}
-
-* Standalone Operation
-
- By default, Hadoop is configured to run in a non-distributed mode, as a
- single Java process. This is useful for debugging.
-
- The following example copies the unpacked conf directory to use as input
- and then finds and displays every match of the given regular expression.
- Output is written to the given output directory.
-
-----
- $ mkdir input
- $ cp etc/hadoop/*.xml input
- $ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar grep input output 'dfs[a-z.]+'
- $ cat output/*
-----
-
-* Pseudo-Distributed Operation
-
- Hadoop can also be run on a single-node in a pseudo-distributed mode where
- each Hadoop daemon runs in a separate Java process.
-
-** Configuration
-
- Use the following:
-
- etc/hadoop/core-site.xml:
-
-+---+
-<configuration>
- <property>
- <name>fs.defaultFS</name>
- <value>hdfs://localhost:9000</value>
- </property>
-</configuration>
-+---+
-
- etc/hadoop/hdfs-site.xml:
-
-+---+
-<configuration>
- <property>
- <name>dfs.replication</name>
- <value>1</value>
- </property>
-</configuration>
-+---+
-
-** Setup passphraseless ssh
-
- Now check that you can ssh to the localhost without a passphrase:
-
-----
- $ ssh localhost
-----
-
- If you cannot ssh to localhost without a passphrase, execute the
- following commands:
-
-----
- $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
- $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
- $ chmod 0700 ~/.ssh/authorized_keys
-----
-
-** Execution
-
- The following instructions are to run a MapReduce job locally.
- If you want to execute a job on YARN, see {{YARN on Single Node}}.
-
- [[1]] Format the filesystem:
-
-----
- $ bin/hdfs namenode -format
-----
-
- [[2]] Start NameNode daemon and DataNode daemon:
-
-----
- $ sbin/start-dfs.sh
-----
-
- The hadoop daemon log output is written to the <<<${HADOOP_LOG_DIR}>>>
- directory (defaults to <<<${HADOOP_HOME}/logs>>>).
-
- [[3]] Browse the web interface for the NameNode; by default it is
- available at:
-
- * NameNode - <<<http://localhost:50070/>>>
-
- [[4]] Make the HDFS directories required to execute MapReduce jobs:
-
-----
- $ bin/hdfs dfs -mkdir /user
- $ bin/hdfs dfs -mkdir /user/<username>
-----
-
- [[5]] Copy the input files into the distributed filesystem:
-
-----
- $ bin/hdfs dfs -put etc/hadoop input
-----
-
- [[6]] Run some of the examples provided:
-
-----
- $ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar grep input output 'dfs[a-z.]+'
-----
-
- [[7]] Examine the output files:
-
- Copy the output files from the distributed filesystem to the local
- filesystem and examine them:
-
-----
- $ bin/hdfs dfs -get output output
- $ cat output/*
-----
-
- or
-
- View the output files on the distributed filesystem:
-
-----
- $ bin/hdfs dfs -cat output/*
-----
-
- [[8]] When you're done, stop the daemons with:
-
-----
- $ sbin/stop-dfs.sh
-----
-
-** YARN on a Single Node
-
- You can run a MapReduce job on YARN in a pseudo-distributed mode by setting
- a few parameters and running ResourceManager daemon and NodeManager daemon
- in addition.
-
- The following instructions assume that 1. ~ 4. steps of
- {{{Execution}the above instructions}} are already executed.
-
- [[1]] Configure parameters as follows:
-
- <<<etc/hadoop/mapred-site.xml>>>:
-
-+---+
-<configuration>
- <property>
- <name>mapreduce.framework.name</name>
- <value>yarn</value>
- </property>
-</configuration>
-+---+
-
- <<<etc/hadoop/yarn-site.xml>>>:
-
-+---+
-<configuration>
- <property>
- <name>yarn.nodemanager.aux-services</name>
- <value>mapreduce_shuffle</value>
- </property>
-</configuration>
-+---+
-
- [[2]] Start ResourceManager daemon and NodeManager daemon:
-
-----
- $ sbin/start-yarn.sh
-----
-
- [[3]] Browse the web interface for the ResourceManager; by default it is
- available at:
-
- * ResourceManager - <<<http://localhost:8088/>>>
-
- [[4]] Run a MapReduce job.
-
- [[5]] When you're done, stop the daemons with:
-
-----
- $ sbin/stop-yarn.sh
-----
-
-* Fully-Distributed Operation
-
- For information on setting up fully-distributed, non-trivial clusters
- see {{{./ClusterSetup.html}Cluster Setup}}.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm
deleted file mode 100644
index eb0c801..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/SingleNodeSetup.apt.vm
+++ /dev/null
@@ -1,24 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Single Node Setup
- ---
- ---
- ${maven.build.timestamp}
-
-Single Node Setup
-
- This page will be removed in the next major release.
-
- See {{{./SingleCluster.html}Single Cluster Setup}} to set up and configure a
- single-node Hadoop installation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm
deleted file mode 100644
index 78ed9a4..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/Superusers.apt.vm
+++ /dev/null
@@ -1,144 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Proxy user - Superusers Acting On Behalf Of Other Users
- ---
- ---
- ${maven.build.timestamp}
-
-Proxy user - Superusers Acting On Behalf Of Other Users
-
-%{toc|section=1|fromDepth=0}
-
-* Introduction
-
- This document describes how a superuser can submit jobs or access hdfs
- on behalf of another user.
-
-* Use Case
-
- The code example described in the next section is applicable for the
- following use case.
-
- A superuser with username 'super' wants to submit job and access hdfs
- on behalf of a user joe. The superuser has kerberos credentials but
- user joe doesn't have any. The tasks are required to run as user joe
- and any file accesses on namenode are required to be done as user joe.
- It is required that user joe can connect to the namenode or job tracker
- on a connection authenticated with super's kerberos credentials. In
- other words super is impersonating the user joe.
-
- Some products such as Apache Oozie need this.
-
-
-* Code example
-
- In this example super's credentials are used for login and a
- proxy user ugi object is created for joe. The operations are performed
- within the doAs method of this proxy user ugi object.
-
-----
- ...
- //Create ugi for joe. The login user is 'super'.
- UserGroupInformation ugi =
- UserGroupInformation.createProxyUser("joe", UserGroupInformation.getLoginUser());
- ugi.doAs(new PrivilegedExceptionAction<Void>() {
- public Void run() throws Exception {
- //Submit a job
- JobClient jc = new JobClient(conf);
- jc.submitJob(conf);
- //OR access hdfs
- FileSystem fs = FileSystem.get(conf);
- fs.mkdir(someFilePath);
- }
- }
-----
-
-* Configurations
-
- You can configure proxy user using properties
- <<<hadoop.proxyuser.${superuser}.hosts>>> along with either or both of
- <<<hadoop.proxyuser.${superuser}.groups>>>
- and <<<hadoop.proxyuser.${superuser}.users>>>.
-
- By specifying as below in core-site.xml,
- the superuser named <<<super>>> can connect
- only from <<<host1>>> and <<<host2>>>
- to impersonate a user belonging to <<<group1>>> and <<<group2>>>.
-
-----
- <property>
- <name>hadoop.proxyuser.super.hosts</name>
- <value>host1,host2</value>
- </property>
- <property>
- <name>hadoop.proxyuser.super.groups</name>
- <value>group1,group2</value>
- </property>
-
-----
-
- If these configurations are not present, impersonation will not be
- allowed and connection will fail.
-
- If more lax security is preferred, the wildcard value * may be used to
- allow impersonation from any host or of any user.
- For example, by specifying as below in core-site.xml,
- user named <<<oozie>>> accessing from any host
- can impersonate any user belonging to any group.
-
-----
- <property>
- <name>hadoop.proxyuser.oozie.hosts</name>
- <value>*</value>
- </property>
- <property>
- <name>hadoop.proxyuser.oozie.groups</name>
- <value>*</value>
- </property>
-----
-
- The <<<hadoop.proxyuser.${superuser}.hosts>>> accepts list of ip addresses,
- ip address ranges in CIDR format and/or host names.
- For example, by specifying as below,
- user named <<<super>>> accessing from hosts in the range
- <<<10.222.0.0-15>>> and <<<10.113.221.221>>> can impersonate
- <<<user1>>> and <<<user2>>>.
-
-----
- <property>
- <name>hadoop.proxyuser.super.hosts</name>
- <value>10.222.0.0/16,10.113.221.221</value>
- </property>
- <property>
- <name>hadoop.proxyuser.super.users</name>
- <value>user1,user2</value>
- </property>
-----
-
-
-* Caveats
-
- If the cluster is running in {{{./SecureMode.html}Secure Mode}},
- the superuser must have kerberos credentials to be able to impersonate
- another user.
-
- It cannot use delegation tokens for this feature. It
- would be wrong if superuser adds its own delegation token to the proxy
- user ugi, as it will allow the proxy user to connect to the service
- with the privileges of the superuser.
-
- However, if the superuser does want to give a delegation token to joe,
- it must first impersonate joe and get a delegation token for joe, in
- the same way as the code example above, and add it to the ugi of joe.
- In this way the delegation token will have the owner as joe.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm
deleted file mode 100644
index c51037b..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/Tracing.apt.vm
+++ /dev/null
@@ -1,233 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Distributed File System-${project.version} - Enabling Dapper-like Tracing
- ---
- ---
- ${maven.build.timestamp}
-
-Enabling Dapper-like Tracing in Hadoop
-
-%{toc|section=1|fromDepth=0}
-
-* {Dapper-like Tracing in Hadoop}
-
-** HTrace
-
- {{{https://issues.apache.org/jira/browse/HDFS-5274}HDFS-5274}}
- added support for tracing requests through HDFS,
- using the open source tracing library, {{{https://git-wip-us.apache.org/repos/asf/incubator-htrace.git}Apache HTrace}}.
- Setting up tracing is quite simple, however it requires some very minor changes to your client code.
-
-** Samplers
- Configure the samplers in <<<core-site.xml>>> property: <<<hadoop.htrace.sampler>>>.
- The value can be NeverSampler, AlwaysSampler or ProbabilitySampler. NeverSampler: HTrace is OFF
- for all spans; AlwaysSampler: HTrace is ON for all spans; ProbabilitySampler: HTrace is ON for
- some percentage% of top-level spans.
-
-+----
- <property>
- <name>hadoop.htrace.sampler</name>
- <value>NeverSampler</value>
- </property>
-+----
-
-** SpanReceivers
-
- The tracing system works by collecting information in structs called 'Spans'.
- It is up to you to choose how you want to receive this information
- by implementing the SpanReceiver interface, which defines one method:
-
-+----
-public void receiveSpan(Span span);
-+----
-
- Configure what SpanReceivers you'd like to use
- by putting a comma separated list of the fully-qualified class name of
- classes implementing SpanReceiver
- in <<<core-site.xml>>> property: <<<hadoop.htrace.spanreceiver.classes>>>.
-
-+----
- <property>
- <name>hadoop.htrace.spanreceiver.classes</name>
- <value>org.apache.htrace.impl.LocalFileSpanReceiver</value>
- </property>
- <property>
- <name>hadoop.htrace.local-file-span-receiver.path</name>
- <value>/var/log/hadoop/htrace.out</value>
- </property>
-+----
-
- You can omit package name prefix if you use span receiver bundled with HTrace.
-
-+----
- <property>
- <name>hadoop.htrace.spanreceiver.classes</name>
- <value>LocalFileSpanReceiver</value>
- </property>
-+----
-
-
-
-** Setting up ZipkinSpanReceiver
-
- Instead of implementing SpanReceiver by yourself,
- you can use <<<ZipkinSpanReceiver>>> which uses
- {{{https://github.com/twitter/zipkin}Zipkin}}
- for collecting and displaying tracing data.
-
- In order to use <<<ZipkinSpanReceiver>>>,
- you need to download and setup {{{https://github.com/twitter/zipkin}Zipkin}} first.
-
- you also need to add the jar of <<<htrace-zipkin>>> to the classpath of Hadoop on each node.
- Here is example setup procedure.
-
-+----
- $ git clone https://github.com/cloudera/htrace
- $ cd htrace/htrace-zipkin
- $ mvn compile assembly:single
- $ cp target/htrace-zipkin-*-jar-with-dependencies.jar $HADOOP_HOME/share/hadoop/common/lib/
-+----
-
- The sample configuration for <<<ZipkinSpanReceiver>>> is shown below.
- By adding these to <<<core-site.xml>>> of NameNode and DataNodes,
- <<<ZipkinSpanReceiver>>> is initialized on the startup.
- You also need this configuration on the client node in addition to the servers.
-
-+----
- <property>
- <name>hadoop.htrace.spanreceiver.classes</name>
- <value>ZipkinSpanReceiver</value>
- </property>
- <property>
- <name>hadoop.htrace.zipkin.collector-hostname</name>
- <value>192.168.1.2</value>
- </property>
- <property>
- <name>hadoop.htrace.zipkin.collector-port</name>
- <value>9410</value>
- </property>
-+----
-
-
-** Dynamic update of tracing configuration
-
- You can use <<<hadoop trace>>> command to see and update the tracing configuration of each servers.
- You must specify IPC server address of namenode or datanode by <<<-host>>> option.
- You need to run the command against all servers if you want to update the configuration of all servers.
-
- <<<hadoop trace -list>>> shows list of loaded span receivers associated with the id.
-
-+----
- $ hadoop trace -list -host 192.168.56.2:9000
- ID CLASS
- 1 org.apache.htrace.impl.LocalFileSpanReceiver
-
- $ hadoop trace -list -host 192.168.56.2:50020
- ID CLASS
- 1 org.apache.htrace.impl.LocalFileSpanReceiver
-+----
-
- <<<hadoop trace -remove>>> removes span receiver from server.
- <<<-remove>>> options takes id of span receiver as argument.
-
-+----
- $ hadoop trace -remove 1 -host 192.168.56.2:9000
- Removed trace span receiver 1
-+----
-
- <<<hadoop trace -add>>> adds span receiver to server.
- You need to specify the class name of span receiver as argument of <<<-class>>> option.
- You can specify the configuration associated with span receiver by <<<-Ckey=value>>> options.
-
-+----
- $ hadoop trace -add -class LocalFileSpanReceiver -Chadoop.htrace.local-file-span-receiver.path=/tmp/htrace.out -host 192.168.56.2:9000
- Added trace span receiver 2 with configuration hadoop.htrace.local-file-span-receiver.path = /tmp/htrace.out
-
- $ hadoop trace -list -host 192.168.56.2:9000
- ID CLASS
- 2 org.apache.htrace.impl.LocalFileSpanReceiver
-+----
-
-
-** Starting tracing spans by HTrace API
-
- In order to trace,
- you will need to wrap the traced logic with <<tracing span>> as shown below.
- When there is running tracing spans,
- the tracing information is propagated to servers along with RPC requests.
-
- In addition, you need to initialize <<<SpanReceiver>>> once per process.
-
-+----
-import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.tracing.SpanReceiverHost;
-import org.apache.htrace.Sampler;
-import org.apache.htrace.Trace;
-import org.apache.htrace.TraceScope;
-
-...
-
- SpanReceiverHost.getInstance(new HdfsConfiguration());
-
-...
-
- TraceScope ts = Trace.startSpan("Gets", Sampler.ALWAYS);
- try {
- ... // traced logic
- } finally {
- if (ts != null) ts.close();
- }
-+----
-
-** Sample code for tracing
-
- The <<<TracingFsShell.java>>> shown below is the wrapper of FsShell
- which start tracing span before invoking HDFS shell command.
-
-+----
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FsShell;
-import org.apache.hadoop.tracing.SpanReceiverHost;
-import org.apache.hadoop.util.ToolRunner;
-import org.apache.htrace.Sampler;
-import org.apache.htrace.Trace;
-import org.apache.htrace.TraceScope;
-
-public class TracingFsShell {
- public static void main(String argv[]) throws Exception {
- Configuration conf = new Configuration();
- FsShell shell = new FsShell();
- conf.setQuietMode(false);
- shell.setConf(conf);
- SpanReceiverHost.getInstance(conf);
- int res = 0;
- TraceScope ts = null;
- try {
- ts = Trace.startSpan("FsShell", Sampler.ALWAYS);
- res = ToolRunner.run(shell, argv);
- } finally {
- shell.close();
- if (ts != null) ts.close();
- }
- System.exit(res);
- }
-}
-+----
-
- You can compile and execute this code as shown below.
-
-+----
-$ javac -cp `hadoop classpath` TracingFsShell.java
-$ java -cp .:`hadoop classpath` TracingFsShell -ls /
-+----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
new file mode 100644
index 0000000..25fecda
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CLIMiniCluster.md.vm
@@ -0,0 +1,68 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop: CLI MiniCluster.
+========================
+
+* [Hadoop: CLI MiniCluster.](#Hadoop:_CLI_MiniCluster.)
+ * [Purpose](#Purpose)
+ * [Hadoop Tarball](#Hadoop_Tarball)
+ * [Running the MiniCluster](#Running_the_MiniCluster)
+
+Purpose
+-------
+
+Using the CLI MiniCluster, users can simply start and stop a single-node Hadoop cluster with a single command, and without the need to set any environment variables or manage configuration files. The CLI MiniCluster starts both a `YARN`/`MapReduce` & `HDFS` clusters.
+
+This is useful for cases where users want to quickly experiment with a real Hadoop cluster or test non-Java programs that rely on significant Hadoop functionality.
+
+Hadoop Tarball
+--------------
+
+You should be able to obtain the Hadoop tarball from the release. Also, you can directly create a tarball from the source:
+
+ $ mvn clean install -DskipTests
+ $ mvn package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip
+
+**NOTE:** You will need [protoc 2.5.0](http://code.google.com/p/protobuf/) installed.
+
+The tarball should be available in `hadoop-dist/target/` directory.
+
+Running the MiniCluster
+-----------------------
+
+From inside the root directory of the extracted tarball, you can start the CLI MiniCluster using the following command:
+
+ $ bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT
+
+In the example command above, `RM_PORT` and `JHS_PORT` should be replaced by the user's choice of these port numbers. If not specified, random free ports will be used.
+
+There are a number of command line arguments that the users can use to control which services to start, and to pass other configuration properties. The available command line arguments:
+
+ $ -D <property=value> Options to pass into configuration object
+ $ -datanodes <arg> How many datanodes to start (default 1)
+ $ -format Format the DFS (default false)
+ $ -help Prints option help.
+ $ -jhsport <arg> JobHistoryServer port (default 0--we choose)
+ $ -namenode <arg> URL of the namenode (default is either the DFS
+ $ cluster or a temporary dir)
+ $ -nnport <arg> NameNode port (default 0--we choose)
+ $ -nodemanagers <arg> How many nodemanagers to start (default 1)
+ $ -nodfs Don't start a mini DFS cluster
+ $ -nomr Don't start a mini MR cluster
+ $ -rmport <arg> ResourceManager port (default 0--we choose)
+ $ -writeConfig <path> Save configuration to this XML file.
+ $ -writeDetails <path> Write basic information to this JSON file.
+
+To display this full list of available arguments, the user can pass the `-help` argument to the above command.
[27/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
deleted file mode 100644
index 02ff28b..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/Metrics.apt.vm
+++ /dev/null
@@ -1,879 +0,0 @@
-~~ Licensed to the Apache Software Foundation (ASF) under one or more
-~~ contributor license agreements. See the NOTICE file distributed with
-~~ this work for additional information regarding copyright ownership.
-~~ The ASF licenses this file to You under the Apache License, Version 2.0
-~~ (the "License"); you may not use this file except in compliance with
-~~ the License. You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Metrics Guide
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc}
-
-Overview
-
- Metrics are statistical information exposed by Hadoop daemons,
- used for monitoring, performance tuning and debug.
- There are many metrics available by default
- and they are very useful for troubleshooting.
- This page shows the details of the available metrics.
-
- Each section describes each context into which metrics are grouped.
-
- The documentation of Metrics 2.0 framework is
- {{{../../api/org/apache/hadoop/metrics2/package-summary.html}here}}.
-
-jvm context
-
-* JvmMetrics
-
- Each metrics record contains tags such as ProcessName, SessionID
- and Hostname as additional information along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<MemNonHeapUsedM>>> | Current non-heap memory used in MB
-*-------------------------------------+--------------------------------------+
-|<<<MemNonHeapCommittedM>>> | Current non-heap memory committed in MB
-*-------------------------------------+--------------------------------------+
-|<<<MemNonHeapMaxM>>> | Max non-heap memory size in MB
-*-------------------------------------+--------------------------------------+
-|<<<MemHeapUsedM>>> | Current heap memory used in MB
-*-------------------------------------+--------------------------------------+
-|<<<MemHeapCommittedM>>> | Current heap memory committed in MB
-*-------------------------------------+--------------------------------------+
-|<<<MemHeapMaxM>>> | Max heap memory size in MB
-*-------------------------------------+--------------------------------------+
-|<<<MemMaxM>>> | Max memory size in MB
-*-------------------------------------+--------------------------------------+
-|<<<ThreadsNew>>> | Current number of NEW threads
-*-------------------------------------+--------------------------------------+
-|<<<ThreadsRunnable>>> | Current number of RUNNABLE threads
-*-------------------------------------+--------------------------------------+
-|<<<ThreadsBlocked>>> | Current number of BLOCKED threads
-*-------------------------------------+--------------------------------------+
-|<<<ThreadsWaiting>>> | Current number of WAITING threads
-*-------------------------------------+--------------------------------------+
-|<<<ThreadsTimedWaiting>>> | Current number of TIMED_WAITING threads
-*-------------------------------------+--------------------------------------+
-|<<<ThreadsTerminated>>> | Current number of TERMINATED threads
-*-------------------------------------+--------------------------------------+
-|<<<GcInfo>>> | Total GC count and GC time in msec, grouped by the kind of GC. \
- | ex.) GcCountPS Scavenge=6, GCTimeMillisPS Scavenge=40,
- | GCCountPS MarkSweep=0, GCTimeMillisPS MarkSweep=0
-*-------------------------------------+--------------------------------------+
-|<<<GcCount>>> | Total GC count
-*-------------------------------------+--------------------------------------+
-|<<<GcTimeMillis>>> | Total GC time in msec
-*-------------------------------------+--------------------------------------+
-|<<<LogFatal>>> | Total number of FATAL logs
-*-------------------------------------+--------------------------------------+
-|<<<LogError>>> | Total number of ERROR logs
-*-------------------------------------+--------------------------------------+
-|<<<LogWarn>>> | Total number of WARN logs
-*-------------------------------------+--------------------------------------+
-|<<<LogInfo>>> | Total number of INFO logs
-*-------------------------------------+--------------------------------------+
-|<<<GcNumWarnThresholdExceeded>>> | Number of times that the GC warn
- | threshold is exceeded
-*-------------------------------------+--------------------------------------+
-|<<<GcNumInfoThresholdExceeded>>> | Number of times that the GC info
- | threshold is exceeded
-*-------------------------------------+--------------------------------------+
-|<<<GcTotalExtraSleepTime>>> | Total GC extra sleep time in msec
-*-------------------------------------+--------------------------------------+
-
-rpc context
-
-* rpc
-
- Each metrics record contains tags such as Hostname
- and port (number to which server is bound)
- as additional information along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<ReceivedBytes>>> | Total number of received bytes
-*-------------------------------------+--------------------------------------+
-|<<<SentBytes>>> | Total number of sent bytes
-*-------------------------------------+--------------------------------------+
-|<<<RpcQueueTimeNumOps>>> | Total number of RPC calls
-*-------------------------------------+--------------------------------------+
-|<<<RpcQueueTimeAvgTime>>> | Average queue time in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<RpcProcessingTimeNumOps>>> | Total number of RPC calls (same to
- | RpcQueueTimeNumOps)
-*-------------------------------------+--------------------------------------+
-|<<<RpcProcessingAvgTime>>> | Average Processing time in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<RpcAuthenticationFailures>>> | Total number of authentication failures
-*-------------------------------------+--------------------------------------+
-|<<<RpcAuthenticationSuccesses>>> | Total number of authentication successes
-*-------------------------------------+--------------------------------------+
-|<<<RpcAuthorizationFailures>>> | Total number of authorization failures
-*-------------------------------------+--------------------------------------+
-|<<<RpcAuthorizationSuccesses>>> | Total number of authorization successes
-*-------------------------------------+--------------------------------------+
-|<<<NumOpenConnections>>> | Current number of open connections
-*-------------------------------------+--------------------------------------+
-|<<<CallQueueLength>>> | Current length of the call queue
-*-------------------------------------+--------------------------------------+
-|<<<rpcQueueTime>>><num><<<sNumOps>>> | Shows total number of RPC calls
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcQueueTime>>><num><<<s50thPercentileLatency>>> |
-| | Shows the 50th percentile of RPC queue time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcQueueTime>>><num><<<s75thPercentileLatency>>> |
-| | Shows the 75th percentile of RPC queue time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcQueueTime>>><num><<<s90thPercentileLatency>>> |
-| | Shows the 90th percentile of RPC queue time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcQueueTime>>><num><<<s95thPercentileLatency>>> |
-| | Shows the 95th percentile of RPC queue time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcQueueTime>>><num><<<s99thPercentileLatency>>> |
-| | Shows the 99th percentile of RPC queue time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcProcessingTime>>><num><<<sNumOps>>> | Shows total number of RPC calls
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcProcessingTime>>><num><<<s50thPercentileLatency>>> |
-| | Shows the 50th percentile of RPC processing time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcProcessingTime>>><num><<<s75thPercentileLatency>>> |
-| | Shows the 75th percentile of RPC processing time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcProcessingTime>>><num><<<s90thPercentileLatency>>> |
-| | Shows the 90th percentile of RPC processing time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcProcessingTime>>><num><<<s95thPercentileLatency>>> |
-| | Shows the 95th percentile of RPC processing time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<rpcProcessingTime>>><num><<<s99thPercentileLatency>>> |
-| | Shows the 99th percentile of RPC processing time in milliseconds
-| | (<num> seconds granularity) if <<<rpc.metrics.quantile.enable>>> is set to
-| | true. <num> is specified by <<<rpc.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-
-* RetryCache/NameNodeRetryCache
-
- RetryCache metrics is useful to monitor NameNode fail-over.
- Each metrics record contains Hostname tag.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<CacheHit>>> | Total number of RetryCache hit
-*-------------------------------------+--------------------------------------+
-|<<<CacheCleared>>> | Total number of RetryCache cleared
-*-------------------------------------+--------------------------------------+
-|<<<CacheUpdated>>> | Total number of RetryCache updated
-*-------------------------------------+--------------------------------------+
-
-rpcdetailed context
-
- Metrics of rpcdetailed context are exposed in unified manner by RPC
- layer. Two metrics are exposed for each RPC based on its name.
- Metrics named "(RPC method name)NumOps" indicates total number of
- method calls, and metrics named "(RPC method name)AvgTime" shows
- average turn around time for method calls in milliseconds.
-
-* rpcdetailed
-
- Each metrics record contains tags such as Hostname
- and port (number to which server is bound)
- as additional information along with metrics.
-
- The Metrics about RPCs which is not called are not included
- in metrics record.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<methodname><<<NumOps>>> | Total number of the times the method is called
-*-------------------------------------+--------------------------------------+
-|<methodname><<<AvgTime>>> | Average turn around time of the method in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-
-dfs context
-
-* namenode
-
- Each metrics record contains tags such as ProcessName, SessionId,
- and Hostname as additional information along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<CreateFileOps>>> | Total number of files created
-*-------------------------------------+--------------------------------------+
-|<<<FilesCreated>>> | Total number of files and directories created by create
- | or mkdir operations
-*-------------------------------------+--------------------------------------+
-|<<<FilesAppended>>> | Total number of files appended
-*-------------------------------------+--------------------------------------+
-|<<<GetBlockLocations>>> | Total number of getBlockLocations operations
-*-------------------------------------+--------------------------------------+
-|<<<FilesRenamed>>> | Total number of rename <<operations>> (NOT number of
- | files/dirs renamed)
-*-------------------------------------+--------------------------------------+
-|<<<GetListingOps>>> | Total number of directory listing operations
-*-------------------------------------+--------------------------------------+
-|<<<DeleteFileOps>>> | Total number of delete operations
-*-------------------------------------+--------------------------------------+
-|<<<FilesDeleted>>> | Total number of files and directories deleted by delete
- | or rename operations
-*-------------------------------------+--------------------------------------+
-|<<<FileInfoOps>>> | Total number of getFileInfo and getLinkFileInfo
- | operations
-*-------------------------------------+--------------------------------------+
-|<<<AddBlockOps>>> | Total number of addBlock operations succeeded
-*-------------------------------------+--------------------------------------+
-|<<<GetAdditionalDatanodeOps>>> | Total number of getAdditionalDatanode
- | operations
-*-------------------------------------+--------------------------------------+
-|<<<CreateSymlinkOps>>> | Total number of createSymlink operations
-*-------------------------------------+--------------------------------------+
-|<<<GetLinkTargetOps>>> | Total number of getLinkTarget operations
-*-------------------------------------+--------------------------------------+
-|<<<FilesInGetListingOps>>> | Total number of files and directories listed by
- | directory listing operations
-*-------------------------------------+--------------------------------------+
-|<<<AllowSnapshotOps>>> | Total number of allowSnapshot operations
-*-------------------------------------+--------------------------------------+
-|<<<DisallowSnapshotOps>>> | Total number of disallowSnapshot operations
-*-------------------------------------+--------------------------------------+
-|<<<CreateSnapshotOps>>> | Total number of createSnapshot operations
-*-------------------------------------+--------------------------------------+
-|<<<DeleteSnapshotOps>>> | Total number of deleteSnapshot operations
-*-------------------------------------+--------------------------------------+
-|<<<RenameSnapshotOps>>> | Total number of renameSnapshot operations
-*-------------------------------------+--------------------------------------+
-|<<<ListSnapshottableDirOps>>> | Total number of snapshottableDirectoryStatus
- | operations
-*-------------------------------------+--------------------------------------+
-|<<<SnapshotDiffReportOps>>> | Total number of getSnapshotDiffReport
- | operations
-*-------------------------------------+--------------------------------------+
-|<<<TransactionsNumOps>>> | Total number of Journal transactions
-*-------------------------------------+--------------------------------------+
-|<<<TransactionsAvgTime>>> | Average time of Journal transactions in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<SyncsNumOps>>> | Total number of Journal syncs
-*-------------------------------------+--------------------------------------+
-|<<<SyncsAvgTime>>> | Average time of Journal syncs in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<TransactionsBatchedInSync>>> | Total number of Journal transactions batched
- | in sync
-*-------------------------------------+--------------------------------------+
-|<<<BlockReportNumOps>>> | Total number of processing block reports from
- | DataNode
-*-------------------------------------+--------------------------------------+
-|<<<BlockReportAvgTime>>> | Average time of processing block reports in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<CacheReportNumOps>>> | Total number of processing cache reports from
- | DataNode
-*-------------------------------------+--------------------------------------+
-|<<<CacheReportAvgTime>>> | Average time of processing cache reports in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<SafeModeTime>>> | The interval between FSNameSystem starts and the last
- | time safemode leaves in milliseconds. \
- | (sometimes not equal to the time in SafeMode,
- | see {{{https://issues.apache.org/jira/browse/HDFS-5156}HDFS-5156}})
-*-------------------------------------+--------------------------------------+
-|<<<FsImageLoadTime>>> | Time loading FS Image at startup in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<FsImageLoadTime>>> | Time loading FS Image at startup in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<GetEditNumOps>>> | Total number of edits downloads from SecondaryNameNode
-*-------------------------------------+--------------------------------------+
-|<<<GetEditAvgTime>>> | Average edits download time in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<GetImageNumOps>>> |Total number of fsimage downloads from SecondaryNameNode
-*-------------------------------------+--------------------------------------+
-|<<<GetImageAvgTime>>> | Average fsimage download time in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<PutImageNumOps>>> | Total number of fsimage uploads to SecondaryNameNode
-*-------------------------------------+--------------------------------------+
-|<<<PutImageAvgTime>>> | Average fsimage upload time in milliseconds
-*-------------------------------------+--------------------------------------+
-
-* FSNamesystem
-
- Each metrics record contains tags such as HAState and Hostname
- as additional information along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<MissingBlocks>>> | Current number of missing blocks
-*-------------------------------------+--------------------------------------+
-|<<<ExpiredHeartbeats>>> | Total number of expired heartbeats
-*-------------------------------------+--------------------------------------+
-|<<<TransactionsSinceLastCheckpoint>>> | Total number of transactions since
- | last checkpoint
-*-------------------------------------+--------------------------------------+
-|<<<TransactionsSinceLastLogRoll>>> | Total number of transactions since last
- | edit log roll
-*-------------------------------------+--------------------------------------+
-|<<<LastWrittenTransactionId>>> | Last transaction ID written to the edit log
-*-------------------------------------+--------------------------------------+
-|<<<LastCheckpointTime>>> | Time in milliseconds since epoch of last checkpoint
-*-------------------------------------+--------------------------------------+
-|<<<CapacityTotal>>> | Current raw capacity of DataNodes in bytes
-*-------------------------------------+--------------------------------------+
-|<<<CapacityTotalGB>>> | Current raw capacity of DataNodes in GB
-*-------------------------------------+--------------------------------------+
-|<<<CapacityUsed>>> | Current used capacity across all DataNodes in bytes
-*-------------------------------------+--------------------------------------+
-|<<<CapacityUsedGB>>> | Current used capacity across all DataNodes in GB
-*-------------------------------------+--------------------------------------+
-|<<<CapacityRemaining>>> | Current remaining capacity in bytes
-*-------------------------------------+--------------------------------------+
-|<<<CapacityRemainingGB>>> | Current remaining capacity in GB
-*-------------------------------------+--------------------------------------+
-|<<<CapacityUsedNonDFS>>> | Current space used by DataNodes for non DFS
- | purposes in bytes
-*-------------------------------------+--------------------------------------+
-|<<<TotalLoad>>> | Current number of connections
-*-------------------------------------+--------------------------------------+
-|<<<SnapshottableDirectories>>> | Current number of snapshottable directories
-*-------------------------------------+--------------------------------------+
-|<<<Snapshots>>> | Current number of snapshots
-*-------------------------------------+--------------------------------------+
-|<<<BlocksTotal>>> | Current number of allocated blocks in the system
-*-------------------------------------+--------------------------------------+
-|<<<FilesTotal>>> | Current number of files and directories
-*-------------------------------------+--------------------------------------+
-|<<<PendingReplicationBlocks>>> | Current number of blocks pending to be
- | replicated
-*-------------------------------------+--------------------------------------+
-|<<<UnderReplicatedBlocks>>> | Current number of blocks under replicated
-*-------------------------------------+--------------------------------------+
-|<<<CorruptBlocks>>> | Current number of blocks with corrupt replicas.
-*-------------------------------------+--------------------------------------+
-|<<<ScheduledReplicationBlocks>>> | Current number of blocks scheduled for
- | replications
-*-------------------------------------+--------------------------------------+
-|<<<PendingDeletionBlocks>>> | Current number of blocks pending deletion
-*-------------------------------------+--------------------------------------+
-|<<<ExcessBlocks>>> | Current number of excess blocks
-*-------------------------------------+--------------------------------------+
-|<<<PostponedMisreplicatedBlocks>>> | (HA-only) Current number of blocks
- | postponed to replicate
-*-------------------------------------+--------------------------------------+
-|<<<PendingDataNodeMessageCourt>>> | (HA-only) Current number of pending
- | block-related messages for later
- | processing in the standby NameNode
-*-------------------------------------+--------------------------------------+
-|<<<MillisSinceLastLoadedEdits>>> | (HA-only) Time in milliseconds since the
- | last time standby NameNode load edit log.
- | In active NameNode, set to 0
-*-------------------------------------+--------------------------------------+
-|<<<BlockCapacity>>> | Current number of block capacity
-*-------------------------------------+--------------------------------------+
-|<<<StaleDataNodes>>> | Current number of DataNodes marked stale due to delayed
- | heartbeat
-*-------------------------------------+--------------------------------------+
-|<<<TotalFiles>>> |Current number of files and directories (same as FilesTotal)
-*-------------------------------------+--------------------------------------+
-
-* JournalNode
-
- The server-side metrics for a journal from the JournalNode's perspective.
- Each metrics record contains Hostname tag as additional information
- along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<Syncs60sNumOps>>> | Number of sync operations (1 minute granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs60s50thPercentileLatencyMicros>>> | The 50th percentile of sync
-| | latency in microseconds (1 minute granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs60s75thPercentileLatencyMicros>>> | The 75th percentile of sync
-| | latency in microseconds (1 minute granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs60s90thPercentileLatencyMicros>>> | The 90th percentile of sync
-| | latency in microseconds (1 minute granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs60s95thPercentileLatencyMicros>>> | The 95th percentile of sync
-| | latency in microseconds (1 minute granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs60s99thPercentileLatencyMicros>>> | The 99th percentile of sync
-| | latency in microseconds (1 minute granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs300sNumOps>>> | Number of sync operations (5 minutes granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs300s50thPercentileLatencyMicros>>> | The 50th percentile of sync
-| | latency in microseconds (5 minutes granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs300s75thPercentileLatencyMicros>>> | The 75th percentile of sync
-| | latency in microseconds (5 minutes granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs300s90thPercentileLatencyMicros>>> | The 90th percentile of sync
-| | latency in microseconds (5 minutes granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs300s95thPercentileLatencyMicros>>> | The 95th percentile of sync
-| | latency in microseconds (5 minutes granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs300s99thPercentileLatencyMicros>>> | The 99th percentile of sync
-| | latency in microseconds (5 minutes granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs3600sNumOps>>> | Number of sync operations (1 hour granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs3600s50thPercentileLatencyMicros>>> | The 50th percentile of sync
-| | latency in microseconds (1 hour granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs3600s75thPercentileLatencyMicros>>> | The 75th percentile of sync
-| | latency in microseconds (1 hour granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs3600s90thPercentileLatencyMicros>>> | The 90th percentile of sync
-| | latency in microseconds (1 hour granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs3600s95thPercentileLatencyMicros>>> | The 95th percentile of sync
-| | latency in microseconds (1 hour granularity)
-*-------------------------------------+--------------------------------------+
-|<<<Syncs3600s99thPercentileLatencyMicros>>> | The 99th percentile of sync
-| | latency in microseconds (1 hour granularity)
-*-------------------------------------+--------------------------------------+
-|<<<BatchesWritten>>> | Total number of batches written since startup
-*-------------------------------------+--------------------------------------+
-|<<<TxnsWritten>>> | Total number of transactions written since startup
-*-------------------------------------+--------------------------------------+
-|<<<BytesWritten>>> | Total number of bytes written since startup
-*-------------------------------------+--------------------------------------+
-|<<<BatchesWrittenWhileLagging>>> | Total number of batches written where this
-| | node was lagging
-*-------------------------------------+--------------------------------------+
-|<<<LastWriterEpoch>>> | Current writer's epoch number
-*-------------------------------------+--------------------------------------+
-|<<<CurrentLagTxns>>> | The number of transactions that this JournalNode is
-| | lagging
-*-------------------------------------+--------------------------------------+
-|<<<LastWrittenTxId>>> | The highest transaction id stored on this JournalNode
-*-------------------------------------+--------------------------------------+
-|<<<LastPromisedEpoch>>> | The last epoch number which this node has promised
-| | not to accept any lower epoch, or 0 if no promises have been made
-*-------------------------------------+--------------------------------------+
-
-* datanode
-
- Each metrics record contains tags such as SessionId and Hostname
- as additional information along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<BytesWritten>>> | Total number of bytes written to DataNode
-*-------------------------------------+--------------------------------------+
-|<<<BytesRead>>> | Total number of bytes read from DataNode
-*-------------------------------------+--------------------------------------+
-|<<<BlocksWritten>>> | Total number of blocks written to DataNode
-*-------------------------------------+--------------------------------------+
-|<<<BlocksRead>>> | Total number of blocks read from DataNode
-*-------------------------------------+--------------------------------------+
-|<<<BlocksReplicated>>> | Total number of blocks replicated
-*-------------------------------------+--------------------------------------+
-|<<<BlocksRemoved>>> | Total number of blocks removed
-*-------------------------------------+--------------------------------------+
-|<<<BlocksVerified>>> | Total number of blocks verified
-*-------------------------------------+--------------------------------------+
-|<<<BlockVerificationFailures>>> | Total number of verifications failures
-*-------------------------------------+--------------------------------------+
-|<<<BlocksCached>>> | Total number of blocks cached
-*-------------------------------------+--------------------------------------+
-|<<<BlocksUncached>>> | Total number of blocks uncached
-*-------------------------------------+--------------------------------------+
-|<<<ReadsFromLocalClient>>> | Total number of read operations from local client
-*-------------------------------------+--------------------------------------+
-|<<<ReadsFromRemoteClient>>> | Total number of read operations from remote
- | client
-*-------------------------------------+--------------------------------------+
-|<<<WritesFromLocalClient>>> | Total number of write operations from local
- | client
-*-------------------------------------+--------------------------------------+
-|<<<WritesFromRemoteClient>>> | Total number of write operations from remote
- | client
-*-------------------------------------+--------------------------------------+
-|<<<BlocksGetLocalPathInfo>>> | Total number of operations to get local path
- | names of blocks
-*-------------------------------------+--------------------------------------+
-|<<<FsyncCount>>> | Total number of fsync
-*-------------------------------------+--------------------------------------+
-|<<<VolumeFailures>>> | Total number of volume failures occurred
-*-------------------------------------+--------------------------------------+
-|<<<ReadBlockOpNumOps>>> | Total number of read operations
-*-------------------------------------+--------------------------------------+
-|<<<ReadBlockOpAvgTime>>> | Average time of read operations in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<WriteBlockOpNumOps>>> | Total number of write operations
-*-------------------------------------+--------------------------------------+
-|<<<WriteBlockOpAvgTime>>> | Average time of write operations in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<BlockChecksumOpNumOps>>> | Total number of blockChecksum operations
-*-------------------------------------+--------------------------------------+
-|<<<BlockChecksumOpAvgTime>>> | Average time of blockChecksum operations in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<CopyBlockOpNumOps>>> | Total number of block copy operations
-*-------------------------------------+--------------------------------------+
-|<<<CopyBlockOpAvgTime>>> | Average time of block copy operations in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<ReplaceBlockOpNumOps>>> | Total number of block replace operations
-*-------------------------------------+--------------------------------------+
-|<<<ReplaceBlockOpAvgTime>>> | Average time of block replace operations in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<HeartbeatsNumOps>>> | Total number of heartbeats
-*-------------------------------------+--------------------------------------+
-|<<<HeartbeatsAvgTime>>> | Average heartbeat time in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<BlockReportsNumOps>>> | Total number of block report operations
-*-------------------------------------+--------------------------------------+
-|<<<BlockReportsAvgTime>>> | Average time of block report operations in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<CacheReportsNumOps>>> | Total number of cache report operations
-*-------------------------------------+--------------------------------------+
-|<<<CacheReportsAvgTime>>> | Average time of cache report operations in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<PacketAckRoundTripTimeNanosNumOps>>> | Total number of ack round trip
-*-------------------------------------+--------------------------------------+
-|<<<PacketAckRoundTripTimeNanosAvgTime>>> | Average time from ack send to
-| | receive minus the downstream ack time in nanoseconds
-*-------------------------------------+--------------------------------------+
-|<<<FlushNanosNumOps>>> | Total number of flushes
-*-------------------------------------+--------------------------------------+
-|<<<FlushNanosAvgTime>>> | Average flush time in nanoseconds
-*-------------------------------------+--------------------------------------+
-|<<<FsyncNanosNumOps>>> | Total number of fsync
-*-------------------------------------+--------------------------------------+
-|<<<FsyncNanosAvgTime>>> | Average fsync time in nanoseconds
-*-------------------------------------+--------------------------------------+
-|<<<SendDataPacketBlockedOnNetworkNanosNumOps>>> | Total number of sending
- | packets
-*-------------------------------------+--------------------------------------+
-|<<<SendDataPacketBlockedOnNetworkNanosAvgTime>>> | Average waiting time of
-| | sending packets in nanoseconds
-*-------------------------------------+--------------------------------------+
-|<<<SendDataPacketTransferNanosNumOps>>> | Total number of sending packets
-*-------------------------------------+--------------------------------------+
-|<<<SendDataPacketTransferNanosAvgTime>>> | Average transfer time of sending
- | packets in nanoseconds
-*-------------------------------------+--------------------------------------+
-
-yarn context
-
-* ClusterMetrics
-
- ClusterMetrics shows the metrics of the YARN cluster from the
- ResourceManager's perspective. Each metrics record contains
- Hostname tag as additional information along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<NumActiveNMs>>> | Current number of active NodeManagers
-*-------------------------------------+--------------------------------------+
-|<<<NumDecommissionedNMs>>> | Current number of decommissioned NodeManagers
-*-------------------------------------+--------------------------------------+
-|<<<NumLostNMs>>> | Current number of lost NodeManagers for not sending
- | heartbeats
-*-------------------------------------+--------------------------------------+
-|<<<NumUnhealthyNMs>>> | Current number of unhealthy NodeManagers
-*-------------------------------------+--------------------------------------+
-|<<<NumRebootedNMs>>> | Current number of rebooted NodeManagers
-*-------------------------------------+--------------------------------------+
-
-* QueueMetrics
-
- QueueMetrics shows an application queue from the
- ResourceManager's perspective. Each metrics record shows
- the statistics of each queue, and contains tags such as
- queue name and Hostname as additional information along with metrics.
-
- In <<<running_>>><num> metrics such as <<<running_0>>>, you can set the
- property <<<yarn.resourcemanager.metrics.runtime.buckets>>> in yarn-site.xml
- to change the buckets. The default values is <<<60,300,1440>>>.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<running_0>>> | Current number of running applications whose elapsed time are
- | less than 60 minutes
-*-------------------------------------+--------------------------------------+
-|<<<running_60>>> | Current number of running applications whose elapsed time are
- | between 60 and 300 minutes
-*-------------------------------------+--------------------------------------+
-|<<<running_300>>> | Current number of running applications whose elapsed time are
- | between 300 and 1440 minutes
-*-------------------------------------+--------------------------------------+
-|<<<running_1440>>> | Current number of running applications elapsed time are
- | more than 1440 minutes
-*-------------------------------------+--------------------------------------+
-|<<<AppsSubmitted>>> | Total number of submitted applications
-*-------------------------------------+--------------------------------------+
-|<<<AppsRunning>>> | Current number of running applications
-*-------------------------------------+--------------------------------------+
-|<<<AppsPending>>> | Current number of applications that have not yet been
- | assigned by any containers
-*-------------------------------------+--------------------------------------+
-|<<<AppsCompleted>>> | Total number of completed applications
-*-------------------------------------+--------------------------------------+
-|<<<AppsKilled>>> | Total number of killed applications
-*-------------------------------------+--------------------------------------+
-|<<<AppsFailed>>> | Total number of failed applications
-*-------------------------------------+--------------------------------------+
-|<<<AllocatedMB>>> | Current allocated memory in MB
-*-------------------------------------+--------------------------------------+
-|<<<AllocatedVCores>>> | Current allocated CPU in virtual cores
-*-------------------------------------+--------------------------------------+
-|<<<AllocatedContainers>>> | Current number of allocated containers
-*-------------------------------------+--------------------------------------+
-|<<<AggregateContainersAllocated>>> | Total number of allocated containers
-*-------------------------------------+--------------------------------------+
-|<<<AggregateContainersReleased>>> | Total number of released containers
-*-------------------------------------+--------------------------------------+
-|<<<AvailableMB>>> | Current available memory in MB
-*-------------------------------------+--------------------------------------+
-|<<<AvailableVCores>>> | Current available CPU in virtual cores
-*-------------------------------------+--------------------------------------+
-|<<<PendingMB>>> | Current pending memory resource requests in MB that are
- | not yet fulfilled by the scheduler
-*-------------------------------------+--------------------------------------+
-|<<<PendingVCores>>> | Current pending CPU allocation requests in virtual
- | cores that are not yet fulfilled by the scheduler
-*-------------------------------------+--------------------------------------+
-|<<<PendingContainers>>> | Current pending resource requests that are not
- | yet fulfilled by the scheduler
-*-------------------------------------+--------------------------------------+
-|<<<ReservedMB>>> | Current reserved memory in MB
-*-------------------------------------+--------------------------------------+
-|<<<ReservedVCores>>> | Current reserved CPU in virtual cores
-*-------------------------------------+--------------------------------------+
-|<<<ReservedContainers>>> | Current number of reserved containers
-*-------------------------------------+--------------------------------------+
-|<<<ActiveUsers>>> | Current number of active users
-*-------------------------------------+--------------------------------------+
-|<<<ActiveApplications>>> | Current number of active applications
-*-------------------------------------+--------------------------------------+
-|<<<FairShareMB>>> | (FairScheduler only) Current fair share of memory in MB
-*-------------------------------------+--------------------------------------+
-|<<<FairShareVCores>>> | (FairScheduler only) Current fair share of CPU in
- | virtual cores
-*-------------------------------------+--------------------------------------+
-|<<<MinShareMB>>> | (FairScheduler only) Minimum share of memory in MB
-*-------------------------------------+--------------------------------------+
-|<<<MinShareVCores>>> | (FairScheduler only) Minimum share of CPU in virtual
- | cores
-*-------------------------------------+--------------------------------------+
-|<<<MaxShareMB>>> | (FairScheduler only) Maximum share of memory in MB
-*-------------------------------------+--------------------------------------+
-|<<<MaxShareVCores>>> | (FairScheduler only) Maximum share of CPU in virtual
- | cores
-*-------------------------------------+--------------------------------------+
-
-* NodeManagerMetrics
-
- NodeManagerMetrics shows the statistics of the containers in the node.
- Each metrics record contains Hostname tag as additional information
- along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<containersLaunched>>> | Total number of launched containers
-*-------------------------------------+--------------------------------------+
-|<<<containersCompleted>>> | Total number of successfully completed containers
-*-------------------------------------+--------------------------------------+
-|<<<containersFailed>>> | Total number of failed containers
-*-------------------------------------+--------------------------------------+
-|<<<containersKilled>>> | Total number of killed containers
-*-------------------------------------+--------------------------------------+
-|<<<containersIniting>>> | Current number of initializing containers
-*-------------------------------------+--------------------------------------+
-|<<<containersRunning>>> | Current number of running containers
-*-------------------------------------+--------------------------------------+
-|<<<allocatedContainers>>> | Current number of allocated containers
-*-------------------------------------+--------------------------------------+
-|<<<allocatedGB>>> | Current allocated memory in GB
-*-------------------------------------+--------------------------------------+
-|<<<availableGB>>> | Current available memory in GB
-*-------------------------------------+--------------------------------------+
-
-ugi context
-
-* UgiMetrics
-
- UgiMetrics is related to user and group information.
- Each metrics record contains Hostname tag as additional information
- along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<LoginSuccessNumOps>>> | Total number of successful kerberos logins
-*-------------------------------------+--------------------------------------+
-|<<<LoginSuccessAvgTime>>> | Average time for successful kerberos logins in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<LoginFailureNumOps>>> | Total number of failed kerberos logins
-*-------------------------------------+--------------------------------------+
-|<<<LoginFailureAvgTime>>> | Average time for failed kerberos logins in
- | milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<getGroupsNumOps>>> | Total number of group resolutions
-*-------------------------------------+--------------------------------------+
-|<<<getGroupsAvgTime>>> | Average time for group resolution in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<getGroups>>><num><<<sNumOps>>> |
-| | Total number of group resolutions (<num> seconds granularity). <num> is
-| | specified by <<<hadoop.user.group.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<getGroups>>><num><<<s50thPercentileLatency>>> |
-| | Shows the 50th percentile of group resolution time in milliseconds
-| | (<num> seconds granularity). <num> is specified by
-| | <<<hadoop.user.group.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<getGroups>>><num><<<s75thPercentileLatency>>> |
-| | Shows the 75th percentile of group resolution time in milliseconds
-| | (<num> seconds granularity). <num> is specified by
-| | <<<hadoop.user.group.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<getGroups>>><num><<<s90thPercentileLatency>>> |
-| | Shows the 90th percentile of group resolution time in milliseconds
-| | (<num> seconds granularity). <num> is specified by
-| | <<<hadoop.user.group.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<getGroups>>><num><<<s95thPercentileLatency>>> |
-| | Shows the 95th percentile of group resolution time in milliseconds
-| | (<num> seconds granularity). <num> is specified by
-| | <<<hadoop.user.group.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-|<<<getGroups>>><num><<<s99thPercentileLatency>>> |
-| | Shows the 99th percentile of group resolution time in milliseconds
-| | (<num> seconds granularity). <num> is specified by
-| | <<<hadoop.user.group.metrics.percentiles.intervals>>>.
-*-------------------------------------+--------------------------------------+
-
-metricssystem context
-
-* MetricsSystem
-
- MetricsSystem shows the statistics for metrics snapshots and publishes.
- Each metrics record contains Hostname tag as additional information
- along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<NumActiveSources>>> | Current number of active metrics sources
-*-------------------------------------+--------------------------------------+
-|<<<NumAllSources>>> | Total number of metrics sources
-*-------------------------------------+--------------------------------------+
-|<<<NumActiveSinks>>> | Current number of active sinks
-*-------------------------------------+--------------------------------------+
-|<<<NumAllSinks>>> | Total number of sinks \
- | (BUT usually less than <<<NumActiveSinks>>>,
- | see {{{https://issues.apache.org/jira/browse/HADOOP-9946}HADOOP-9946}})
-*-------------------------------------+--------------------------------------+
-|<<<SnapshotNumOps>>> | Total number of operations to snapshot statistics from
- | a metrics source
-*-------------------------------------+--------------------------------------+
-|<<<SnapshotAvgTime>>> | Average time in milliseconds to snapshot statistics
- | from a metrics source
-*-------------------------------------+--------------------------------------+
-|<<<PublishNumOps>>> | Total number of operations to publish statistics to a
- | sink
-*-------------------------------------+--------------------------------------+
-|<<<PublishAvgTime>>> | Average time in milliseconds to publish statistics to
- | a sink
-*-------------------------------------+--------------------------------------+
-|<<<DroppedPubAll>>> | Total number of dropped publishes
-*-------------------------------------+--------------------------------------+
-|<<<Sink_>>><instance><<<NumOps>>> | Total number of sink operations for the
- | <instance>
-*-------------------------------------+--------------------------------------+
-|<<<Sink_>>><instance><<<AvgTime>>> | Average time in milliseconds of sink
- | operations for the <instance>
-*-------------------------------------+--------------------------------------+
-|<<<Sink_>>><instance><<<Dropped>>> | Total number of dropped sink operations
- | for the <instance>
-*-------------------------------------+--------------------------------------+
-|<<<Sink_>>><instance><<<Qsize>>> | Current queue length of sink operations \
- | (BUT always set to 0 because nothing to
- | increment this metrics, see
- | {{{https://issues.apache.org/jira/browse/HADOOP-9941}HADOOP-9941}})
-*-------------------------------------+--------------------------------------+
-
-default context
-
-* StartupProgress
-
- StartupProgress metrics shows the statistics of NameNode startup.
- Four metrics are exposed for each startup phase based on its name.
- The startup <phase>s are <<<LoadingFsImage>>>, <<<LoadingEdits>>>,
- <<<SavingCheckpoint>>>, and <<<SafeMode>>>.
- Each metrics record contains Hostname tag as additional information
- along with metrics.
-
-*-------------------------------------+--------------------------------------+
-|| Name || Description
-*-------------------------------------+--------------------------------------+
-|<<<ElapsedTime>>> | Total elapsed time in milliseconds
-*-------------------------------------+--------------------------------------+
-|<<<PercentComplete>>> | Current rate completed in NameNode startup progress \
- | (The max value is not 100 but 1.0)
-*-------------------------------------+--------------------------------------+
-|<phase><<<Count>>> | Total number of steps completed in the phase
-*-------------------------------------+--------------------------------------+
-|<phase><<<ElapsedTime>>> | Total elapsed time in the phase in milliseconds
-*-------------------------------------+--------------------------------------+
-|<phase><<<Total>>> | Total number of steps in the phase
-*-------------------------------------+--------------------------------------+
-|<phase><<<PercentComplete>>> | Current rate completed in the phase \
- | (The max value is not 100 but 1.0)
-*-------------------------------------+--------------------------------------+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
deleted file mode 100644
index 866b428..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/NativeLibraries.apt.vm
+++ /dev/null
@@ -1,205 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Native Libraries Guide
- ---
- ---
- ${maven.build.timestamp}
-
-Native Libraries Guide
-
-%{toc|section=1|fromDepth=0}
-
-* Overview
-
- This guide describes the native hadoop library and includes a small
- discussion about native shared libraries.
-
- Note: Depending on your environment, the term "native libraries" could
- refer to all *.so's you need to compile; and, the term "native
- compression" could refer to all *.so's you need to compile that are
- specifically related to compression. Currently, however, this document
- only addresses the native hadoop library (<<<libhadoop.so>>>).
- The document for libhdfs library (<<<libhdfs.so>>>) is
- {{{../hadoop-hdfs/LibHdfs.html}here}}.
-
-* Native Hadoop Library
-
- Hadoop has native implementations of certain components for performance
- reasons and for non-availability of Java implementations. These
- components are available in a single, dynamically-linked native library
- called the native hadoop library. On the *nix platforms the library is
- named <<<libhadoop.so>>>.
-
-* Usage
-
- It is fairly easy to use the native hadoop library:
-
- [[1]] Review the components.
-
- [[2]] Review the supported platforms.
-
- [[3]] Either download a hadoop release, which will include a pre-built
- version of the native hadoop library, or build your own version of
- the native hadoop library. Whether you download or build, the name
- for the library is the same: libhadoop.so
-
- [[4]] Install the compression codec development packages (>zlib-1.2,
- >gzip-1.2):
-
- * If you download the library, install one or more development
- packages - whichever compression codecs you want to use with
- your deployment.
-
- * If you build the library, it is mandatory to install both
- development packages.
-
- [[5]] Check the runtime log files.
-
-* Components
-
- The native hadoop library includes various components:
-
- * Compression Codecs (bzip2, lz4, snappy, zlib)
-
- * Native IO utilities for {{{../hadoop-hdfs/ShortCircuitLocalReads.html}
- HDFS Short-Circuit Local Reads}} and
- {{{../hadoop-hdfs/CentralizedCacheManagement.html}Centralized Cache
- Management in HDFS}}
-
- * CRC32 checksum implementation
-
-* Supported Platforms
-
- The native hadoop library is supported on *nix platforms only. The
- library does not to work with Cygwin or the Mac OS X platform.
-
- The native hadoop library is mainly used on the GNU/Linus platform and
- has been tested on these distributions:
-
- * RHEL4/Fedora
-
- * Ubuntu
-
- * Gentoo
-
- On all the above distributions a 32/64 bit native hadoop library will
- work with a respective 32/64 bit jvm.
-
-* Download
-
- The pre-built 32-bit i386-Linux native hadoop library is available as
- part of the hadoop distribution and is located in the <<<lib/native>>>
- directory. You can download the hadoop distribution from Hadoop Common
- Releases.
-
- Be sure to install the zlib and/or gzip development packages -
- whichever compression codecs you want to use with your deployment.
-
-* Build
-
- The native hadoop library is written in ANSI C and is built using the
- GNU autotools-chain (autoconf, autoheader, automake, autoscan,
- libtool). This means it should be straight-forward to build the library
- on any platform with a standards-compliant C compiler and the GNU
- autotools-chain (see the supported platforms).
-
- The packages you need to install on the target platform are:
-
- * C compiler (e.g. GNU C Compiler)
-
- * GNU Autools Chain: autoconf, automake, libtool
-
- * zlib-development package (stable version >= 1.2.0)
-
- * openssl-development package(e.g. libssl-dev)
-
- Once you installed the prerequisite packages use the standard hadoop
- pom.xml file and pass along the native flag to build the native hadoop
- library:
-
-----
- $ mvn package -Pdist,native -DskipTests -Dtar
-----
-
- You should see the newly-built library in:
-
-----
- $ hadoop-dist/target/hadoop-${project.version}/lib/native
-----
-
- Please note the following:
-
- * It is mandatory to install both the zlib and gzip development
- packages on the target platform in order to build the native hadoop
- library; however, for deployment it is sufficient to install just
- one package if you wish to use only one codec.
-
- * It is necessary to have the correct 32/64 libraries for zlib,
- depending on the 32/64 bit jvm for the target platform, in order to
- build and deploy the native hadoop library.
-
-* Runtime
-
- The bin/hadoop script ensures that the native hadoop library is on the
- library path via the system property:
- <<<-Djava.library.path=<path> >>>
-
- During runtime, check the hadoop log files for your MapReduce tasks.
-
- * If everything is all right, then:
- <<<DEBUG util.NativeCodeLoader - Trying to load the custom-built native-hadoop library...>>>
- <<<INFO util.NativeCodeLoader - Loaded the native-hadoop library>>>
-
- * If something goes wrong, then:
- <<<INFO util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable>>>
-
-* Check
-
- NativeLibraryChecker is a tool to check whether native libraries are loaded correctly.
- You can launch NativeLibraryChecker as follows:
-
-----
- $ hadoop checknative -a
- 14/12/06 01:30:45 WARN bzip2.Bzip2Factory: Failed to load/initialize native-bzip2 library system-native, will use pure-Java version
- 14/12/06 01:30:45 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
- Native library checking:
- hadoop: true /home/ozawa/hadoop/lib/native/libhadoop.so.1.0.0
- zlib: true /lib/x86_64-linux-gnu/libz.so.1
- snappy: true /usr/lib/libsnappy.so.1
- lz4: true revision:99
- bzip2: false
-----
-
-
-* Native Shared Libraries
-
- You can load any native shared library using DistributedCache for
- distributing and symlinking the library files.
-
- This example shows you how to distribute a shared library, mylib.so,
- and load it from a MapReduce task.
-
- [[1]] First copy the library to the HDFS:
- <<<bin/hadoop fs -copyFromLocal mylib.so.1 /libraries/mylib.so.1>>>
-
- [[2]] The job launching program should contain the following:
- <<<DistributedCache.createSymlink(conf);>>>
- <<<DistributedCache.addCacheFile("hdfs://host:port/libraries/mylib.so. 1#mylib.so", conf);>>>
-
- [[3]] The MapReduce task can contain:
- <<<System.loadLibrary("mylib.so");>>>
-
- Note: If you downloaded or built the native hadoop library, you don’t
- need to use DistibutedCache to make the library available to your
- MapReduce tasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm
deleted file mode 100644
index dbd8d92..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/RackAwareness.apt.vm
+++ /dev/null
@@ -1,140 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop ${project.version} - Rack Awareness
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc|section=1|fromDepth=0}
-
-Rack Awareness
-
- Hadoop components are rack-aware. For example, HDFS block placement will
- use rack awareness for fault tolerance by placing one block replica on a
- different rack. This provides data availability in the event of a network
- switch failure or partition within the cluster.
-
- Hadoop master daemons obtain the rack id of the cluster slaves by invoking
- either an external script or java class as specified by configuration files.
- Using either the java class or external script for topology, output must
- adhere to the java <<org.apache.hadoop.net.DNSToSwitchMapping>>
- interface. The interface expects a one-to-one correspondence to be
- maintained and the topology information in the format of '/myrack/myhost',
- where '/' is the topology delimiter, 'myrack' is the rack identifier, and
- 'myhost' is the individual host. Assuming a single /24 subnet per rack,
- one could use the format of '/192.168.100.0/192.168.100.5' as a
- unique rack-host topology mapping.
-
- To use the java class for topology mapping, the class name is specified by
- the <<topology.node.switch.mapping.impl>> parameter in the configuration
- file. An example, NetworkTopology.java, is included with the hadoop
- distribution and can be customized by the Hadoop administrator. Using a
- Java class instead of an external script has a performance benefit in
- that Hadoop doesn't need to fork an external process when a new slave node
- registers itself.
-
- If implementing an external script, it will be specified with the
- <<topology.script.file.name>> parameter in the configuration files. Unlike
- the java class, the external topology script is not included with the Hadoop
- distribution and is provided by the administrator. Hadoop will send
- multiple IP addresses to ARGV when forking the topology script. The
- number of IP addresses sent to the topology script is controlled with
- <<net.topology.script.number.args>> and defaults to 100. If
- <<net.topology.script.number.args>> was changed to 1, a topology script
- would get forked for each IP submitted by DataNodes and/or NodeManagers.
-
- If <<topology.script.file.name>> or <<topology.node.switch.mapping.impl>> is
- not set, the rack id '/default-rack' is returned for any passed IP address.
- While this behavior appears desirable, it can cause issues with HDFS block
- replication as default behavior is to write one replicated block off rack
- and is unable to do so as there is only a single rack named '/default-rack'.
-
- An additional configuration setting is
- <<mapreduce.jobtracker.taskcache.levels>> which determines the number of
- levels (in the network topology) of caches MapReduce will use. So, for
- example, if it is the default value of 2, two levels of caches will be
- constructed - one for hosts (host -> task mapping) and another for racks
- (rack -> task mapping). Giving us our one-to-one mapping of '/myrack/myhost'.
-
-* {python Example}
-
-+-------------------------------+
- #!/usr/bin/python
- # this script makes assumptions about the physical environment.
- # 1) each rack is its own layer 3 network with a /24 subnet, which
- # could be typical where each rack has its own
- # switch with uplinks to a central core router.
- #
- # +-----------+
- # |core router|
- # +-----------+
- # / \
- # +-----------+ +-----------+
- # |rack switch| |rack switch|
- # +-----------+ +-----------+
- # | data node | | data node |
- # +-----------+ +-----------+
- # | data node | | data node |
- # +-----------+ +-----------+
- #
- # 2) topology script gets list of IP's as input, calculates network address, and prints '/network_address/ip'.
-
- import netaddr
- import sys
- sys.argv.pop(0) # discard name of topology script from argv list as we just want IP addresses
-
- netmask = '255.255.255.0' # set netmask to what's being used in your environment. The example uses a /24
-
- for ip in sys.argv: # loop over list of datanode IP's
- address = '{0}/{1}'.format(ip, netmask) # format address string so it looks like 'ip/netmask' to make netaddr work
- try:
- network_address = netaddr.IPNetwork(address).network # calculate and print network address
- print "/{0}".format(network_address)
- except:
- print "/rack-unknown" # print catch-all value if unable to calculate network address
-+-------------------------------+
-
-* {bash Example}
-
-+-------------------------------+
- #!/bin/bash
- # Here's a bash example to show just how simple these scripts can be
- # Assuming we have flat network with everything on a single switch, we can fake a rack topology.
- # This could occur in a lab environment where we have limited nodes,like 2-8 physical machines on a unmanaged switch.
- # This may also apply to multiple virtual machines running on the same physical hardware.
- # The number of machines isn't important, but that we are trying to fake a network topology when there isn't one.
- #
- # +----------+ +--------+
- # |jobtracker| |datanode|
- # +----------+ +--------+
- # \ /
- # +--------+ +--------+ +--------+
- # |datanode|--| switch |--|datanode|
- # +--------+ +--------+ +--------+
- # / \
- # +--------+ +--------+
- # |datanode| |namenode|
- # +--------+ +--------+
- #
- # With this network topology, we are treating each host as a rack. This is being done by taking the last octet
- # in the datanode's IP and prepending it with the word '/rack-'. The advantage for doing this is so HDFS
- # can create its 'off-rack' block copy.
- # 1) 'echo $@' will echo all ARGV values to xargs.
- # 2) 'xargs' will enforce that we print a single argv value per line
- # 3) 'awk' will split fields on dots and append the last field to the string '/rack-'. If awk
- # fails to split on four dots, it will still print '/rack-' last field value
-
- echo $@ | xargs -n 1 | awk -F '.' '{print "/rack-"$NF}'
-+-------------------------------+
-
[48/50] [abbrv] hadoop git commit: HDFS-7720. Update CHANGES.txt to
reflect merge to branch-2.
Posted by zj...@apache.org.
HDFS-7720. Update CHANGES.txt to reflect merge to branch-2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/078f3a9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/078f3a9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/078f3a9b
Branch: refs/heads/YARN-2928
Commit: 078f3a9bc7ce9d06ae2de3e65a099ee655bce483
Parents: 5dae97a
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Feb 11 11:08:32 2015 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Feb 11 11:08:32 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/078f3a9b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 625735e..d60a20f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -20,9 +20,6 @@ Trunk (Unreleased)
HDFS-3689. Add support for variable length block. (jing9)
- HDFS-7720. Quota by Storage Type API, tools and ClientNameNode Protocol
- changes. (Xiaoyu Yao via Arpit Agarwal)
-
HDFS-7723. Quota By Storage Type namenode implemenation. (Xiaoyu Yao via
Arpit Agarwal)
@@ -332,6 +329,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7655. Expose truncate API for Web HDFS. (yliu)
+ HDFS-7720. Quota by Storage Type API, tools and ClientNameNode Protocol
+ changes. (Xiaoyu Yao via Arpit Agarwal)
+
IMPROVEMENTS
HDFS-7055. Add tracing to DFSInputStream (cmccabe)
[07/50] [abbrv] hadoop git commit: HDFS-7669. HDFS Design Doc
references commands that no longer exist. (Brahma Reddy Battula via aw)
Posted by zj...@apache.org.
HDFS-7669. HDFS Design Doc references commands that no longer exist. (Brahma Reddy Battula via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcad031d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcad031d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcad031d
Branch: refs/heads/YARN-2928
Commit: fcad031d72508251b38eb5b96c02434a285e090d
Parents: 5764598
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 12:56:48 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 12:56:48 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm | 12 +++++-------
2 files changed, 8 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcad031d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0d9000a..74474ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -290,6 +290,9 @@ Trunk (Unreleased)
HDFS-7670. HDFS Quota guide has typos, incomplete command lines
(Brahma Reddy Battula via aw)
+ HDFS-7669. HDFS Design Doc references commands that no longer exist.
+ (Brahma Reddy Battula via aw)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcad031d/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm
index 9cd95fa..00628b1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm
@@ -13,9 +13,7 @@
---
HDFS Architecture
---
- Dhruba Borthakur
- ---
- ${maven.build.timestamp}
+ ${maven.build.timestamp}
HDFS Architecture
@@ -439,7 +437,7 @@ HDFS Architecture
*---------+---------+
| Create a directory named <<</foodir>>> | <<<bin/hadoop dfs -mkdir /foodir>>>
*---------+---------+
-| Remove a directory named <<</foodir>>> | <<<bin/hadoop dfs -rmr /foodir>>>
+| Remove a directory named <<</foodir>>> | <<<bin/hadoop fs -rm -R /foodir>>>
*---------+---------+
| View the contents of a file named <<</foodir/myfile.txt>>> | <<<bin/hadoop dfs -cat /foodir/myfile.txt>>>
*---------+---------+
@@ -456,11 +454,11 @@ HDFS Architecture
*---------+---------+
|| Action | Command
*---------+---------+
-|Put the cluster in Safemode | <<<bin/hadoop dfsadmin -safemode enter>>>
+|Put the cluster in Safemode | <<<bin/hdfs dfsadmin -safemode enter>>>
*---------+---------+
-|Generate a list of DataNodes | <<<bin/hadoop dfsadmin -report>>>
+|Generate a list of DataNodes | <<<bin/hdfs dfsadmin -report>>>
*---------+---------+
-|Recommission or decommission DataNode(s) | <<<bin/hadoop dfsadmin -refreshNodes>>>
+|Recommission or decommission DataNode(s) | <<<bin/hdfs dfsadmin -refreshNodes>>>
*---------+---------+
** Browser Interface
[32/50] [abbrv] hadoop git commit: HDFS-7769. TestHDFSCLI should not
create files in hdfs project root dir.
Posted by zj...@apache.org.
HDFS-7769. TestHDFSCLI should not create files in hdfs project root dir.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c6b6547
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c6b6547
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c6b6547
Branch: refs/heads/YARN-2928
Commit: 7c6b6547eeed110e1a842e503bfd33afe04fa814
Parents: 7eeca90
Author: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Authored: Tue Feb 10 17:48:57 2015 -0800
Committer: Tsz-Wo Nicholas Sze <sz...@hortonworks.com>
Committed: Tue Feb 10 17:48:57 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/test/resources/testHDFSConf.xml | 4 ++--
2 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6b6547/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 59d498d..2e90478 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -901,6 +901,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause
DataNode to register successfully with only one NameNode.(vinayakumarb)
+ HDFS-7769. TestHDFSCLI should not create files in hdfs project root dir.
+ (szetszwo)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c6b6547/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 05546a2..935cd58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -16483,8 +16483,8 @@
<command>-fs NAMENODE -mkdir -p /user/USERNAME/dir1</command>
<command>-fs NAMENODE -copyFromLocal CLITEST_DATA/data15bytes /user/USERNAME/dir1</command>
<command>-fs NAMENODE -copyFromLocal CLITEST_DATA/data30bytes /user/USERNAME/dir1</command>
- <command>-fs NAMENODE -getmerge /user/USERNAME/dir1 data</command>
- <command>-cat data</command>
+ <command>-fs NAMENODE -getmerge /user/USERNAME/dir1 CLITEST_DATA/file</command>
+ <command>-cat CLITEST_DATA/file</command>
</test-commands>
<cleanup-commands>
<command>-fs NAMENODE -rm -r /user/USERNAME</command>
[12/50] [abbrv] hadoop git commit: YARN-2971. RM uses conf instead of
token service address to renew timeline delegation tokens (jeagles)
Posted by zj...@apache.org.
YARN-2971. RM uses conf instead of token service address to renew timeline delegation tokens (jeagles)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/af084258
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/af084258
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/af084258
Branch: refs/heads/YARN-2928
Commit: af0842589359ad800427337ad2c84fac09907f72
Parents: aab459c
Author: Jonathan Eagles <je...@gmail.com>
Authored: Mon Feb 9 17:56:05 2015 -0600
Committer: Jonathan Eagles <je...@gmail.com>
Committed: Mon Feb 9 17:56:05 2015 -0600
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../client/api/impl/TimelineClientImpl.java | 22 ++++++++++++++------
.../client/api/impl/TestTimelineClient.java | 14 +++++++++++--
3 files changed, 31 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af084258/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 578a8cc..634a0e7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -519,6 +519,9 @@ Release 2.7.0 - UNRELEASED
YARN-3094. Reset timer for liveness monitors after RM recovery. (Jun Gong
via jianhe)
+ YARN-2971. RM uses conf instead of token service address to renew timeline
+ delegation tokens (jeagles)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af084258/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index de9d8da..0b88632 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
import java.net.ConnectException;
import java.net.HttpURLConnection;
+import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URL;
import java.net.URLConnection;
@@ -45,6 +46,7 @@ import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
@@ -373,12 +375,14 @@ public class TimelineClientImpl extends TimelineClient {
== UserGroupInformation.AuthenticationMethod.PROXY;
final String doAsUser = isProxyAccess ?
UserGroupInformation.getCurrentUser().getShortUserName() : null;
+ boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
+ final String scheme = useHttps ? "https" : "http";
+ final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
PrivilegedExceptionAction<Long> renewDTAction =
new PrivilegedExceptionAction<Long>() {
@Override
- public Long run()
- throws Exception {
+ public Long run() throws Exception {
// If the timeline DT to renew is different than cached, replace it.
// Token to set every time for retry, because when exception happens,
// DelegationTokenAuthenticatedURL will reset it to null;
@@ -388,8 +392,10 @@ public class TimelineClientImpl extends TimelineClient {
DelegationTokenAuthenticatedURL authUrl =
new DelegationTokenAuthenticatedURL(authenticator,
connConfigurator);
+ final URI serviceURI = new URI(scheme, null, address.getHostName(),
+ address.getPort(), RESOURCE_URI_STR, null, null);
return authUrl
- .renewDelegationToken(resURI.toURL(), token, doAsUser);
+ .renewDelegationToken(serviceURI.toURL(), token, doAsUser);
}
};
return (Long) operateDelegationToken(renewDTAction);
@@ -405,12 +411,14 @@ public class TimelineClientImpl extends TimelineClient {
== UserGroupInformation.AuthenticationMethod.PROXY;
final String doAsUser = isProxyAccess ?
UserGroupInformation.getCurrentUser().getShortUserName() : null;
+ boolean useHttps = YarnConfiguration.useHttps(this.getConfig());
+ final String scheme = useHttps ? "https" : "http";
+ final InetSocketAddress address = SecurityUtil.getTokenServiceAddr(timelineDT);
PrivilegedExceptionAction<Void> cancelDTAction =
new PrivilegedExceptionAction<Void>() {
@Override
- public Void run()
- throws Exception {
+ public Void run() throws Exception {
// If the timeline DT to cancel is different than cached, replace it.
// Token to set every time for retry, because when exception happens,
// DelegationTokenAuthenticatedURL will reset it to null;
@@ -420,7 +428,9 @@ public class TimelineClientImpl extends TimelineClient {
DelegationTokenAuthenticatedURL authUrl =
new DelegationTokenAuthenticatedURL(authenticator,
connConfigurator);
- authUrl.cancelDelegationToken(resURI.toURL(), token, doAsUser);
+ final URI serviceURI = new URI(scheme, null, address.getHostName(),
+ address.getPort(), RESOURCE_URI_STR, null, null);
+ authUrl.cancelDelegationToken(serviceURI.toURL(), token, doAsUser);
return null;
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/af084258/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
index c8027a2..859a6c9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestTimelineClient.java
@@ -238,7 +238,10 @@ public class TestTimelineClient {
new TimelineDelegationTokenIdentifier(
new Text("tester"), new Text("tester"), new Text("tester"));
client.renewDelegationToken(
- new Token<TimelineDelegationTokenIdentifier>(timelineDT, dtManager));
+ new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(),
+ dtManager.createPassword(timelineDT),
+ timelineDT.getKind(),
+ new Text("0.0.0.0:8188")));
assertFail();
} catch (RuntimeException ce) {
assertException(client, ce);
@@ -250,7 +253,10 @@ public class TestTimelineClient {
new TimelineDelegationTokenIdentifier(
new Text("tester"), new Text("tester"), new Text("tester"));
client.cancelDelegationToken(
- new Token<TimelineDelegationTokenIdentifier>(timelineDT, dtManager));
+ new Token<TimelineDelegationTokenIdentifier>(timelineDT.getBytes(),
+ dtManager.createPassword(timelineDT),
+ timelineDT.getKind(),
+ new Text("0.0.0.0:8188")));
assertFail();
} catch (RuntimeException ce) {
assertException(client, ce);
@@ -371,5 +377,9 @@ public class TestTimelineClient {
return new TimelineDelegationTokenIdentifier();
}
+ @Override
+ public synchronized byte[] createPassword(TimelineDelegationTokenIdentifier identifier) {
+ return super.createPassword(identifier);
+ }
}
}
[49/50] [abbrv] hadoop git commit: HDFS-7736. Fix typos in
dfsadmin/fsck/snapshotDiff usage messages. Contributed by Brahma Reddy
Battula.
Posted by zj...@apache.org.
HDFS-7736. Fix typos in dfsadmin/fsck/snapshotDiff usage messages. Contributed by Brahma Reddy Battula.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f80c9888
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f80c9888
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f80c9888
Branch: refs/heads/YARN-2928
Commit: f80c9888fa0c1a11967560be3c37dfc1e30da2c3
Parents: 078f3a9
Author: Haohui Mai <wh...@apache.org>
Authored: Wed Feb 11 11:21:33 2015 -0800
Committer: Haohui Mai <wh...@apache.org>
Committed: Wed Feb 11 11:21:33 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/hdfs/server/mover/Mover.java | 4 ++--
.../main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java | 10 +++++-----
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 2 +-
.../hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java | 2 +-
.../apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java | 2 +-
6 files changed, 13 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80c9888/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index d60a20f..dd90c86 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -915,6 +915,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7753. Fix Multithreaded correctness Warnings in BackupImage.
(Rakesh R and shv)
+ HDFS-7736. Fix typos in dfsadmin/fsck/snapshotDiff usage messages.
+ (Brahma Reddy Battula via wheat9)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80c9888/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index 6fa6963..346d511 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -560,8 +560,8 @@ public class Mover {
}
static class Cli extends Configured implements Tool {
- private static final String USAGE = "Usage: java "
- + Mover.class.getSimpleName() + " [-p <files/dirs> | -f <local file>]"
+ private static final String USAGE = "Usage: hdfs mover "
+ + "[-p <files/dirs> | -f <local file>]"
+ "\n\t-p <files/dirs>\ta space separated list of HDFS files/dirs to migrate."
+ "\n\t-f <local file>\ta local file containing a list of HDFS files/dirs to migrate.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80c9888/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index 77c77fe..ae9088c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -545,7 +545,7 @@ public class DFSAdmin extends FsShell {
/**
* Safe mode maintenance command.
- * Usage: java DFSAdmin -safemode [enter | leave | get]
+ * Usage: hdfs dfsadmin -safemode [enter | leave | get]
* @param argv List of of command line parameters.
* @param idx The index of the command that is being processed.
* @exception IOException if the filesystem does not exist.
@@ -661,7 +661,7 @@ public class DFSAdmin extends FsShell {
/**
* Allow snapshot on a directory.
- * Usage: java DFSAdmin -allowSnapshot snapshotDir
+ * Usage: hdfs dfsadmin -allowSnapshot snapshotDir
* @param argv List of of command line parameters.
* @exception IOException
*/
@@ -677,7 +677,7 @@ public class DFSAdmin extends FsShell {
/**
* Allow snapshot on a directory.
- * Usage: java DFSAdmin -disallowSnapshot snapshotDir
+ * Usage: hdfs dfsadmin -disallowSnapshot snapshotDir
* @param argv List of of command line parameters.
* @exception IOException
*/
@@ -1585,7 +1585,7 @@ public class DFSAdmin extends FsShell {
System.err.println("Usage: hdfs dfsadmin"
+ " [-refreshCallQueue]");
} else if ("-reconfig".equals(cmd)) {
- System.err.println("Usage: java DFSAdmin"
+ System.err.println("Usage: hdfs dfsadmin"
+ " [-reconfig <datanode|...> <host:port> <start|status>]");
} else if ("-refresh".equals(cmd)) {
System.err.println("Usage: hdfs dfsadmin"
@@ -1612,7 +1612,7 @@ public class DFSAdmin extends FsShell {
System.err.println("Usage: hdfs dfsadmin"
+ " [-getDatanodeInfo <datanode_host:ipc_port>]");
} else if ("-triggerBlockReport".equals(cmd)) {
- System.err.println("Usage: java DFSAdmin"
+ System.err.println("Usage: hdfs dfsadmin"
+ " [-triggerBlockReport [-incremental] <datanode_host:ipc_port>]");
} else {
System.err.println("Usage: hdfs dfsadmin");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80c9888/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 98f2030..8c934c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -75,7 +75,7 @@ public class DFSck extends Configured implements Tool {
HdfsConfiguration.init();
}
- private static final String USAGE = "Usage: DFSck <path> "
+ private static final String USAGE = "Usage: hdfs fsck <path> "
+ "[-list-corruptfileblocks | "
+ "[-move | -delete | -openforwrite] "
+ "[-files [-blocks [-locations | -racks]]]] "
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80c9888/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
index b2b9ce1..2f030b4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/LsSnapshottableDir.java
@@ -37,7 +37,7 @@ import org.apache.hadoop.util.ToolRunner;
public class LsSnapshottableDir extends Configured implements Tool {
@Override
public int run(String[] argv) throws Exception {
- String description = "LsSnapshottableDir: \n" +
+ String description = "hdfs lsSnapshottableDir: \n" +
"\tGet the list of snapshottable directories that are owned by the current user.\n" +
"\tReturn all the snapshottable directories if the current user is a super user.\n";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f80c9888/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
index 21c63ba..b87d69f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/snapshot/SnapshotDiff.java
@@ -61,7 +61,7 @@ public class SnapshotDiff extends Configured implements Tool {
@Override
public int run(String[] argv) throws Exception {
- String description = "SnapshotDiff <snapshotDir> <from> <to>:\n" +
+ String description = "hdfs snapshotDiff <snapshotDir> <from> <to>:\n" +
"\tGet the difference between two snapshots, \n" +
"\tor between a snapshot and the current tree of a directory.\n" +
"\tFor <from>/<to>, users can use \".\" to present the current status,\n" +
[47/50] [abbrv] hadoop git commit: HDFS-7723. Quota By Storage Type
namenode implemenation. (Contributed by Xiaoyu Yao)
Posted by zj...@apache.org.
HDFS-7723. Quota By Storage Type namenode implemenation. (Contributed by Xiaoyu Yao)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5dae97a5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5dae97a5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5dae97a5
Branch: refs/heads/YARN-2928
Commit: 5dae97a584d30cef3e34141edfaca49c4ec57913
Parents: e42fc1a
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Feb 11 10:41:50 2015 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Feb 11 10:41:50 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../server/blockmanagement/BlockManager.java | 4 +
.../hdfs/server/namenode/BackupImage.java | 4 +-
.../ContentSummaryComputationContext.java | 5 +
.../namenode/DirectoryWithQuotaFeature.java | 228 +++++---
.../hdfs/server/namenode/FSDirAttrOp.java | 64 ++-
.../hdfs/server/namenode/FSDirConcatOp.java | 45 +-
.../hdfs/server/namenode/FSDirDeleteOp.java | 11 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 47 +-
.../hdfs/server/namenode/FSDirectory.java | 171 ++++--
.../hadoop/hdfs/server/namenode/FSEditLog.java | 10 +
.../hdfs/server/namenode/FSEditLogLoader.java | 15 +-
.../hdfs/server/namenode/FSEditLogOp.java | 85 +++
.../hdfs/server/namenode/FSEditLogOpCodes.java | 1 +
.../hadoop/hdfs/server/namenode/FSImage.java | 60 ++-
.../hdfs/server/namenode/FSImageFormat.java | 15 +-
.../server/namenode/FSImageFormatPBINode.java | 80 ++-
.../server/namenode/FSImageSerialization.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 13 +-
.../hadoop/hdfs/server/namenode/INode.java | 76 +--
.../hdfs/server/namenode/INodeDirectory.java | 108 ++--
.../namenode/INodeDirectoryAttributes.java | 30 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 84 ++-
.../hadoop/hdfs/server/namenode/INodeMap.java | 19 +-
.../hdfs/server/namenode/INodeReference.java | 79 +--
.../hdfs/server/namenode/INodeSymlink.java | 18 +-
.../server/namenode/NameNodeLayoutVersion.java | 3 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 8 +-
.../hadoop/hdfs/server/namenode/Quota.java | 2 +-
.../namenode/QuotaByStorageTypeEntry.java | 85 +++
.../hdfs/server/namenode/QuotaCounts.java | 179 +++++++
.../namenode/snapshot/AbstractINodeDiff.java | 11 +-
.../snapshot/AbstractINodeDiffList.java | 12 +-
.../snapshot/DirectorySnapshottableFeature.java | 11 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 103 ++--
.../snapshot/FSImageFormatPBSnapshot.java | 37 +-
.../hdfs/server/namenode/snapshot/FileDiff.java | 12 +-
.../server/namenode/snapshot/FileDiffList.java | 5 +-
.../snapshot/FileWithSnapshotFeature.java | 69 ++-
.../namenode/snapshot/SnapshotManager.java | 3 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 36 +-
.../hadoop-hdfs/src/main/proto/fsimage.proto | 10 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 2 +
.../hdfs/TestDFSInotifyEventInputStream.java | 2 +-
.../hdfs/server/namenode/FSImageTestUtil.java | 2 +-
.../namenode/TestDiskspaceQuotaUpdate.java | 34 +-
.../namenode/TestFSImageWithSnapshot.java | 2 +-
.../server/namenode/TestQuotaByStorageType.java | 524 +++++++++++++++++++
.../snapshot/TestRenameWithSnapshots.java | 49 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 16 +-
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5803 -> 5850 bytes
.../src/test/resources/editsStored.xml | 373 ++++++-------
52 files changed, 2154 insertions(+), 717 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 758bcb1..625735e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -23,6 +23,9 @@ Trunk (Unreleased)
HDFS-7720. Quota by Storage Type API, tools and ClientNameNode Protocol
changes. (Xiaoyu Yao via Arpit Agarwal)
+ HDFS-7723. Quota By Storage Type namenode implemenation. (Xiaoyu Yao via
+ Arpit Agarwal)
+
IMPROVEMENTS
HDFS-4665. Move TestNetworkTopologyWithNodeGroup to common.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 3fe47af..b05d6a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -421,6 +421,10 @@ public class BlockManager {
}
}
+ public BlockStoragePolicySuite getStoragePolicySuite() {
+ return storagePolicySuite;
+ }
+
/** get the BlockTokenSecretManager */
@VisibleForTesting
public BlockTokenSecretManager getBlockTokenSecretManager() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
index dc44959..f0879ee 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
@@ -216,7 +216,9 @@ public class BackupImage extends FSImage {
}
lastAppliedTxId = logLoader.getLastAppliedTxId();
- FSImage.updateCountForQuota(getNamesystem().dir.getRoot()); // inefficient!
+ FSImage.updateCountForQuota(
+ getNamesystem().dir.getBlockStoragePolicySuite(),
+ getNamesystem().dir.rootDir); // inefficient!
} finally {
backupInputStream.clear();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
index dab64ec..63fa8c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ContentSummaryComputationContext.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
@InterfaceAudience.Private
@InterfaceStability.Unstable
@@ -116,4 +117,8 @@ public class ContentSummaryComputationContext {
public Content.Counts getCounts() {
return counts;
}
+
+ public BlockStoragePolicySuite getBlockStoragePolicySuite() {
+ return fsn.getBlockManager().getStoragePolicySuite();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
index 05742b2..e7eeba6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/DirectoryWithQuotaFeature.java
@@ -21,47 +21,97 @@ import org.apache.hadoop.hdfs.protocol.DSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.util.EnumCounters;
/**
* Quota feature for {@link INodeDirectory}.
*/
public final class DirectoryWithQuotaFeature implements INode.Feature {
public static final long DEFAULT_NAMESPACE_QUOTA = Long.MAX_VALUE;
- public static final long DEFAULT_DISKSPACE_QUOTA = HdfsConstants.QUOTA_RESET;
-
- /** Name space quota */
- private long nsQuota = DEFAULT_NAMESPACE_QUOTA;
- /** Name space count */
- private long namespace = 1L;
- /** Disk space quota */
- private long dsQuota = DEFAULT_DISKSPACE_QUOTA;
- /** Disk space count */
- private long diskspace = 0L;
-
- DirectoryWithQuotaFeature(long nsQuota, long dsQuota) {
- this.nsQuota = nsQuota;
- this.dsQuota = dsQuota;
+ public static final long DEFAULT_SPACE_QUOTA = HdfsConstants.QUOTA_RESET;
+
+ private QuotaCounts quota;
+ private QuotaCounts usage;
+
+ public static class Builder {
+ private QuotaCounts quota;
+ private QuotaCounts usage;
+
+ public Builder() {
+ this.quota = new QuotaCounts.Builder().nameCount(DEFAULT_NAMESPACE_QUOTA).
+ spaceCount(DEFAULT_SPACE_QUOTA).typeCounts(DEFAULT_SPACE_QUOTA).build();
+ this.usage = new QuotaCounts.Builder().nameCount(1).build();
+ }
+
+ public Builder nameSpaceQuota(long nameSpaceQuota) {
+ this.quota.setNameSpace(nameSpaceQuota);
+ return this;
+ }
+
+ public Builder spaceQuota(long spaceQuota) {
+ this.quota.setDiskSpace(spaceQuota);
+ return this;
+ }
+
+ public Builder typeQuotas(EnumCounters<StorageType> typeQuotas) {
+ this.quota.setTypeSpaces(typeQuotas);
+ return this;
+ }
+
+ public Builder typeQuota(StorageType type, long quota) {
+ this.quota.setTypeSpace(type, quota);
+ return this;
+ }
+
+ public DirectoryWithQuotaFeature build() {
+ return new DirectoryWithQuotaFeature(this);
+ }
+ }
+
+ private DirectoryWithQuotaFeature(Builder builder) {
+ this.quota = builder.quota;
+ this.usage = builder.usage;
}
/** @return the quota set or -1 if it is not set. */
- Quota.Counts getQuota() {
- return Quota.Counts.newInstance(nsQuota, dsQuota);
+ QuotaCounts getQuota() {
+ return new QuotaCounts.Builder().quotaCount(this.quota).build();
}
-
+
/** Set this directory's quota
*
* @param nsQuota Namespace quota to be set
* @param dsQuota Diskspace quota to be set
+ * @param type Storage type quota to be set
+ * * To set traditional space/namespace quota, type must be null
*/
+ void setQuota(long nsQuota, long dsQuota, StorageType type) {
+ if (type != null) {
+ this.quota.setTypeSpace(type, dsQuota);
+ } else {
+ setQuota(nsQuota, dsQuota);
+ }
+ }
+
void setQuota(long nsQuota, long dsQuota) {
- this.nsQuota = nsQuota;
- this.dsQuota = dsQuota;
+ this.quota.setNameSpace(nsQuota);
+ this.quota.setDiskSpace(dsQuota);
}
-
- Quota.Counts addNamespaceDiskspace(Quota.Counts counts) {
- counts.add(Quota.NAMESPACE, namespace);
- counts.add(Quota.DISKSPACE, diskspace);
+
+ void setQuota(long dsQuota, StorageType type) {
+ this.quota.setTypeSpace(type, dsQuota);
+ }
+
+ // Set in a batch only during FSImage load
+ void setQuota(EnumCounters<StorageType> typeQuotas) {
+ this.quota.setTypeSpaces(typeQuotas);
+ }
+
+ QuotaCounts addNamespaceDiskspace(QuotaCounts counts) {
+ counts.add(this.usage);
return counts;
}
@@ -76,45 +126,42 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
}
return summary;
}
-
+
private void checkDiskspace(final INodeDirectory dir, final long computed) {
- if (-1 != getQuota().get(Quota.DISKSPACE) && diskspace != computed) {
+ if (-1 != quota.getDiskSpace() && usage.getDiskSpace() != computed) {
NameNode.LOG.error("BUG: Inconsistent diskspace for directory "
- + dir.getFullPathName() + ". Cached = " + diskspace
+ + dir.getFullPathName() + ". Cached = " + usage.getDiskSpace()
+ " != Computed = " + computed);
}
}
- void addSpaceConsumed(final INodeDirectory dir, final long nsDelta,
- final long dsDelta, boolean verify) throws QuotaExceededException {
- if (dir.isQuotaSet()) {
- // The following steps are important:
+ void addSpaceConsumed(final INodeDirectory dir, final QuotaCounts counts,
+ boolean verify) throws QuotaExceededException {
+ if (dir.isQuotaSet()) {
+ // The following steps are important:
// check quotas in this inode and all ancestors before changing counts
// so that no change is made if there is any quota violation.
-
// (1) verify quota in this inode
if (verify) {
- verifyQuota(nsDelta, dsDelta);
+ verifyQuota(counts);
}
- // (2) verify quota and then add count in ancestors
- dir.addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
+ // (2) verify quota and then add count in ancestors
+ dir.addSpaceConsumed2Parent(counts, verify);
// (3) add count in this inode
- addSpaceConsumed2Cache(nsDelta, dsDelta);
+ addSpaceConsumed2Cache(counts);
} else {
- dir.addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
+ dir.addSpaceConsumed2Parent(counts, verify);
}
}
- /** Update the size of the tree
+ /** Update the space/namespace/type usage of the tree
*
- * @param nsDelta the change of the tree size
- * @param dsDelta change to disk space occupied
+ * @param delta the change of the namespace/space/type usage
*/
- public void addSpaceConsumed2Cache(long nsDelta, long dsDelta) {
- namespace += nsDelta;
- diskspace += dsDelta;
+ public void addSpaceConsumed2Cache(QuotaCounts delta) {
+ usage.add(delta);
}
-
+
/**
* Sets namespace and diskspace take by the directory rooted
* at this INode. This should be used carefully. It does not check
@@ -122,52 +169,103 @@ public final class DirectoryWithQuotaFeature implements INode.Feature {
*
* @param namespace size of the directory to be set
* @param diskspace disk space take by all the nodes under this directory
+ * @param typeUsed counters of storage type usage
*/
- void setSpaceConsumed(long namespace, long diskspace) {
- this.namespace = namespace;
- this.diskspace = diskspace;
+ void setSpaceConsumed(long namespace, long diskspace,
+ EnumCounters<StorageType> typeUsed) {
+ usage.setNameSpace(namespace);
+ usage.setDiskSpace(diskspace);
+ usage.setTypeSpaces(typeUsed);
}
-
+
+ void setSpaceConsumed(QuotaCounts c) {
+ usage.setNameSpace(c.getNameSpace());
+ usage.setDiskSpace(c.getDiskSpace());
+ usage.setTypeSpaces(c.getTypeSpaces());
+ }
+
/** @return the namespace and diskspace consumed. */
- public Quota.Counts getSpaceConsumed() {
- return Quota.Counts.newInstance(namespace, diskspace);
+ public QuotaCounts getSpaceConsumed() {
+ return new QuotaCounts.Builder().quotaCount(usage).build();
}
/** Verify if the namespace quota is violated after applying delta. */
private void verifyNamespaceQuota(long delta) throws NSQuotaExceededException {
- if (Quota.isViolated(nsQuota, namespace, delta)) {
- throw new NSQuotaExceededException(nsQuota, namespace + delta);
+ if (Quota.isViolated(quota.getNameSpace(), usage.getNameSpace(), delta)) {
+ throw new NSQuotaExceededException(quota.getNameSpace(),
+ usage.getNameSpace() + delta);
}
}
/** Verify if the diskspace quota is violated after applying delta. */
private void verifyDiskspaceQuota(long delta) throws DSQuotaExceededException {
- if (Quota.isViolated(dsQuota, diskspace, delta)) {
- throw new DSQuotaExceededException(dsQuota, diskspace + delta);
+ if (Quota.isViolated(quota.getDiskSpace(), usage.getDiskSpace(), delta)) {
+ throw new DSQuotaExceededException(quota.getDiskSpace(),
+ usage.getDiskSpace() + delta);
+ }
+ }
+
+ private void verifyQuotaByStorageType(EnumCounters<StorageType> typeDelta)
+ throws QuotaByStorageTypeExceededException {
+ if (!isQuotaByStorageTypeSet()) {
+ return;
+ }
+ for (StorageType t: StorageType.getTypesSupportingQuota()) {
+ if (!isQuotaByStorageTypeSet(t)) {
+ continue;
+ }
+ if (Quota.isViolated(quota.getTypeSpace(t), usage.getTypeSpace(t),
+ typeDelta.get(t))) {
+ throw new QuotaByStorageTypeExceededException(
+ quota.getTypeSpace(t), usage.getTypeSpace(t) + typeDelta.get(t), t);
+ }
}
}
/**
- * @throws QuotaExceededException if namespace or diskspace quotas is
- * violated after applying the deltas.
+ * @throws QuotaExceededException if namespace, diskspace or storage type quotas
+ * is violated after applying the deltas.
*/
- void verifyQuota(long nsDelta, long dsDelta) throws QuotaExceededException {
- verifyNamespaceQuota(nsDelta);
- verifyDiskspaceQuota(dsDelta);
+ void verifyQuota(QuotaCounts counts) throws QuotaExceededException {
+ verifyNamespaceQuota(counts.getNameSpace());
+ verifyDiskspaceQuota(counts.getDiskSpace());
+ verifyQuotaByStorageType(counts.getTypeSpaces());
}
-
+
boolean isQuotaSet() {
- return nsQuota >= 0 || dsQuota >= 0;
+ return quota.anyNsSpCountGreaterOrEqual(0) ||
+ quota.anyTypeCountGreaterOrEqual(0);
+ }
+
+ boolean isQuotaByStorageTypeSet() {
+ return quota.anyTypeCountGreaterOrEqual(0);
+ }
+
+ boolean isQuotaByStorageTypeSet(StorageType t) {
+ return quota.getTypeSpace(t) >= 0;
}
private String namespaceString() {
- return "namespace: " + (nsQuota < 0? "-": namespace + "/" + nsQuota);
+ return "namespace: " + (quota.getNameSpace() < 0? "-":
+ usage.getNameSpace() + "/" + quota.getNameSpace());
}
private String diskspaceString() {
- return "diskspace: " + (dsQuota < 0? "-": diskspace + "/" + dsQuota);
+ return "diskspace: " + (quota.getDiskSpace() < 0? "-":
+ usage.getDiskSpace() + "/" + quota.getDiskSpace());
}
-
+
+ private String quotaByStorageTypeString() {
+ StringBuilder sb = new StringBuilder();
+ for (StorageType t : StorageType.getTypesSupportingQuota()) {
+ sb.append("StorageType: " + t +
+ (quota.getTypeSpace(t) < 0? "-":
+ usage.getTypeSpace(t) + "/" + usage.getTypeSpace(t)));
+ }
+ return sb.toString();
+ }
+
@Override
public String toString() {
- return "Quota[" + namespaceString() + ", " + diskspaceString() + "]";
+ return "Quota[" + namespaceString() + ", " + diskspaceString() +
+ ", " + quotaByStorageTypeString() + "]";
}
-}
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
index 6c1890e..5843b4c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAttrOp.java
@@ -32,6 +32,8 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.security.AccessControlException;
import java.io.FileNotFoundException;
@@ -41,6 +43,7 @@ import java.util.EnumSet;
import java.util.List;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
public class FSDirAttrOp {
@@ -216,11 +219,11 @@ public class FSDirAttrOp {
}
/**
- * Set the namespace quota and diskspace quota for a directory.
+ * Set the namespace quota, diskspace and typeSpace quota for a directory.
*
* Note: This does not support ".inodes" relative path.
*/
- static void setQuota(FSDirectory fsd, String src, long nsQuota, long dsQuota)
+ static void setQuota(FSDirectory fsd, String src, long nsQuota, long dsQuota, StorageType type)
throws IOException {
if (fsd.isPermissionEnabled()) {
FSPermissionChecker pc = fsd.getPermissionChecker();
@@ -229,11 +232,15 @@ public class FSDirAttrOp {
fsd.writeLock();
try {
- INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, dsQuota);
+ INodeDirectory changed = unprotectedSetQuota(fsd, src, nsQuota, dsQuota, type);
if (changed != null) {
- final Quota.Counts q = changed.getQuotaCounts();
- fsd.getEditLog().logSetQuota(
- src, q.get(Quota.NAMESPACE), q.get(Quota.DISKSPACE));
+ final QuotaCounts q = changed.getQuotaCounts();
+ if (type == null) {
+ fsd.getEditLog().logSetQuota(src, q.getNameSpace(), q.getDiskSpace());
+ } else {
+ fsd.getEditLog().logSetQuotaByStorageType(
+ src, q.getTypeSpaces().get(type), type);
+ }
}
} finally {
fsd.writeUnlock();
@@ -294,7 +301,8 @@ public class FSDirAttrOp {
}
/**
- * See {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String, long, long)}
+ * See {@link org.apache.hadoop.hdfs.protocol.ClientProtocol#setQuota(String,
+ * long, long, StorageType)}
* for the contract.
* Sets quota for for a directory.
* @return INodeDirectory if any of the quotas have changed. null otherwise.
@@ -306,10 +314,10 @@ public class FSDirAttrOp {
* @throws SnapshotAccessControlException if path is in RO snapshot
*/
static INodeDirectory unprotectedSetQuota(
- FSDirectory fsd, String src, long nsQuota, long dsQuota)
+ FSDirectory fsd, String src, long nsQuota, long dsQuota, StorageType type)
throws FileNotFoundException, PathIsNotDirectoryException,
QuotaExceededException, UnresolvedLinkException,
- SnapshotAccessControlException {
+ SnapshotAccessControlException, UnsupportedActionException {
assert fsd.hasWriteLock();
// sanity check
if ((nsQuota < 0 && nsQuota != HdfsConstants.QUOTA_DONT_SET &&
@@ -320,6 +328,15 @@ public class FSDirAttrOp {
"dsQuota : " + nsQuota + " and " +
dsQuota);
}
+ // sanity check for quota by storage type
+ if ((type != null) && (!fsd.isQuotaByStorageTypeEnabled() ||
+ nsQuota != HdfsConstants.QUOTA_DONT_SET)) {
+ throw new UnsupportedActionException(
+ "Failed to set quota by storage type because either" +
+ DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY + " is set to " +
+ fsd.isQuotaByStorageTypeEnabled() + " or nsQuota value is illegal " +
+ nsQuota);
+ }
String srcs = FSDirectory.normalizePath(src);
final INodesInPath iip = fsd.getINodesInPath4Write(srcs, true);
@@ -327,22 +344,33 @@ public class FSDirAttrOp {
if (dirNode.isRoot() && nsQuota == HdfsConstants.QUOTA_RESET) {
throw new IllegalArgumentException("Cannot clear namespace quota on root.");
} else { // a directory inode
- final Quota.Counts oldQuota = dirNode.getQuotaCounts();
- final long oldNsQuota = oldQuota.get(Quota.NAMESPACE);
- final long oldDsQuota = oldQuota.get(Quota.DISKSPACE);
+ final QuotaCounts oldQuota = dirNode.getQuotaCounts();
+ final long oldNsQuota = oldQuota.getNameSpace();
+ final long oldDsQuota = oldQuota.getDiskSpace();
+
if (nsQuota == HdfsConstants.QUOTA_DONT_SET) {
nsQuota = oldNsQuota;
}
if (dsQuota == HdfsConstants.QUOTA_DONT_SET) {
dsQuota = oldDsQuota;
}
- if (oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
+
+ // unchanged space/namespace quota
+ if (type == null && oldNsQuota == nsQuota && oldDsQuota == dsQuota) {
return null;
}
+ // unchanged type quota
+ if (type != null) {
+ EnumCounters<StorageType> oldTypeQuotas = oldQuota.getTypeSpaces();
+ if (oldTypeQuotas != null && oldTypeQuotas.get(type) == dsQuota) {
+ return null;
+ }
+ }
+
final int latest = iip.getLatestSnapshotId();
dirNode.recordModification(latest);
- dirNode.setQuota(nsQuota, dsQuota);
+ dirNode.setQuota(fsd.getBlockStoragePolicySuite(), nsQuota, dsQuota, type);
return dirNode;
}
}
@@ -365,8 +393,8 @@ public class FSDirAttrOp {
// if replication > oldBR, then newBR == replication.
// if replication < oldBR, we don't know newBR yet.
if (replication > oldBR) {
- long dsDelta = (replication - oldBR)*(file.diskspaceConsumed()/oldBR);
- fsd.updateCount(iip, 0, dsDelta, true);
+ long dsDelta = file.diskspaceConsumed()/oldBR;
+ fsd.updateCount(iip, 0L, dsDelta, oldBR, replication, true);
}
file.setFileReplication(replication, iip.getLatestSnapshotId());
@@ -374,8 +402,8 @@ public class FSDirAttrOp {
final short newBR = file.getBlockReplication();
// check newBR < oldBR case.
if (newBR < oldBR) {
- long dsDelta = (newBR - oldBR)*(file.diskspaceConsumed()/newBR);
- fsd.updateCount(iip, 0, dsDelta, true);
+ long dsDelta = file.diskspaceConsumed()/newBR;
+ fsd.updateCount(iip, 0L, dsDelta, oldBR, newBR, true);
}
if (blockRepls != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
index ecfd2e1..d8cf42a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirConcatOp.java
@@ -20,14 +20,17 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.StorageType;
import java.io.IOException;
import java.util.Arrays;
import java.util.HashSet;
import java.util.Set;
+import java.util.List;
import static org.apache.hadoop.util.Time.now;
@@ -140,25 +143,42 @@ class FSDirConcatOp {
return si.toArray(new INodeFile[si.size()]);
}
- private static long computeQuotaDelta(INodeFile target, INodeFile[] srcList) {
- long delta = 0;
+ private static QuotaCounts computeQuotaDeltas(FSDirectory fsd, INodeFile target, INodeFile[] srcList) {
+ QuotaCounts deltas = new QuotaCounts.Builder().build();
short targetRepl = target.getBlockReplication();
for (INodeFile src : srcList) {
- if (targetRepl != src.getBlockReplication()) {
- delta += src.computeFileSize() *
- (targetRepl - src.getBlockReplication());
+ short srcRepl = src.getBlockReplication();
+ long fileSize = src.computeFileSize();
+ if (targetRepl != srcRepl) {
+ deltas.addDiskSpace(fileSize * (targetRepl - srcRepl));
+ BlockStoragePolicy bsp =
+ fsd.getBlockStoragePolicySuite().getPolicy(src.getStoragePolicyID());
+ if (bsp != null) {
+ List<StorageType> srcTypeChosen = bsp.chooseStorageTypes(srcRepl);
+ for (StorageType t : srcTypeChosen) {
+ if (t.supportTypeQuota()) {
+ deltas.addTypeSpace(t, -fileSize);
+ }
+ }
+ List<StorageType> targetTypeChosen = bsp.chooseStorageTypes(targetRepl);
+ for (StorageType t : targetTypeChosen) {
+ if (t.supportTypeQuota()) {
+ deltas.addTypeSpace(t, fileSize);
+ }
+ }
+ }
}
}
- return delta;
+ return deltas;
}
private static void verifyQuota(FSDirectory fsd, INodesInPath targetIIP,
- long delta) throws QuotaExceededException {
+ QuotaCounts deltas) throws QuotaExceededException {
if (!fsd.getFSNamesystem().isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
// Do not check quota if editlog is still being processed
return;
}
- FSDirectory.verifyQuota(targetIIP, targetIIP.length() - 1, 0, delta, null);
+ FSDirectory.verifyQuota(targetIIP, targetIIP.length() - 1, deltas, null);
}
/**
@@ -174,8 +194,8 @@ class FSDirConcatOp {
}
final INodeFile trgInode = targetIIP.getLastINode().asFile();
- long delta = computeQuotaDelta(trgInode, srcList);
- verifyQuota(fsd, targetIIP, delta);
+ QuotaCounts deltas = computeQuotaDeltas(fsd, trgInode, srcList);
+ verifyQuota(fsd, targetIIP, deltas);
// the target file can be included in a snapshot
trgInode.recordModification(targetIIP.getLatestSnapshotId());
@@ -195,8 +215,7 @@ class FSDirConcatOp {
trgInode.setModificationTime(timestamp, targetIIP.getLatestSnapshotId());
trgParent.updateModificationTime(timestamp, targetIIP.getLatestSnapshotId());
- // update quota on the parent directory ('count' files removed, 0 space)
- FSDirectory.unprotectedUpdateCount(targetIIP, targetIIP.length() - 1,
- -count, delta);
+ // update quota on the parent directory with deltas
+ FSDirectory.unprotectedUpdateCount(targetIIP, targetIIP.length() - 1, deltas);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
index 47c641c..2fc4711 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirDeleteOp.java
@@ -230,13 +230,14 @@ class FSDirDeleteOp {
// collect block and update quota
if (!targetNode.isInLatestSnapshot(latestSnapshot)) {
- targetNode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ targetNode.destroyAndCollectBlocks(fsd.getBlockStoragePolicySuite(),
+ collectedBlocks, removedINodes);
} else {
- Quota.Counts counts = targetNode.cleanSubtree(CURRENT_STATE_ID,
+ QuotaCounts counts = targetNode.cleanSubtree(
+ fsd.getBlockStoragePolicySuite(), CURRENT_STATE_ID,
latestSnapshot, collectedBlocks, removedINodes);
- removed = counts.get(Quota.NAMESPACE);
- fsd.updateCountNoQuotaCheck(iip, iip.length() - 1,
- -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+ removed = counts.getNameSpace();
+ fsd.updateCountNoQuotaCheck(iip, iip.length() -1, counts.negation());
}
if (NameNode.stateChangeLog.isDebugEnabled()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
index 6a58093..c57cae2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirRenameOp.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -90,16 +91,15 @@ class FSDirRenameOp {
int i = 0;
while(src.getINode(i) == dst.getINode(i)) { i++; }
// src[i - 1] is the last common ancestor.
-
- final Quota.Counts delta = src.getLastINode().computeQuotaUsage();
+ BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
+ final QuotaCounts delta = src.getLastINode().computeQuotaUsage(bsps);
// Reduce the required quota by dst that is being removed
final INode dstINode = dst.getLastINode();
if (dstINode != null) {
- delta.subtract(dstINode.computeQuotaUsage());
+ delta.subtract(dstINode.computeQuotaUsage(bsps));
}
- FSDirectory.verifyQuota(dst, dst.length() - 1, delta.get(Quota.NAMESPACE),
- delta.get(Quota.DISKSPACE), src.getINode(i - 1));
+ FSDirectory.verifyQuota(dst, dst.length() - 1, delta, src.getINode(i - 1));
}
/**
@@ -207,7 +207,7 @@ class FSDirRenameOp {
}
tx.updateMtimeAndLease(timestamp);
- tx.updateQuotasInSourceTree();
+ tx.updateQuotasInSourceTree(fsd.getBlockStoragePolicySuite());
return true;
}
@@ -356,6 +356,7 @@ class FSDirRenameOp {
throw new IOException(error);
}
+ BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
fsd.ezManager.checkMoveValidity(srcIIP, dstIIP, src);
final INode dstInode = dstIIP.getLastINode();
List<INodeDirectory> snapshottableDirs = new ArrayList<>();
@@ -412,7 +413,7 @@ class FSDirRenameOp {
if (undoRemoveDst) {
undoRemoveDst = false;
if (removedNum > 0) {
- filesDeleted = tx.cleanDst(collectedBlocks);
+ filesDeleted = tx.cleanDst(bsps, collectedBlocks);
}
}
@@ -422,7 +423,7 @@ class FSDirRenameOp {
fsd.getFSNamesystem().removeSnapshottableDirs(snapshottableDirs);
}
- tx.updateQuotasInSourceTree();
+ tx.updateQuotasInSourceTree(bsps);
return filesDeleted;
}
} finally {
@@ -430,7 +431,7 @@ class FSDirRenameOp {
tx.restoreSource();
}
if (undoRemoveDst) { // Rename failed - restore dst
- tx.restoreDst();
+ tx.restoreDst(bsps);
}
}
NameNode.stateChangeLog.warn("DIR* FSDirectory.unprotectedRenameTo: " +
@@ -566,7 +567,7 @@ class FSDirRenameOp {
private final byte[] srcChildName;
private final boolean isSrcInSnapshot;
private final boolean srcChildIsReference;
- private final Quota.Counts oldSrcCounts;
+ private final QuotaCounts oldSrcCounts;
private INode srcChild;
private INode oldDstChild;
@@ -581,6 +582,7 @@ class FSDirRenameOp {
this.srcParentIIP = srcIIP.getParentINodesInPath();
this.dstParentIIP = dstIIP.getParentINodesInPath();
+ BlockStoragePolicySuite bsps = fsd.getBlockStoragePolicySuite();
srcChild = this.srcIIP.getLastINode();
srcChildName = srcChild.getLocalNameBytes();
final int srcLatestSnapshotId = srcIIP.getLatestSnapshotId();
@@ -598,7 +600,7 @@ class FSDirRenameOp {
// check srcChild for reference
srcRefDstSnapshot = srcChildIsReference ?
srcChild.asReference().getDstSnapshotId() : Snapshot.CURRENT_STATE_ID;
- oldSrcCounts = Quota.Counts.newInstance();
+ oldSrcCounts = new QuotaCounts.Builder().build();
if (isSrcInSnapshot) {
final INodeReference.WithName withName = srcParent
.replaceChild4ReferenceWithName(srcChild, srcLatestSnapshotId);
@@ -607,7 +609,7 @@ class FSDirRenameOp {
this.srcIIP = INodesInPath.replace(srcIIP, srcIIP.length() - 1,
srcChild);
// get the counts before rename
- withCount.getReferredINode().computeQuotaUsage(oldSrcCounts, true);
+ withCount.getReferredINode().computeQuotaUsage(bsps, oldSrcCounts, true);
} else if (srcChildIsReference) {
// srcChild is reference but srcChild is not in latest snapshot
withCount = (INodeReference.WithCount) srcChild.asReference()
@@ -709,11 +711,11 @@ class FSDirRenameOp {
}
}
- void restoreDst() throws QuotaExceededException {
+ void restoreDst(BlockStoragePolicySuite bsps) throws QuotaExceededException {
Preconditions.checkState(oldDstChild != null);
final INodeDirectory dstParent = dstParentIIP.getLastINode().asDirectory();
if (dstParent.isWithSnapshot()) {
- dstParent.undoRename4DstParent(oldDstChild, dstIIP.getLatestSnapshotId());
+ dstParent.undoRename4DstParent(bsps, oldDstChild, dstIIP.getLatestSnapshotId());
} else {
fsd.addLastINodeNoQuotaCheck(dstParentIIP, oldDstChild);
}
@@ -725,32 +727,31 @@ class FSDirRenameOp {
}
}
- boolean cleanDst(BlocksMapUpdateInfo collectedBlocks)
+ boolean cleanDst(BlockStoragePolicySuite bsps, BlocksMapUpdateInfo collectedBlocks)
throws QuotaExceededException {
Preconditions.checkState(oldDstChild != null);
List<INode> removedINodes = new ChunkedArrayList<>();
final boolean filesDeleted;
if (!oldDstChild.isInLatestSnapshot(dstIIP.getLatestSnapshotId())) {
- oldDstChild.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ oldDstChild.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
filesDeleted = true;
} else {
- filesDeleted = oldDstChild.cleanSubtree(Snapshot.CURRENT_STATE_ID,
+ filesDeleted = oldDstChild.cleanSubtree(bsps, Snapshot.CURRENT_STATE_ID,
dstIIP.getLatestSnapshotId(), collectedBlocks, removedINodes)
- .get(Quota.NAMESPACE) >= 0;
+ .getNameSpace() >= 0;
}
fsd.getFSNamesystem().removeLeasesAndINodes(src, removedINodes, false);
return filesDeleted;
}
- void updateQuotasInSourceTree() throws QuotaExceededException {
+ void updateQuotasInSourceTree(BlockStoragePolicySuite bsps) throws QuotaExceededException {
// update the quota usage in src tree
if (isSrcInSnapshot) {
// get the counts after rename
- Quota.Counts newSrcCounts = srcChild.computeQuotaUsage(
- Quota.Counts.newInstance(), false);
+ QuotaCounts newSrcCounts = srcChild.computeQuotaUsage(bsps,
+ new QuotaCounts.Builder().build(), false);
newSrcCounts.subtract(oldSrcCounts);
- srcParent.addSpaceConsumed(newSrcCounts.get(Quota.NAMESPACE),
- newSrcCounts.get(Quota.DISKSPACE), false);
+ srcParent.addSpaceConsumed(newSrcCounts, false);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 7450249..7f1437d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -43,6 +43,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.AclException;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
import org.apache.hadoop.hdfs.protocol.FSLimitException.PathComponentTooLongException;
@@ -56,9 +57,12 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguousUnderConstruction;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.util.ByteArray;
+import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
@@ -76,6 +80,8 @@ import java.util.concurrent.locks.ReentrantReadWriteLock;
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_ACCESSTIME_PRECISION_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_STORAGE_POLICY_ENABLED_KEY;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
@@ -100,8 +106,9 @@ public class FSDirectory implements Closeable {
namesystem.createFsOwnerPermissions(new FsPermission((short) 0755)),
0L);
r.addDirectoryWithQuotaFeature(
- DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA,
- DirectoryWithQuotaFeature.DEFAULT_DISKSPACE_QUOTA);
+ new DirectoryWithQuotaFeature.Builder().
+ nameSpaceQuota(DirectoryWithQuotaFeature.DEFAULT_NAMESPACE_QUOTA).
+ spaceQuota(DirectoryWithQuotaFeature.DEFAULT_SPACE_QUOTA).build());
r.addSnapshottableFeature();
r.setSnapshotQuota(0);
return r;
@@ -149,6 +156,8 @@ public class FSDirectory implements Closeable {
private final long accessTimePrecision;
// whether setStoragePolicy is allowed.
private final boolean storagePolicyEnabled;
+ // whether quota by storage type is allowed
+ private final boolean quotaByStorageTypeEnabled;
private final String fsOwnerShortUserName;
private final String supergroup;
@@ -236,6 +245,10 @@ public class FSDirectory implements Closeable {
conf.getBoolean(DFS_STORAGE_POLICY_ENABLED_KEY,
DFS_STORAGE_POLICY_ENABLED_DEFAULT);
+ this.quotaByStorageTypeEnabled =
+ conf.getBoolean(DFS_QUOTA_BY_STORAGETYPE_ENABLED_KEY,
+ DFS_QUOTA_BY_STORAGETYPE_ENABLED_DEFAULT);
+
int configuredLimit = conf.getInt(
DFSConfigKeys.DFS_LIST_LIMIT, DFSConfigKeys.DFS_LIST_LIMIT_DEFAULT);
this.lsLimit = configuredLimit>0 ?
@@ -291,6 +304,10 @@ public class FSDirectory implements Closeable {
return rootDir;
}
+ public BlockStoragePolicySuite getBlockStoragePolicySuite() {
+ return getBlockManager().getStoragePolicySuite();
+ }
+
boolean isPermissionEnabled() {
return isPermissionEnabled;
}
@@ -307,6 +324,9 @@ public class FSDirectory implements Closeable {
boolean isAccessTimeSupported() {
return accessTimePrecision > 0;
}
+ boolean isQuotaByStorageTypeEnabled() {
+ return quotaByStorageTypeEnabled;
+ }
int getLsLimit() {
@@ -452,7 +472,8 @@ public class FSDirectory implements Closeable {
Preconditions.checkState(fileINode.isUnderConstruction());
// check quota limits and updated space consumed
- updateCount(inodesInPath, 0, fileINode.getPreferredBlockDiskspace(), true);
+ updateCount(inodesInPath, 0, fileINode.getPreferredBlockSize(),
+ fileINode.getBlockReplication(), true);
// associate new last block for the file
BlockInfoContiguousUnderConstruction blockInfo =
@@ -508,7 +529,8 @@ public class FSDirectory implements Closeable {
}
// update space consumed
- updateCount(iip, 0, -fileNode.getPreferredBlockDiskspace(), true);
+ updateCount(iip, 0, -fileNode.getPreferredBlockSize(),
+ fileNode.getBlockReplication(), true);
return true;
}
@@ -584,19 +606,20 @@ public class FSDirectory implements Closeable {
* @param iip the INodesInPath instance containing all the INodes for
* updating quota usage
* @param nsDelta the delta change of namespace
- * @param dsDelta the delta change of diskspace
+ * @param dsDelta the delta change of space consumed without replication
+ * @param replication the replication factor of the block consumption change
* @throws QuotaExceededException if the new count violates any quota limit
* @throws FileNotFoundException if path does not exist.
*/
- void updateSpaceConsumed(INodesInPath iip, long nsDelta, long dsDelta)
- throws QuotaExceededException, FileNotFoundException,
- UnresolvedLinkException, SnapshotAccessControlException {
+ void updateSpaceConsumed(INodesInPath iip, long nsDelta, long dsDelta, short replication)
+ throws QuotaExceededException, FileNotFoundException,
+ UnresolvedLinkException, SnapshotAccessControlException {
writeLock();
try {
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
}
- updateCount(iip, nsDelta, dsDelta, true);
+ updateCount(iip, nsDelta, dsDelta, replication, true);
} finally {
writeUnlock();
}
@@ -610,29 +633,52 @@ public class FSDirectory implements Closeable {
void updateCountForDelete(final INode inode, final INodesInPath iip) {
if (getFSNamesystem().isImageLoaded() &&
!inode.isInLatestSnapshot(iip.getLatestSnapshotId())) {
- Quota.Counts counts = inode.computeQuotaUsage();
- unprotectedUpdateCount(iip, iip.length() - 1,
- -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+ QuotaCounts counts = inode.computeQuotaUsage(getBlockStoragePolicySuite());
+ unprotectedUpdateCount(iip, iip.length() - 1, counts.negation());
}
}
- void updateCount(INodesInPath iip, long nsDelta, long dsDelta,
+ /**
+ * Update usage count without replication factor change
+ */
+ void updateCount(INodesInPath iip, long nsDelta, long dsDelta, short replication,
boolean checkQuota) throws QuotaExceededException {
- updateCount(iip, iip.length() - 1, nsDelta, dsDelta, checkQuota);
+ final INodeFile fileINode = iip.getLastINode().asFile();
+ EnumCounters<StorageType> typeSpaceDeltas =
+ getStorageTypeDeltas(fileINode.getStoragePolicyID(), dsDelta,
+ replication, replication);;
+ updateCount(iip, iip.length() - 1,
+ new QuotaCounts.Builder().nameCount(nsDelta).spaceCount(dsDelta * replication).
+ typeCounts(typeSpaceDeltas).build(),
+ checkQuota);
+ }
+
+ /**
+ * Update usage count with replication factor change due to setReplication
+ */
+ void updateCount(INodesInPath iip, long nsDelta, long dsDelta, short oldRep,
+ short newRep, boolean checkQuota) throws QuotaExceededException {
+ final INodeFile fileINode = iip.getLastINode().asFile();
+ EnumCounters<StorageType> typeSpaceDeltas =
+ getStorageTypeDeltas(fileINode.getStoragePolicyID(), dsDelta, oldRep, newRep);
+ updateCount(iip, iip.length() - 1,
+ new QuotaCounts.Builder().nameCount(nsDelta).
+ spaceCount(dsDelta * (newRep - oldRep)).
+ typeCounts(typeSpaceDeltas).build(),
+ checkQuota);
}
/** update count of each inode with quota
*
* @param iip inodes in a path
* @param numOfINodes the number of inodes to update starting from index 0
- * @param nsDelta the delta change of namespace
- * @param dsDelta the delta change of diskspace
+ * @param counts the count of space/namespace/type usage to be update
* @param checkQuota if true then check if quota is exceeded
* @throws QuotaExceededException if the new count violates any quota limit
*/
- private void updateCount(INodesInPath iip, int numOfINodes,
- long nsDelta, long dsDelta, boolean checkQuota)
- throws QuotaExceededException {
+ void updateCount(INodesInPath iip, int numOfINodes,
+ QuotaCounts counts, boolean checkQuota)
+ throws QuotaExceededException {
assert hasWriteLock();
if (!namesystem.isImageLoaded()) {
//still initializing. do not check or update quotas.
@@ -642,20 +688,20 @@ public class FSDirectory implements Closeable {
numOfINodes = iip.length();
}
if (checkQuota && !skipQuotaCheck) {
- verifyQuota(iip, numOfINodes, nsDelta, dsDelta, null);
+ verifyQuota(iip, numOfINodes, counts, null);
}
- unprotectedUpdateCount(iip, numOfINodes, nsDelta, dsDelta);
+ unprotectedUpdateCount(iip, numOfINodes, counts);
}
/**
* update quota of each inode and check to see if quota is exceeded.
- * See {@link #updateCount(INodesInPath, long, long, boolean)}
+ * See {@link #updateCount(INodesInPath, int, QuotaCounts, boolean)}
*/
- void updateCountNoQuotaCheck(INodesInPath inodesInPath, int numOfINodes,
- long nsDelta, long dsDelta) {
+ void updateCountNoQuotaCheck(INodesInPath inodesInPath,
+ int numOfINodes, QuotaCounts counts) {
assert hasWriteLock();
try {
- updateCount(inodesInPath, numOfINodes, nsDelta, dsDelta, false);
+ updateCount(inodesInPath, numOfINodes, counts, false);
} catch (QuotaExceededException e) {
NameNode.LOG.error("BUG: unexpected exception ", e);
}
@@ -666,15 +712,49 @@ public class FSDirectory implements Closeable {
* callers responsibility is to make sure quota is not exceeded
*/
static void unprotectedUpdateCount(INodesInPath inodesInPath,
- int numOfINodes, long nsDelta, long dsDelta) {
+ int numOfINodes, QuotaCounts counts) {
for(int i=0; i < numOfINodes; i++) {
if (inodesInPath.getINode(i).isQuotaSet()) { // a directory with quota
inodesInPath.getINode(i).asDirectory().getDirectoryWithQuotaFeature()
- .addSpaceConsumed2Cache(nsDelta, dsDelta);
+ .addSpaceConsumed2Cache(counts);
}
}
}
-
+
+ public EnumCounters<StorageType> getStorageTypeDeltas(byte storagePolicyID,
+ long dsDelta, short oldRep, short newRep) {
+ EnumCounters<StorageType> typeSpaceDeltas =
+ new EnumCounters<StorageType>(StorageType.class);
+ // Storage type and its quota are only available when storage policy is set
+ if (storagePolicyID != BlockStoragePolicySuite.ID_UNSPECIFIED) {
+ BlockStoragePolicy storagePolicy = getBlockManager().getStoragePolicy(storagePolicyID);
+
+ if (oldRep != newRep) {
+ List<StorageType> oldChosenStorageTypes =
+ storagePolicy.chooseStorageTypes(oldRep);
+
+ for (StorageType t : oldChosenStorageTypes) {
+ if (!t.supportTypeQuota()) {
+ continue;
+ }
+ Preconditions.checkArgument(dsDelta > 0);
+ typeSpaceDeltas.add(t, -dsDelta);
+ }
+ }
+
+ List<StorageType> newChosenStorageTypes =
+ storagePolicy.chooseStorageTypes(newRep);
+
+ for (StorageType t : newChosenStorageTypes) {
+ if (!t.supportTypeQuota()) {
+ continue;
+ }
+ typeSpaceDeltas.add(t, dsDelta);
+ }
+ }
+ return typeSpaceDeltas;
+ }
+
/** Return the name of the path represented by inodes at [0, pos] */
static String getFullPathName(INode[] inodes, int pos) {
StringBuilder fullPathName = new StringBuilder();
@@ -751,16 +831,16 @@ public class FSDirectory implements Closeable {
*
* @param iip INodes corresponding to a path
* @param pos position where a new INode will be added
- * @param nsDelta needed namespace
- * @param dsDelta needed diskspace
+ * @param deltas needed namespace, diskspace and storage types
* @param commonAncestor Last node in inodes array that is a common ancestor
* for a INode that is being moved from one location to the other.
* Pass null if a node is not being moved.
* @throws QuotaExceededException if quota limit is exceeded.
*/
- static void verifyQuota(INodesInPath iip, int pos, long nsDelta,
- long dsDelta, INode commonAncestor) throws QuotaExceededException {
- if (nsDelta <= 0 && dsDelta <= 0) {
+ static void verifyQuota(INodesInPath iip, int pos, QuotaCounts deltas,
+ INode commonAncestor) throws QuotaExceededException {
+ if (deltas.getNameSpace() <= 0 && deltas.getDiskSpace() <= 0
+ && deltas.getTypeSpaces().allLessOrEqual(0L)) {
// if quota is being freed or not being consumed
return;
}
@@ -775,7 +855,7 @@ public class FSDirectory implements Closeable {
= iip.getINode(i).asDirectory().getDirectoryWithQuotaFeature();
if (q != null) { // a directory with quota
try {
- q.verifyQuota(nsDelta, dsDelta);
+ q.verifyQuota(deltas);
} catch (QuotaExceededException e) {
List<INode> inodes = iip.getReadOnlyINodes();
final String path = getFullPathName(inodes.toArray(new INode[inodes.size()]), i);
@@ -845,7 +925,7 @@ public class FSDirectory implements Closeable {
}
}
}
-
+
/**
* Add a child to the end of the path specified by INodesInPath.
* @return an INodesInPath instance containing the new INode
@@ -873,7 +953,7 @@ public class FSDirectory implements Closeable {
// odd. It's because a rename operation deletes the src, tries to add
// to the dest, if that fails, re-adds the src from whence it came.
// The rename code disables the quota when it's restoring to the
- // original location becase a quota violation would cause the the item
+ // original location because a quota violation would cause the the item
// to go "poof". The fs limits must be bypassed for the same reason.
if (checkQuota) {
final String parentPath = existing.getPath(pos - 1);
@@ -883,21 +963,19 @@ public class FSDirectory implements Closeable {
// always verify inode name
verifyINodeName(inode.getLocalNameBytes());
- final Quota.Counts counts = inode.computeQuotaUsage();
- updateCount(existing, pos,
- counts.get(Quota.NAMESPACE), counts.get(Quota.DISKSPACE), checkQuota);
+ final QuotaCounts counts = inode.computeQuotaUsage(getBlockStoragePolicySuite());
+ updateCount(existing, pos, counts, checkQuota);
+
boolean isRename = (inode.getParent() != null);
boolean added;
try {
added = parent.addChild(inode, true, existing.getLatestSnapshotId());
} catch (QuotaExceededException e) {
- updateCountNoQuotaCheck(existing, pos,
- -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+ updateCountNoQuotaCheck(existing, pos, counts.negation());
throw e;
}
if (!added) {
- updateCountNoQuotaCheck(existing, pos,
- -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+ updateCountNoQuotaCheck(existing, pos, counts.negation());
return null;
} else {
if (!isRename) {
@@ -1023,12 +1101,13 @@ public class FSDirectory implements Closeable {
INodeFile file = iip.getLastINode().asFile();
int latestSnapshot = iip.getLatestSnapshotId();
file.recordModification(latestSnapshot, true);
- long oldDiskspace = file.diskspaceConsumed();
+ long oldDiskspaceNoRep = file.diskspaceConsumedNoReplication();
long remainingLength =
file.collectBlocksBeyondMax(newLength, collectedBlocks);
file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
file.setModificationTime(mtime);
- updateCount(iip, 0, file.diskspaceConsumed() - oldDiskspace, true);
+ updateCount(iip, 0, file.diskspaceConsumedNoReplication() - oldDiskspaceNoRep,
+ file.getBlockReplication(), true);
// return whether on a block boundary
return (remainingLength - newLength) == 0;
}
@@ -1102,7 +1181,7 @@ public class FSDirectory implements Closeable {
readLock();
try {
return rootDir.getDirectoryWithQuotaFeature().getSpaceConsumed()
- .get(Quota.NAMESPACE);
+ .getNameSpace();
} finally {
readUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
index 9ce3fa9..7675703 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
@@ -84,6 +84,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetGenstampV2Op;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetOwnerOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetPermissionsOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetQuotaOp;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetQuotaByStorageTypeOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetReplicationOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetStoragePolicyOp;
import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp.SetXAttrOp;
@@ -97,6 +98,7 @@ import org.apache.hadoop.hdfs.server.namenode.metrics.NameNodeMetrics;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.security.token.delegation.DelegationKey;
@@ -872,6 +874,14 @@ public class FSEditLog implements LogsPurgeable {
logEdit(op);
}
+ /** Add set quota by storage type record to edit log */
+ void logSetQuotaByStorageType(String src, long dsQuota, StorageType type) {
+ SetQuotaByStorageTypeOp op = SetQuotaByStorageTypeOp.getInstance(cache.get())
+ .setSource(src)
+ .setQuotaByStorageType(dsQuota, type);
+ logEdit(op);
+ }
+
/** Add set permissions record to edit log */
void logSetPermissions(String src, FsPermission permissions) {
SetPermissionsOp op = SetPermissionsOp.getInstance(cache.get())
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index 5fcad74..a09df82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -595,14 +595,14 @@ public class FSEditLogLoader {
SetNSQuotaOp setNSQuotaOp = (SetNSQuotaOp)op;
FSDirAttrOp.unprotectedSetQuota(
fsDir, renameReservedPathsOnUpgrade(setNSQuotaOp.src, logVersion),
- setNSQuotaOp.nsQuota, HdfsConstants.QUOTA_DONT_SET);
+ setNSQuotaOp.nsQuota, HdfsConstants.QUOTA_DONT_SET, null);
break;
}
case OP_CLEAR_NS_QUOTA: {
ClearNSQuotaOp clearNSQuotaOp = (ClearNSQuotaOp)op;
FSDirAttrOp.unprotectedSetQuota(
fsDir, renameReservedPathsOnUpgrade(clearNSQuotaOp.src, logVersion),
- HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_DONT_SET);
+ HdfsConstants.QUOTA_RESET, HdfsConstants.QUOTA_DONT_SET, null);
break;
}
@@ -610,9 +610,18 @@ public class FSEditLogLoader {
SetQuotaOp setQuotaOp = (SetQuotaOp) op;
FSDirAttrOp.unprotectedSetQuota(fsDir,
renameReservedPathsOnUpgrade(setQuotaOp.src, logVersion),
- setQuotaOp.nsQuota, setQuotaOp.dsQuota);
+ setQuotaOp.nsQuota, setQuotaOp.dsQuota, null);
break;
+ case OP_SET_QUOTA_BY_STORAGETYPE:
+ FSEditLogOp.SetQuotaByStorageTypeOp setQuotaByStorageTypeOp =
+ (FSEditLogOp.SetQuotaByStorageTypeOp) op;
+ FSDirAttrOp.unprotectedSetQuota(fsDir,
+ renameReservedPathsOnUpgrade(setQuotaByStorageTypeOp.src, logVersion),
+ HdfsConstants.QUOTA_DONT_SET, setQuotaByStorageTypeOp.dsQuota,
+ setQuotaByStorageTypeOp.type);
+ break;
+
case OP_TIMES: {
TimesOp timesOp = (TimesOp)op;
FSDirAttrOp.unprotectedSetTimes(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
index 1629d80..dab10d3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOp.java
@@ -64,6 +64,7 @@ import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_TRUNCAT
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_UPDATE_BLOCKS;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_UPDATE_MASTER_KEY;
import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SET_STORAGE_POLICY;
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.OP_SET_QUOTA_BY_STORAGETYPE;
import java.io.DataInput;
import java.io.DataInputStream;
@@ -106,6 +107,7 @@ import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.XAttrEditLogProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.hdfs.util.XMLUtils.InvalidXmlException;
import org.apache.hadoop.hdfs.util.XMLUtils.Stanza;
@@ -209,6 +211,7 @@ public abstract class FSEditLogOp {
inst.put(OP_REMOVE_XATTR, new RemoveXAttrOp());
inst.put(OP_SET_STORAGE_POLICY, new SetStoragePolicyOp());
inst.put(OP_APPEND, new AppendOp());
+ inst.put(OP_SET_QUOTA_BY_STORAGETYPE, new SetQuotaByStorageTypeOp());
}
public FSEditLogOp get(FSEditLogOpCodes opcode) {
@@ -2267,6 +2270,88 @@ public abstract class FSEditLogOp {
}
}
+ /** {@literal @Idempotent} for {@link ClientProtocol#setQuota} */
+ static class SetQuotaByStorageTypeOp extends FSEditLogOp {
+ String src;
+ long dsQuota;
+ StorageType type;
+
+ private SetQuotaByStorageTypeOp() {
+ super(OP_SET_QUOTA_BY_STORAGETYPE);
+ }
+
+ static SetQuotaByStorageTypeOp getInstance(OpInstanceCache cache) {
+ return (SetQuotaByStorageTypeOp)cache.get(OP_SET_QUOTA_BY_STORAGETYPE);
+ }
+
+ @Override
+ void resetSubFields() {
+ src = null;
+ dsQuota = -1L;
+ type = StorageType.DEFAULT;
+ }
+
+ SetQuotaByStorageTypeOp setSource(String src) {
+ this.src = src;
+ return this;
+ }
+
+ SetQuotaByStorageTypeOp setQuotaByStorageType(long dsQuota, StorageType type) {
+ this.type = type;
+ this.dsQuota = dsQuota;
+ return this;
+ }
+
+ @Override
+ public
+ void writeFields(DataOutputStream out) throws IOException {
+ FSImageSerialization.writeString(src, out);
+ FSImageSerialization.writeInt(type.ordinal(), out);
+ FSImageSerialization.writeLong(dsQuota, out);
+ }
+
+ @Override
+ void readFields(DataInputStream in, int logVersion)
+ throws IOException {
+ this.src = FSImageSerialization.readString(in);
+ this.type = StorageType.parseStorageType(FSImageSerialization.readInt(in));
+ this.dsQuota = FSImageSerialization.readLong(in);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder builder = new StringBuilder();
+ builder.append("SetTypeQuotaOp [src=");
+ builder.append(src);
+ builder.append(", storageType=");
+ builder.append(type);
+ builder.append(", dsQuota=");
+ builder.append(dsQuota);
+ builder.append(", opCode=");
+ builder.append(opCode);
+ builder.append(", txid=");
+ builder.append(txid);
+ builder.append("]");
+ return builder.toString();
+ }
+
+ @Override
+ protected void toXml(ContentHandler contentHandler) throws SAXException {
+ XMLUtils.addSaxString(contentHandler, "SRC", src);
+ XMLUtils.addSaxString(contentHandler, "STORAGETYPE",
+ Integer.toString(type.ordinal()));
+ XMLUtils.addSaxString(contentHandler, "DSQUOTA",
+ Long.toString(dsQuota));
+ }
+
+ @Override void fromXml(Stanza st) throws InvalidXmlException {
+ this.src = st.getValue("SRC");
+ this.type = StorageType.parseStorageType(
+ Integer.parseInt(st.getValue("STORAGETYPE")));
+ this.dsQuota = Long.parseLong(st.getValue("DSQUOTA"));
+ }
+ }
+
/** {@literal @Idempotent} for {@link ClientProtocol#setTimes} */
static class TimesOp extends FSEditLogOp {
int length;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
index 6cd1617..1a0a296 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
@@ -75,6 +75,7 @@ public enum FSEditLogOpCodes {
OP_SET_STORAGE_POLICY ((byte) 45),
OP_TRUNCATE ((byte) 46),
OP_APPEND ((byte) 47),
+ OP_SET_QUOTA_BY_STORAGETYPE ((byte) 48),
// Note that the current range of the valid OP code is 0~127
OP_INVALID ((byte) -1);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index 3b5d2c3..c630917 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HAUtil;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.LayoutVersion;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.NamenodeRole;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.RollingUpgradeStartupOption;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
@@ -65,12 +66,13 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeCommand;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocol;
import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.util.Canceler;
+import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.MD5FileUtils;
import org.apache.hadoop.io.MD5Hash;
import org.apache.hadoop.util.Time;
-import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -841,7 +843,8 @@ public class FSImage implements Closeable {
} finally {
FSEditLog.closeAllStreams(editStreams);
// update the counts
- updateCountForQuota(target.dir.rootDir);
+ updateCountForQuota(target.getBlockManager().getStoragePolicySuite(),
+ target.dir.rootDir);
}
prog.endPhase(Phase.LOADING_EDITS);
return lastAppliedTxId - prevLastAppliedTxId;
@@ -855,47 +858,64 @@ public class FSImage implements Closeable {
* This is an update of existing state of the filesystem and does not
* throw QuotaExceededException.
*/
- static void updateCountForQuota(INodeDirectory root) {
- updateCountForQuotaRecursively(root, Quota.Counts.newInstance());
- }
-
- private static void updateCountForQuotaRecursively(INodeDirectory dir,
- Quota.Counts counts) {
- final long parentNamespace = counts.get(Quota.NAMESPACE);
- final long parentDiskspace = counts.get(Quota.DISKSPACE);
-
- dir.computeQuotaUsage4CurrentDirectory(counts);
+ static void updateCountForQuota(BlockStoragePolicySuite bsps,
+ INodeDirectory root) {
+ updateCountForQuotaRecursively(bsps, root, new QuotaCounts.Builder().build());
+ }
+
+ private static void updateCountForQuotaRecursively(BlockStoragePolicySuite bsps,
+ INodeDirectory dir, QuotaCounts counts) {
+ final long parentNamespace = counts.getNameSpace();
+ final long parentDiskspace = counts.getDiskSpace();
+ final EnumCounters<StorageType> parentTypeSpaces = counts.getTypeSpaces();
+
+ dir.computeQuotaUsage4CurrentDirectory(bsps, counts);
for (INode child : dir.getChildrenList(Snapshot.CURRENT_STATE_ID)) {
if (child.isDirectory()) {
- updateCountForQuotaRecursively(child.asDirectory(), counts);
+ updateCountForQuotaRecursively(bsps, child.asDirectory(), counts);
} else {
// file or symlink: count here to reduce recursive calls.
- child.computeQuotaUsage(counts, false);
+ child.computeQuotaUsage(bsps, counts, false);
}
}
if (dir.isQuotaSet()) {
// check if quota is violated. It indicates a software bug.
- final Quota.Counts q = dir.getQuotaCounts();
+ final QuotaCounts q = dir.getQuotaCounts();
- final long namespace = counts.get(Quota.NAMESPACE) - parentNamespace;
- final long nsQuota = q.get(Quota.NAMESPACE);
+ final long namespace = counts.getNameSpace() - parentNamespace;
+ final long nsQuota = q.getNameSpace();
if (Quota.isViolated(nsQuota, namespace)) {
LOG.error("BUG: Namespace quota violation in image for "
+ dir.getFullPathName()
+ " quota = " + nsQuota + " < consumed = " + namespace);
}
- final long diskspace = counts.get(Quota.DISKSPACE) - parentDiskspace;
- final long dsQuota = q.get(Quota.DISKSPACE);
+ final long diskspace = counts.getDiskSpace() - parentDiskspace;
+ final long dsQuota = q.getDiskSpace();
if (Quota.isViolated(dsQuota, diskspace)) {
LOG.error("BUG: Diskspace quota violation in image for "
+ dir.getFullPathName()
+ " quota = " + dsQuota + " < consumed = " + diskspace);
}
- dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, diskspace);
+ final EnumCounters<StorageType> typeSpaces =
+ new EnumCounters<StorageType>(StorageType.class);
+ for (StorageType t : StorageType.getTypesSupportingQuota()) {
+ final long typeSpace = counts.getTypeSpaces().get(t) -
+ parentTypeSpaces.get(t);
+ final long typeQuota = q.getTypeSpaces().get(t);
+ if (Quota.isViolated(typeQuota, typeSpace)) {
+ LOG.error("BUG Disk quota by storage type violation in image for "
+ + dir.getFullPathName()
+ + " type = " + t.toString() + " quota = "
+ + typeQuota + " < consumed " + typeSpace);
+ }
+ }
+
+ dir.getDirectoryWithQuotaFeature().setSpaceConsumed(namespace, diskspace,
+ typeSpaces);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
index ed05d40..e4b86ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormat.java
@@ -447,9 +447,9 @@ public class FSImageFormat {
/** Update the root node's attributes */
private void updateRootAttr(INodeWithAdditionalFields root) {
- final Quota.Counts q = root.getQuotaCounts();
- final long nsQuota = q.get(Quota.NAMESPACE);
- final long dsQuota = q.get(Quota.DISKSPACE);
+ final QuotaCounts q = root.getQuotaCounts();
+ final long nsQuota = q.getNameSpace();
+ final long dsQuota = q.getDiskSpace();
FSDirectory fsDir = namesystem.dir;
if (nsQuota != -1 || dsQuota != -1) {
fsDir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
@@ -823,7 +823,8 @@ public class FSImageFormat {
final INodeDirectory dir = new INodeDirectory(inodeId, localName,
permissions, modificationTime);
if (nsQuota >= 0 || dsQuota >= 0) {
- dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
+ dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
+ nameSpaceQuota(nsQuota).spaceQuota(dsQuota).build());
}
if (withSnapshot) {
dir.addSnapshotFeature(null);
@@ -907,11 +908,11 @@ public class FSImageFormat {
//read quotas
final long nsQuota = in.readLong();
final long dsQuota = in.readLong();
-
+
return nsQuota == -1L && dsQuota == -1L ? new INodeDirectoryAttributes.SnapshotCopy(
name, permissions, null, modificationTime, null)
: new INodeDirectoryAttributes.CopyWithQuota(name, permissions,
- null, modificationTime, nsQuota, dsQuota, null);
+ null, modificationTime, nsQuota, dsQuota, null, null);
}
private void loadFilesUnderConstruction(DataInput in,
@@ -1229,7 +1230,7 @@ public class FSImageFormat {
final FSNamesystem sourceNamesystem = context.getSourceNamesystem();
final INodeDirectory rootDir = sourceNamesystem.dir.rootDir;
final long numINodes = rootDir.getDirectoryWithQuotaFeature()
- .getSpaceConsumed().get(Quota.NAMESPACE);
+ .getSpaceConsumed().getNameSpace();
String sdPath = newFile.getParentFile().getParentFile().getAbsolutePath();
Step step = new Step(StepType.INODES, sdPath);
StartupProgress prog = NameNode.getStartupProgress();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index f20add1..88b1fa2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -37,7 +37,9 @@ import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -52,7 +54,10 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.AclFeatureProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrCompactProto;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.XAttrFeatureProto;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeEntryProto;
+import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection.QuotaByStorageTypeFeatureProto;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import com.google.common.base.Preconditions;
@@ -141,6 +146,18 @@ public final class FSImageFormatPBINode {
return b.build();
}
+ public static ImmutableList<QuotaByStorageTypeEntry> loadQuotaByStorageTypeEntries(
+ QuotaByStorageTypeFeatureProto proto) {
+ ImmutableList.Builder<QuotaByStorageTypeEntry> b = ImmutableList.builder();
+ for (QuotaByStorageTypeEntryProto quotaEntry : proto.getQuotasList()) {
+ StorageType type = PBHelper.convertStorageType(quotaEntry.getStorageType());
+ long quota = quotaEntry.getQuota();
+ b.add(new QuotaByStorageTypeEntry.Builder().setStorageType(type)
+ .setQuota(quota).build());
+ }
+ return b.build();
+ }
+
public static INodeDirectory loadINodeDirectory(INodeSection.INode n,
LoaderContext state) {
assert n.getType() == INodeSection.INode.Type.DIRECTORY;
@@ -150,10 +167,33 @@ public final class FSImageFormatPBINode {
state.getStringTable());
final INodeDirectory dir = new INodeDirectory(n.getId(), n.getName()
.toByteArray(), permissions, d.getModificationTime());
-
final long nsQuota = d.getNsQuota(), dsQuota = d.getDsQuota();
if (nsQuota >= 0 || dsQuota >= 0) {
- dir.addDirectoryWithQuotaFeature(nsQuota, dsQuota);
+ dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.Builder().
+ nameSpaceQuota(nsQuota).spaceQuota(dsQuota).build());
+ }
+ EnumCounters<StorageType> typeQuotas = null;
+ if (d.hasTypeQuotas()) {
+ ImmutableList<QuotaByStorageTypeEntry> qes =
+ loadQuotaByStorageTypeEntries(d.getTypeQuotas());
+ typeQuotas = new EnumCounters<StorageType>(StorageType.class,
+ HdfsConstants.QUOTA_RESET);
+ for (QuotaByStorageTypeEntry qe : qes) {
+ if (qe.getQuota() >= 0 && qe.getStorageType() != null &&
+ qe.getStorageType().supportTypeQuota()) {
+ typeQuotas.set(qe.getStorageType(), qe.getQuota());
+ }
+ }
+
+ if (typeQuotas.anyGreaterOrEqual(0)) {
+ DirectoryWithQuotaFeature q = dir.getDirectoryWithQuotaFeature();
+ if (q == null) {
+ dir.addDirectoryWithQuotaFeature(new DirectoryWithQuotaFeature.
+ Builder().typeQuotas(typeQuotas).build());
+ } else {
+ q.setQuota(typeQuotas);
+ }
+ }
}
if (d.hasAcl()) {
@@ -335,12 +375,16 @@ public final class FSImageFormatPBINode {
private void loadRootINode(INodeSection.INode p) {
INodeDirectory root = loadINodeDirectory(p, parent.getLoaderContext());
- final Quota.Counts q = root.getQuotaCounts();
- final long nsQuota = q.get(Quota.NAMESPACE);
- final long dsQuota = q.get(Quota.DISKSPACE);
+ final QuotaCounts q = root.getQuotaCounts();
+ final long nsQuota = q.getNameSpace();
+ final long dsQuota = q.getDiskSpace();
if (nsQuota != -1 || dsQuota != -1) {
dir.rootDir.getDirectoryWithQuotaFeature().setQuota(nsQuota, dsQuota);
}
+ final EnumCounters<StorageType> typeQuotas = q.getTypeSpaces();
+ if (typeQuotas.anyGreaterOrEqual(0)) {
+ dir.rootDir.getDirectoryWithQuotaFeature().setQuota(typeQuotas);
+ }
dir.rootDir.cloneModificationTime(root);
dir.rootDir.clonePermissionStatus(root);
// root dir supports having extended attributes according to POSIX
@@ -399,6 +443,22 @@ public final class FSImageFormatPBINode {
return b;
}
+ private static QuotaByStorageTypeFeatureProto.Builder
+ buildQuotaByStorageTypeEntries(QuotaCounts q) {
+ QuotaByStorageTypeFeatureProto.Builder b =
+ QuotaByStorageTypeFeatureProto.newBuilder();
+ for (StorageType t: StorageType.getTypesSupportingQuota()) {
+ if (q.getTypeSpace(t) >= 0) {
+ QuotaByStorageTypeEntryProto.Builder eb =
+ QuotaByStorageTypeEntryProto.newBuilder().
+ setStorageType(PBHelper.convertStorageType(t)).
+ setQuota(q.getTypeSpace(t));
+ b.addQuotas(eb);
+ }
+ }
+ return b;
+ }
+
public static INodeSection.INodeFile.Builder buildINodeFile(
INodeFileAttributes file, final SaverContext state) {
INodeSection.INodeFile.Builder b = INodeSection.INodeFile.newBuilder()
@@ -422,13 +482,17 @@ public final class FSImageFormatPBINode {
public static INodeSection.INodeDirectory.Builder buildINodeDirectory(
INodeDirectoryAttributes dir, final SaverContext state) {
- Quota.Counts quota = dir.getQuotaCounts();
+ QuotaCounts quota = dir.getQuotaCounts();
INodeSection.INodeDirectory.Builder b = INodeSection.INodeDirectory
.newBuilder().setModificationTime(dir.getModificationTime())
- .setNsQuota(quota.get(Quota.NAMESPACE))
- .setDsQuota(quota.get(Quota.DISKSPACE))
+ .setNsQuota(quota.getNameSpace())
+ .setDsQuota(quota.getDiskSpace())
.setPermission(buildPermissionStatus(dir, state.getStringMap()));
+ if (quota.getTypeSpaces().anyGreaterOrEqual(0)) {
+ b.setTypeQuotas(buildQuotaByStorageTypeEntries(quota));
+ }
+
AclFeature f = dir.getAclFeature();
if (f != null) {
b.setAcl(buildAclEntries(f, state.getStringMap()));
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
index 33f644d..9721f66 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageSerialization.java
@@ -221,10 +221,10 @@ public class FSImageSerialization {
out.writeLong(file.getPreferredBlockSize());
}
- private static void writeQuota(Quota.Counts quota, DataOutput out)
+ private static void writeQuota(QuotaCounts quota, DataOutput out)
throws IOException {
- out.writeLong(quota.get(Quota.NAMESPACE));
- out.writeLong(quota.get(Quota.DISKSPACE));
+ out.writeLong(quota.getNameSpace());
+ out.writeLong(quota.getDiskSpace());
}
/**
[20/50] [abbrv] hadoop git commit: YARN-3090. DeletionService can
silently ignore deletion task failures. Contributed by Varun Saxena
Posted by zj...@apache.org.
YARN-3090. DeletionService can silently ignore deletion task failures. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4eb5f7fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4eb5f7fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4eb5f7fa
Branch: refs/heads/YARN-2928
Commit: 4eb5f7fa32bab1b9ce3fb58eca51e2cd2e194cd5
Parents: e0ec071
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Feb 10 16:54:21 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Feb 10 16:54:21 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 ++
.../server/nodemanager/DeletionService.java | 40 +++++++++++++++++---
2 files changed, 38 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eb5f7fa/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fbeca6a..5a3a505 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -527,6 +527,9 @@ Release 2.7.0 - UNRELEASED
YARN-2971. RM uses conf instead of token service address to renew timeline
delegation tokens (jeagles)
+ YARN-3090. DeletionService can silently ignore deletion task failures
+ (Varun Saxena via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4eb5f7fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
index e4025f5..4e00a1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
@@ -29,6 +29,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.FutureTask;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
@@ -113,13 +115,13 @@ public class DeletionService extends AbstractService {
.setNameFormat("DeletionService #%d")
.build();
if (conf != null) {
- sched = new ScheduledThreadPoolExecutor(
- conf.getInt(YarnConfiguration.NM_DELETE_THREAD_COUNT, YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT),
- tf);
+ sched = new DelServiceSchedThreadPoolExecutor(
+ conf.getInt(YarnConfiguration.NM_DELETE_THREAD_COUNT,
+ YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT), tf);
debugDelay = conf.getInt(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 0);
} else {
- sched = new ScheduledThreadPoolExecutor(YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT,
- tf);
+ sched = new DelServiceSchedThreadPoolExecutor(
+ YarnConfiguration.DEFAULT_NM_DELETE_THREAD_COUNT, tf);
}
sched.setExecuteExistingDelayedTasksAfterShutdownPolicy(false);
sched.setKeepAliveTime(60L, SECONDS);
@@ -155,6 +157,34 @@ public class DeletionService extends AbstractService {
return getServiceState() == STATE.STOPPED && sched.isTerminated();
}
+ private static class DelServiceSchedThreadPoolExecutor extends
+ ScheduledThreadPoolExecutor {
+ public DelServiceSchedThreadPoolExecutor(int corePoolSize,
+ ThreadFactory threadFactory) {
+ super(corePoolSize, threadFactory);
+ }
+
+ @Override
+ protected void afterExecute(Runnable task, Throwable exception) {
+ if (task instanceof FutureTask<?>) {
+ FutureTask<?> futureTask = (FutureTask<?>) task;
+ if (!futureTask.isCancelled()) {
+ try {
+ futureTask.get();
+ } catch (ExecutionException ee) {
+ exception = ee.getCause();
+ } catch (InterruptedException ie) {
+ exception = ie;
+ }
+ }
+ }
+ if (exception != null) {
+ LOG.error("Exception during execution of task in DeletionService",
+ exception);
+ }
+ }
+ }
+
public static class FileDeletionTask implements Runnable {
public static final int INVALID_TASK_ID = -1;
private int taskId;
[16/50] [abbrv] hadoop git commit: YARN-3100. Made YARN authorization
pluggable. Contributed by Jian He.
Posted by zj...@apache.org.
YARN-3100. Made YARN authorization pluggable. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23bf6c72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23bf6c72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23bf6c72
Branch: refs/heads/YARN-2928
Commit: 23bf6c72071782e3fd5a628e21495d6b974c7a9e
Parents: 02340a2
Author: Zhijie Shen <zj...@apache.org>
Authored: Mon Feb 9 20:34:56 2015 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Mon Feb 9 20:34:56 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 2 +
.../hadoop/yarn/conf/YarnConfiguration.java | 2 +
.../apache/hadoop/yarn/security/AccessType.java | 33 ++++++
.../hadoop/yarn/security/AdminACLsManager.java | 31 -----
.../yarn/security/ConfiguredYarnAuthorizer.java | 97 ++++++++++++++++
.../hadoop/yarn/security/PrivilegedEntity.java | 83 ++++++++++++++
.../security/YarnAuthorizationProvider.java | 112 +++++++++++++++++++
.../org/apache/hadoop/yarn/webapp/WebApps.java | 7 +-
.../server/resourcemanager/AdminService.java | 34 +++---
.../server/resourcemanager/RMServerUtils.java | 25 +++--
.../nodelabels/RMNodeLabelsManager.java | 12 +-
.../scheduler/SchedulerUtils.java | 13 +++
.../scheduler/capacity/AbstractCSQueue.java | 43 +++----
.../scheduler/capacity/CapacityScheduler.java | 20 +++-
.../CapacitySchedulerConfiguration.java | 10 +-
.../scheduler/capacity/LeafQueue.java | 7 +-
.../scheduler/capacity/ParentQueue.java | 7 +-
.../scheduler/capacity/TestParentQueue.java | 5 +-
.../SCMAdminProtocolService.java | 10 +-
19 files changed, 445 insertions(+), 108 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2f0ef7a..fbeca6a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -258,6 +258,8 @@ Release 2.7.0 - UNRELEASED
YARN-3155. Refactor the exception handling code for TimelineClientImpl's
retryOn method (Li Lu via wangda)
+ YARN-3100. Made YARN authorization pluggable. (Jian He via zjshen)
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index d6f6dee..6904543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -435,6 +435,8 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_RM_CONFIGURATION_PROVIDER_CLASS =
"org.apache.hadoop.yarn.LocalConfigurationProvider";
+ public static final String YARN_AUTHORIZATION_PROVIDER = YARN_PREFIX
+ + "authorization-provider";
private static final List<String> RM_SERVICES_ADDRESS_CONF_KEYS_HTTP =
Collections.unmodifiableList(Arrays.asList(
RM_ADDRESS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
new file mode 100644
index 0000000..32459b9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AccessType.java
@@ -0,0 +1,33 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.security;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+
+/**
+ * Access types for a queue or an application.
+ */
+@Private
+@Unstable
+public enum AccessType {
+ // queue
+ SUBMIT_APP,
+ ADMINISTER_QUEUE,
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AdminACLsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AdminACLsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AdminACLsManager.java
index 70c1a6e..a386123 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AdminACLsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AdminACLsManager.java
@@ -99,15 +99,6 @@ public class AdminACLsManager {
}
/**
- * Returns the internal structure used to maintain administrator ACLs
- *
- * @return Structure used to maintain administrator access
- */
- public AccessControlList getAdminAcl() {
- return adminAcl;
- }
-
- /**
* Returns whether the specified user/group is an administrator
*
* @param callerUGI user/group to to check
@@ -117,26 +108,4 @@ public class AdminACLsManager {
public boolean isAdmin(UserGroupInformation callerUGI) {
return adminAcl.isUserAllowed(callerUGI);
}
-
- /**
- * Returns whether the specified user/group has administrator access
- *
- * @param callerUGI user/group to to check
- * @return <tt>true</tt> if the UserGroupInformation specified
- * is a member of the access control list for administrators
- * and ACLs are enabled for this cluster
- *
- * @see #getAdminAcl
- * @see #areACLsEnabled
- */
- public boolean checkAccess(UserGroupInformation callerUGI) {
-
- // Any user may perform this operation if authorization is not enabled
- if (!areACLsEnabled()) {
- return true;
- }
-
- // Administrators may perform any operation
- return isAdmin(callerUGI);
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java
new file mode 100644
index 0000000..90ba77a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ConfiguredYarnAuthorizer.java
@@ -0,0 +1,97 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.security;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType;
+
+/**
+ * A YarnAuthorizationProvider implementation based on configuration files.
+ *
+ */
+@Private
+@Unstable
+public class ConfiguredYarnAuthorizer extends YarnAuthorizationProvider {
+
+ private final ConcurrentMap<PrivilegedEntity, Map<AccessType, AccessControlList>> allAcls =
+ new ConcurrentHashMap<>();
+ private volatile AccessControlList adminAcl = null;
+
+
+ @Override
+ public void init(Configuration conf) {
+ adminAcl =
+ new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ }
+
+ @Override
+ public void setPermission(PrivilegedEntity target,
+ Map<AccessType, AccessControlList> acls, UserGroupInformation ugi) {
+ allAcls.put(target, acls);
+ }
+
+ @Override
+ public boolean checkPermission(AccessType accessType,
+ PrivilegedEntity target, UserGroupInformation user) {
+ boolean ret = false;
+ Map<AccessType, AccessControlList> acls = allAcls.get(target);
+ if (acls != null) {
+ AccessControlList list = acls.get(accessType);
+ if (list != null) {
+ ret = list.isUserAllowed(user);
+ }
+ }
+
+ // recursively look up the queue to see if parent queue has the permission.
+ if (target.getType() == EntityType.QUEUE && !ret) {
+ String queueName = target.getName();
+ if (!queueName.contains(".")) {
+ return ret;
+ }
+ String parentQueueName = queueName.substring(0, queueName.lastIndexOf("."));
+ return checkPermission(accessType, new PrivilegedEntity(target.getType(),
+ parentQueueName), user);
+ }
+ return ret;
+ }
+
+ @Override
+ public void setAdmins(AccessControlList acls, UserGroupInformation ugi) {
+ adminAcl = acls;
+ }
+
+ @Override
+ public boolean isAdmin(UserGroupInformation ugi) {
+ return adminAcl.isUserAllowed(ugi);
+ }
+
+ public AccessControlList getAdminAcls() {
+ return this.adminAcl;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java
new file mode 100644
index 0000000..580bdf4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/PrivilegedEntity.java
@@ -0,0 +1,83 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.security;
+
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.QueueACL;
+
+/**
+ * An entity in YARN that can be guarded with ACLs. The entity could be an
+ * application or a queue etc. An application entity has access types defined in
+ * {@link ApplicationAccessType}, a queue entity has access types defined in
+ * {@link QueueACL}.
+ */
+@Private
+@Unstable
+public class PrivilegedEntity {
+
+ public enum EntityType {
+ QUEUE
+ }
+
+ EntityType type;
+ String name;
+
+ public PrivilegedEntity(EntityType type, String name) {
+ this.type = type;
+ this.name = name;
+ }
+
+ public EntityType getType() {
+ return type;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public int hashCode() {
+ final int prime = 31;
+ int result = 1;
+ result = prime * result + ((name == null) ? 0 : name.hashCode());
+ result = prime * result + ((type == null) ? 0 : type.hashCode());
+ return result;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (obj == null)
+ return false;
+ if (getClass() != obj.getClass())
+ return false;
+ PrivilegedEntity other = (PrivilegedEntity) obj;
+ if (name == null) {
+ if (other.name != null)
+ return false;
+ } else if (!name.equals(other.name))
+ return false;
+ if (type != other.type)
+ return false;
+ return true;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
new file mode 100644
index 0000000..7b2c35c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/YarnAuthorizationProvider.java
@@ -0,0 +1,112 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.security;
+
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
+import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+
+/**
+ * An implementation of the interface will provide authorization related
+ * information and enforce permission check. It is excepted that any of the
+ * methods defined in this interface should be non-blocking call and should not
+ * involve expensive computation as these method could be invoked in RPC.
+ */
+@Private
+@Unstable
+public abstract class YarnAuthorizationProvider {
+
+ private static final Log LOG = LogFactory.getLog(YarnAuthorizationProvider.class);
+
+ private static YarnAuthorizationProvider authorizer = null;
+
+ public static YarnAuthorizationProvider getInstance(Configuration conf) {
+ synchronized (YarnAuthorizationProvider.class) {
+ if (authorizer == null) {
+ Class<?> authorizerClass =
+ conf.getClass(YarnConfiguration.YARN_AUTHORIZATION_PROVIDER,
+ ConfiguredYarnAuthorizer.class);
+ authorizer =
+ (YarnAuthorizationProvider) ReflectionUtils.newInstance(
+ authorizerClass, conf);
+ authorizer.init(conf);
+ LOG.info(authorizerClass.getName() + " is instiantiated.");
+ }
+ }
+ return authorizer;
+ }
+
+ /**
+ * Initialize the provider. Invoked on daemon startup. DefaultYarnAuthorizer is
+ * initialized based on configurations.
+ */
+ public abstract void init(Configuration conf);
+
+ /**
+ * Check if user has the permission to access the target object.
+ *
+ * @param accessType
+ * The type of accessing method.
+ * @param target
+ * The target object being accessed, e.g. app/queue
+ * @param user
+ * User who access the target
+ * @return true if user can access the object, otherwise false.
+ */
+ public abstract boolean checkPermission(AccessType accessType,
+ PrivilegedEntity target, UserGroupInformation user);
+
+ /**
+ * Set ACLs for the target object. AccessControlList class encapsulate the
+ * users and groups who can access the target.
+ *
+ * @param target
+ * The target object.
+ * @param acls
+ * A map from access method to a list of users and/or groups who has
+ * permission to do the access.
+ * @param ugi User who sets the permissions.
+ */
+ public abstract void setPermission(PrivilegedEntity target,
+ Map<AccessType, AccessControlList> acls, UserGroupInformation ugi);
+
+ /**
+ * Set a list of users/groups who have admin access
+ *
+ * @param acls users/groups who have admin access
+ * @param ugi User who sets the admin acls.
+ */
+ public abstract void setAdmins(AccessControlList acls, UserGroupInformation ugi);
+
+ /**
+ * Check if the user is an admin.
+ *
+ * @param ugi the user to be determined if it is an admin
+ * @return true if the given user is an admin
+ */
+ public abstract boolean isAdmin(UserGroupInformation ugi);
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
index f8c6f55..1b5840f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/WebApps.java
@@ -36,11 +36,10 @@ import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.http.HttpConfig.Policy;
-import org.apache.hadoop.http.HttpConfig;
import org.apache.hadoop.http.HttpServer2;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.webapp.util.WebAppUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -240,7 +239,9 @@ public class WebApps {
.addEndpoint(
URI.create(httpScheme + bindAddress
+ ":" + port)).setConf(conf).setFindPort(findPort)
- .setACL(new AdminACLsManager(conf).getAdminAcl())
+ .setACL(new AccessControlList(conf.get(
+ YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)))
.setPathSpec(pathList.toArray(new String[0]));
boolean hasSpnegoConf = spnegoPrincipalKey != null
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
index d79de58..6180995 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
@@ -56,6 +56,8 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.ipc.RPCUtil;
import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.ConfiguredYarnAuthorizer;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.api.ResourceManagerAdministrationProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.AddToClusterNodeLabelsResponse;
@@ -101,7 +103,8 @@ public class AdminService extends CompositeService implements
// Address to use for binding. May be a wildcard address.
private InetSocketAddress masterServiceBindAddress;
- private AccessControlList adminAcl;
+
+ private YarnAuthorizationProvider authorizer;
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
@@ -129,10 +132,11 @@ public class AdminService extends CompositeService implements
YarnConfiguration.RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_ADDRESS,
YarnConfiguration.DEFAULT_RM_ADMIN_PORT);
-
- adminAcl = new AccessControlList(conf.get(
- YarnConfiguration.YARN_ADMIN_ACL,
- YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
+ authorizer.setAdmins(new AccessControlList(conf.get(
+ YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)), UserGroupInformation
+ .getCurrentUser());
rmId = conf.get(YarnConfiguration.RM_HA_ID);
super.serviceInit(conf);
}
@@ -206,7 +210,7 @@ public class AdminService extends CompositeService implements
}
private UserGroupInformation checkAccess(String method) throws IOException {
- return RMServerUtils.verifyAccess(adminAcl, method, LOG);
+ return RMServerUtils.verifyAdminAccess(authorizer, method, LOG);
}
private UserGroupInformation checkAcls(String method) throws YarnException {
@@ -293,7 +297,7 @@ public class AdminService extends CompositeService implements
"transitionToActive", "RMHAProtocolService");
} catch (Exception e) {
RMAuditLogger.logFailure(user.getShortUserName(), "transitionToActive",
- adminAcl.toString(), "RMHAProtocolService",
+ "", "RMHAProtocolService",
"Exception transitioning to active");
throw new ServiceFailedException(
"Error when transitioning to Active mode", e);
@@ -318,7 +322,7 @@ public class AdminService extends CompositeService implements
"transitionToStandby", "RMHAProtocolService");
} catch (Exception e) {
RMAuditLogger.logFailure(user.getShortUserName(), "transitionToStandby",
- adminAcl.toString(), "RMHAProtocolService",
+ "", "RMHAProtocolService",
"Exception transitioning to standby");
throw new ServiceFailedException(
"Error when transitioning to Standby mode", e);
@@ -446,9 +450,10 @@ public class AdminService extends CompositeService implements
Configuration conf =
getConfiguration(new Configuration(false),
YarnConfiguration.YARN_SITE_CONFIGURATION_FILE);
- adminAcl = new AccessControlList(conf.get(
- YarnConfiguration.YARN_ADMIN_ACL,
- YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ authorizer.setAdmins(new AccessControlList(conf.get(
+ YarnConfiguration.YARN_ADMIN_ACL,
+ YarnConfiguration.DEFAULT_YARN_ADMIN_ACL)), UserGroupInformation
+ .getCurrentUser());
RMAuditLogger.logSuccess(user.getShortUserName(), argName,
"AdminService");
@@ -584,9 +589,10 @@ public class AdminService extends CompositeService implements
}
}
+ // only for testing
@VisibleForTesting
public AccessControlList getAccessControlList() {
- return this.adminAcl;
+ return ((ConfiguredYarnAuthorizer)authorizer).getAdminAcls();
}
@VisibleForTesting
@@ -661,7 +667,7 @@ public class AdminService extends CompositeService implements
private void checkRMStatus(String user, String argName, String msg)
throws StandbyException {
if (!isRMActive()) {
- RMAuditLogger.logFailure(user, argName, adminAcl.toString(),
+ RMAuditLogger.logFailure(user, argName, "",
"AdminService", "ResourceManager is not active. Can not " + msg);
throwStandbyException();
}
@@ -670,7 +676,7 @@ public class AdminService extends CompositeService implements
private YarnException logAndWrapException(IOException ioe, String user,
String argName, String msg) throws YarnException {
LOG.info("Exception " + msg, ioe);
- RMAuditLogger.logFailure(user, argName, adminAcl.toString(),
+ RMAuditLogger.logFailure(user, argName, "",
"AdminService", "Exception " + msg);
return RPCUtil.getRemoteException(ioe);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
index c80778c..3d28bb7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMServerUtils.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.exceptions.InvalidContainerReleaseException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceBlacklistRequestException;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
@@ -140,43 +141,43 @@ public class RMServerUtils {
}
}
- public static UserGroupInformation verifyAccess(
- AccessControlList acl, String method, final Log LOG)
+ public static UserGroupInformation verifyAdminAccess(
+ YarnAuthorizationProvider authorizer, String method, final Log LOG)
throws IOException {
// by default, this method will use AdminService as module name
- return verifyAccess(acl, method, "AdminService", LOG);
+ return verifyAdminAccess(authorizer, method, "AdminService", LOG);
}
/**
* Utility method to verify if the current user has access based on the
* passed {@link AccessControlList}
- * @param acl the {@link AccessControlList} to check against
+ * @param authorizer the {@link AccessControlList} to check against
* @param method the method name to be logged
- * @param module, like AdminService or NodeLabelManager
+ * @param module like AdminService or NodeLabelManager
* @param LOG the logger to use
* @return {@link UserGroupInformation} of the current user
* @throws IOException
*/
- public static UserGroupInformation verifyAccess(
- AccessControlList acl, String method, String module, final Log LOG)
+ public static UserGroupInformation verifyAdminAccess(
+ YarnAuthorizationProvider authorizer, String method, String module,
+ final Log LOG)
throws IOException {
UserGroupInformation user;
try {
user = UserGroupInformation.getCurrentUser();
} catch (IOException ioe) {
LOG.warn("Couldn't get current user", ioe);
- RMAuditLogger.logFailure("UNKNOWN", method, acl.toString(),
+ RMAuditLogger.logFailure("UNKNOWN", method, "",
"AdminService", "Couldn't get current user");
throw ioe;
}
- if (!acl.isUserAllowed(user)) {
+ if (!authorizer.isAdmin(user)) {
LOG.warn("User " + user.getShortUserName() + " doesn't have permission" +
" to call '" + method + "'");
- RMAuditLogger.logFailure(user.getShortUserName(), method,
- acl.toString(), module,
- RMAuditLogger.AuditConstants.UNAUTHORIZED_USER);
+ RMAuditLogger.logFailure(user.getShortUserName(), method, "", module,
+ RMAuditLogger.AuditConstants.UNAUTHORIZED_USER);
throw new AccessControlException("User " + user.getShortUserName() +
" doesn't have permission" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
index 1555291..9942d80 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/nodelabels/RMNodeLabelsManager.java
@@ -33,12 +33,11 @@ import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
-import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
import org.apache.hadoop.yarn.nodelabels.NodeLabel;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -60,16 +59,13 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
ConcurrentMap<String, Queue> queueCollections =
new ConcurrentHashMap<String, Queue>();
- protected AccessControlList adminAcl;
-
+ private YarnAuthorizationProvider authorizer;
private RMContext rmContext = null;
@Override
protected void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
- adminAcl =
- new AccessControlList(conf.get(YarnConfiguration.YARN_ADMIN_ACL,
- YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
}
@Override
@@ -479,7 +475,7 @@ public class RMNodeLabelsManager extends CommonNodeLabelsManager {
public boolean checkAccess(UserGroupInformation user) {
// make sure only admin can invoke
// this method
- if (adminAcl.isUserAllowed(user)) {
+ if (authorizer.isAdmin(user)) {
return true;
}
return false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
index c4900c3..65d6859 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerUtils.java
@@ -28,12 +28,14 @@ import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.exceptions.InvalidResourceRequestException;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -348,4 +350,15 @@ public class SchedulerUtils {
}
return true;
}
+
+
+ public static AccessType toAccessType(QueueACL acl) {
+ switch (acl) {
+ case ADMINISTER_QUEUE:
+ return AccessType.ADMINISTER_QUEUE;
+ case SUBMIT_APPLICATIONS:
+ return AccessType.SUBMIT_APP;
+ }
+ return null;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
index e4c2665..753fb14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/AbstractCSQueue.java
@@ -34,12 +34,16 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
+import org.apache.hadoop.yarn.security.AccessType;
+import org.apache.hadoop.yarn.security.PrivilegedEntity;
+import org.apache.hadoop.yarn.security.PrivilegedEntity.EntityType;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
-import org.apache.hadoop.yarn.util.resource.Resources;
+
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.Sets;
@@ -60,7 +64,8 @@ public abstract class AbstractCSQueue implements CSQueue {
Resource maximumAllocation;
QueueState state;
final QueueMetrics metrics;
-
+ protected final PrivilegedEntity queueEntity;
+
final ResourceCalculator resourceCalculator;
Set<String> accessibleLabels;
RMNodeLabelsManager labelManager;
@@ -70,8 +75,8 @@ public abstract class AbstractCSQueue implements CSQueue {
Map<String, Float> absoluteMaxCapacityByNodeLabels;
Map<String, Float> maxCapacityByNodeLabels;
- Map<QueueACL, AccessControlList> acls =
- new HashMap<QueueACL, AccessControlList>();
+ Map<AccessType, AccessControlList> acls =
+ new HashMap<AccessType, AccessControlList>();
boolean reservationsContinueLooking;
private boolean preemptionDisabled;
@@ -81,6 +86,7 @@ public abstract class AbstractCSQueue implements CSQueue {
private final RecordFactory recordFactory =
RecordFactoryProvider.getRecordFactory(null);
private CapacitySchedulerContext csContext;
+ protected YarnAuthorizationProvider authorizer = null;
public AbstractCSQueue(CapacitySchedulerContext cs,
String queueName, CSQueue parent, CSQueue old) throws IOException {
@@ -126,6 +132,8 @@ public abstract class AbstractCSQueue implements CSQueue {
accessibleLabels, labelManager);
this.csContext = cs;
queueUsage = new ResourceUsage();
+ queueEntity = new PrivilegedEntity(EntityType.QUEUE, getQueuePath());
+ authorizer = YarnAuthorizationProvider.getInstance(cs.getConf());
}
@Override
@@ -181,7 +189,11 @@ public abstract class AbstractCSQueue implements CSQueue {
public String getQueueName() {
return queueName;
}
-
+
+ public PrivilegedEntity getPrivilegedEntity() {
+ return queueEntity;
+ }
+
@Override
public synchronized CSQueue getParent() {
return parent;
@@ -195,22 +207,13 @@ public abstract class AbstractCSQueue implements CSQueue {
public Set<String> getAccessibleNodeLabels() {
return accessibleLabels;
}
-
+
@Override
public boolean hasAccess(QueueACL acl, UserGroupInformation user) {
- synchronized (this) {
- if (acls.get(acl).isUserAllowed(user)) {
- return true;
- }
- }
-
- if (parent != null) {
- return parent.hasAccess(acl, user);
- }
-
- return false;
+ return authorizer.checkPermission(SchedulerUtils.toAccessType(acl),
+ queueEntity, user);
}
-
+
@Override
public synchronized void setUsedCapacity(float usedCapacity) {
this.usedCapacity = usedCapacity;
@@ -251,7 +254,7 @@ public abstract class AbstractCSQueue implements CSQueue {
synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity,
- QueueState state, Map<QueueACL, AccessControlList> acls,
+ QueueState state, Map<AccessType, AccessControlList> acls,
Set<String> labels, String defaultLabelExpression,
Map<String, Float> nodeLabelCapacities,
Map<String, Float> maximumNodeLabelCapacities,
@@ -436,7 +439,7 @@ public abstract class AbstractCSQueue implements CSQueue {
}
@Private
- public Map<QueueACL, AccessControlList> getACLs() {
+ public Map<AccessType, AccessControlList> getACLs() {
return acls;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index 916a4db..6b9d846 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -63,6 +63,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore.RMState;
@@ -124,7 +125,8 @@ public class CapacityScheduler extends
PreemptableResourceScheduler, CapacitySchedulerContext, Configurable {
private static final Log LOG = LogFactory.getLog(CapacityScheduler.class);
-
+ private YarnAuthorizationProvider authorizer;
+
private CSQueue root;
// timeout to join when we stop this service
protected final long THREAD_JOIN_TIMEOUT_MS = 1000;
@@ -297,7 +299,7 @@ public class CapacityScheduler extends
new ConcurrentHashMap<ApplicationId,
SchedulerApplication<FiCaSchedulerApp>>();
this.labelManager = rmContext.getNodeLabelManager();
-
+ authorizer = YarnAuthorizationProvider.getInstance(yarnConf);
initializeQueues(this.conf);
scheduleAsynchronously = this.conf.getScheduleAynschronously();
@@ -474,6 +476,7 @@ public class CapacityScheduler extends
labelManager.reinitializeQueueLabels(getQueueToLabels());
LOG.info("Initialized root queue " + root);
initializeQueueMappings();
+ setQueueAcls(authorizer, queues);
}
@Lock(CapacityScheduler.class)
@@ -499,8 +502,19 @@ public class CapacityScheduler extends
root.updateClusterResource(clusterResource);
labelManager.reinitializeQueueLabels(getQueueToLabels());
+ setQueueAcls(authorizer, queues);
}
-
+
+ @VisibleForTesting
+ public static void setQueueAcls(YarnAuthorizationProvider authorizer,
+ Map<String, CSQueue> queues) throws IOException {
+ for (CSQueue queue : queues.values()) {
+ AbstractCSQueue csQueue = (AbstractCSQueue) queue;
+ authorizer.setPermission(csQueue.getPrivilegedEntity(),
+ csQueue.getACLs(), UserGroupInformation.getCurrentUser());
+ }
+ }
+
private Map<String, Set<String>> getQueueToLabels() {
Map<String, Set<String>> queueToLabels = new HashMap<String, Set<String>>();
for (CSQueue queue : queues.values()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
index 268cc6c..b49a60a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerConfiguration.java
@@ -40,8 +40,10 @@ import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.reservation.ReservationSchedulerConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerUtils;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
import org.apache.hadoop.yarn.util.resource.ResourceCalculator;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -530,11 +532,11 @@ public class CapacitySchedulerConfiguration extends ReservationSchedulerConfigur
set(queuePrefix + getAclKey(acl), aclString);
}
- public Map<QueueACL, AccessControlList> getAcls(String queue) {
- Map<QueueACL, AccessControlList> acls =
- new HashMap<QueueACL, AccessControlList>();
+ public Map<AccessType, AccessControlList> getAcls(String queue) {
+ Map<AccessType, AccessControlList> acls =
+ new HashMap<AccessType, AccessControlList>();
for (QueueACL acl : QueueACL.values()) {
- acls.put(acl, getAcl(queue, acl));
+ acls.put(SchedulerUtils.toAccessType(acl), getAcl(queue, acl));
}
return acls;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index c143210..5258223 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -153,7 +154,7 @@ public class LeafQueue extends AbstractCSQueue {
QueueState state = cs.getConfiguration().getState(getQueuePath());
- Map<QueueACL, AccessControlList> acls =
+ Map<AccessType, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
@@ -189,7 +190,7 @@ public class LeafQueue extends AbstractCSQueue {
int userLimit, float userLimitFactor,
int maxApplications, float maxAMResourcePerQueuePercent,
int maxApplicationsPerUser, QueueState state,
- Map<QueueACL, AccessControlList> acls, int nodeLocalityDelay,
+ Map<AccessType, AccessControlList> acls, int nodeLocalityDelay,
Set<String> labels, String defaultLabelExpression,
Map<String, Float> capacitieByLabel,
Map<String, Float> maximumCapacitiesByLabel,
@@ -247,7 +248,7 @@ public class LeafQueue extends AbstractCSQueue {
maximumAllocation);
StringBuilder aclsString = new StringBuilder();
- for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
+ for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
index 5a2e234..29a8ba3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
@@ -49,6 +49,7 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.nodelabels.CommonNodeLabelsManager;
+import org.apache.hadoop.yarn.security.AccessType;
import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
@@ -107,7 +108,7 @@ public class ParentQueue extends AbstractCSQueue {
QueueState state = cs.getConfiguration().getState(getQueuePath());
- Map<QueueACL, AccessControlList> acls =
+ Map<AccessType, AccessControlList> acls =
cs.getConfiguration().getAcls(getQueuePath());
setupQueueConfigs(cs.getClusterResource(), capacity, absoluteCapacity,
@@ -124,7 +125,7 @@ public class ParentQueue extends AbstractCSQueue {
synchronized void setupQueueConfigs(Resource clusterResource, float capacity,
float absoluteCapacity, float maximumCapacity, float absoluteMaxCapacity,
- QueueState state, Map<QueueACL, AccessControlList> acls,
+ QueueState state, Map<AccessType, AccessControlList> acls,
Set<String> accessibleLabels, String defaultLabelExpression,
Map<String, Float> nodeLabelCapacities,
Map<String, Float> maximumCapacitiesByLabel,
@@ -134,7 +135,7 @@ public class ParentQueue extends AbstractCSQueue {
defaultLabelExpression, nodeLabelCapacities, maximumCapacitiesByLabel,
reservationContinueLooking, maximumAllocation);
StringBuilder aclsString = new StringBuilder();
- for (Map.Entry<QueueACL, AccessControlList> e : acls.entrySet()) {
+ for (Map.Entry<AccessType, AccessControlList> e : acls.entrySet()) {
aclsString.append(e.getKey() + ":" + e.getValue().getAclString());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
index 72983ca..696ad7a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestParentQueue.java
@@ -37,7 +37,6 @@ import java.util.List;
import java.util.Map;
import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.UserGroupInformation;
@@ -45,6 +44,7 @@ import org.apache.hadoop.yarn.api.records.QueueACL;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
@@ -726,6 +726,9 @@ public class TestParentQueue {
CapacityScheduler.parseQueue(csContext, csConf, null,
CapacitySchedulerConfiguration.ROOT, queues, queues,
TestUtils.spyHook);
+ YarnAuthorizationProvider authorizer =
+ YarnAuthorizationProvider.getInstance(conf);
+ CapacityScheduler.setQueueAcls(authorizer, queues);
UserGroupInformation user = UserGroupInformation.getCurrentUser();
// Setup queue configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23bf6c72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java
index 3ecca02..6f2baf6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-sharedcachemanager/src/main/java/org/apache/hadoop/yarn/server/sharedcachemanager/SCMAdminProtocolService.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.security.YarnAuthorizationProvider;
import org.apache.hadoop.yarn.server.api.SCMAdminProtocol;
import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RunSharedCacheCleanerTaskResponse;
@@ -58,8 +59,7 @@ public class SCMAdminProtocolService extends AbstractService implements
private Server server;
InetSocketAddress clientBindAddress;
private final CleanerService cleanerService;
- private AccessControlList adminAcl;
-
+ private YarnAuthorizationProvider authorizer;
public SCMAdminProtocolService(CleanerService cleanerService) {
super(SCMAdminProtocolService.class.getName());
this.cleanerService = cleanerService;
@@ -68,9 +68,7 @@ public class SCMAdminProtocolService extends AbstractService implements
@Override
protected void serviceInit(Configuration conf) throws Exception {
this.clientBindAddress = getBindAddress(conf);
- adminAcl = new AccessControlList(conf.get(
- YarnConfiguration.YARN_ADMIN_ACL,
- YarnConfiguration.DEFAULT_YARN_ADMIN_ACL));
+ authorizer = YarnAuthorizationProvider.getInstance(conf);
super.serviceInit(conf);
}
@@ -119,7 +117,7 @@ public class SCMAdminProtocolService extends AbstractService implements
throw RPCUtil.getRemoteException(ioe);
}
- if (!adminAcl.isUserAllowed(user)) {
+ if (!authorizer.isAdmin(user)) {
LOG.warn("User " + user.getShortUserName() + " doesn't have permission" +
" to call '" + method + "'");
[23/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
new file mode 100644
index 0000000..ae057bc
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -0,0 +1,375 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Hadoop in Secure Mode](#Hadoop_in_Secure_Mode)
+ * [Introduction](#Introduction)
+ * [Authentication](#Authentication)
+ * [End User Accounts](#End_User_Accounts)
+ * [User Accounts for Hadoop Daemons](#User_Accounts_for_Hadoop_Daemons)
+ * [Kerberos principals for Hadoop Daemons and Users](#Kerberos_principals_for_Hadoop_Daemons_and_Users)
+ * [Mapping from Kerberos principal to OS user account](#Mapping_from_Kerberos_principal_to_OS_user_account)
+ * [Mapping from user to group](#Mapping_from_user_to_group)
+ * [Proxy user](#Proxy_user)
+ * [Secure DataNode](#Secure_DataNode)
+ * [Data confidentiality](#Data_confidentiality)
+ * [Data Encryption on RPC](#Data_Encryption_on_RPC)
+ * [Data Encryption on Block data transfer.](#Data_Encryption_on_Block_data_transfer.)
+ * [Data Encryption on HTTP](#Data_Encryption_on_HTTP)
+ * [Configuration](#Configuration)
+ * [Permissions for both HDFS and local fileSystem paths](#Permissions_for_both_HDFS_and_local_fileSystem_paths)
+ * [Common Configurations](#Common_Configurations)
+ * [NameNode](#NameNode)
+ * [Secondary NameNode](#Secondary_NameNode)
+ * [DataNode](#DataNode)
+ * [WebHDFS](#WebHDFS)
+ * [ResourceManager](#ResourceManager)
+ * [NodeManager](#NodeManager)
+ * [Configuration for WebAppProxy](#Configuration_for_WebAppProxy)
+ * [LinuxContainerExecutor](#LinuxContainerExecutor)
+ * [MapReduce JobHistory Server](#MapReduce_JobHistory_Server)
+
+Hadoop in Secure Mode
+=====================
+
+Introduction
+------------
+
+This document describes how to configure authentication for Hadoop in secure mode.
+
+By default Hadoop runs in non-secure mode in which no actual authentication is required. By configuring Hadoop runs in secure mode, each user and service needs to be authenticated by Kerberos in order to use Hadoop services.
+
+Security features of Hadoop consist of [authentication](#Authentication), [service level authorization](./ServiceLevelAuth.html), [authentication for Web consoles](./HttpAuthentication.html) and [data confidenciality](#Data_confidentiality).
+
+Authentication
+--------------
+
+### End User Accounts
+
+When service level authentication is turned on, end users using Hadoop in secure mode needs to be authenticated by Kerberos. The simplest way to do authentication is using `kinit` command of Kerberos.
+
+### User Accounts for Hadoop Daemons
+
+Ensure that HDFS and YARN daemons run as different Unix users, e.g. `hdfs` and `yarn`. Also, ensure that the MapReduce JobHistory server runs as different user such as `mapred`.
+
+It's recommended to have them share a Unix group, for e.g. `hadoop`. See also "[Mapping from user to group](#Mapping_from_user_to_group)" for group management.
+
+| User:Group | Daemons |
+|:---- |:---- |
+| hdfs:hadoop | NameNode, Secondary NameNode, JournalNode, DataNode |
+| yarn:hadoop | ResourceManager, NodeManager |
+| mapred:hadoop | MapReduce JobHistory Server |
+
+### Kerberos principals for Hadoop Daemons and Users
+
+For running hadoop service daemons in Hadoop in secure mode, Kerberos principals are required. Each service reads auhenticate information saved in keytab file with appropriate permission.
+
+HTTP web-consoles should be served by principal different from RPC's one.
+
+Subsections below shows the examples of credentials for Hadoop services.
+
+#### HDFS
+
+The NameNode keytab file, on the NameNode host, should look like the following:
+
+ $ klist -e -k -t /etc/security/keytab/nn.service.keytab
+ Keytab name: FILE:/etc/security/keytab/nn.service.keytab
+ KVNO Timestamp Principal
+ 4 07/18/11 21:08:09 nn/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 nn/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 nn/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+
+The Secondary NameNode keytab file, on that host, should look like the following:
+
+ $ klist -e -k -t /etc/security/keytab/sn.service.keytab
+ Keytab name: FILE:/etc/security/keytab/sn.service.keytab
+ KVNO Timestamp Principal
+ 4 07/18/11 21:08:09 sn/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 sn/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 sn/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+
+The DataNode keytab file, on each host, should look like the following:
+
+ $ klist -e -k -t /etc/security/keytab/dn.service.keytab
+ Keytab name: FILE:/etc/security/keytab/dn.service.keytab
+ KVNO Timestamp Principal
+ 4 07/18/11 21:08:09 dn/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 dn/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 dn/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+
+#### YARN
+
+The ResourceManager keytab file, on the ResourceManager host, should look like the following:
+
+ $ klist -e -k -t /etc/security/keytab/rm.service.keytab
+ Keytab name: FILE:/etc/security/keytab/rm.service.keytab
+ KVNO Timestamp Principal
+ 4 07/18/11 21:08:09 rm/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 rm/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 rm/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+
+The NodeManager keytab file, on each host, should look like the following:
+
+ $ klist -e -k -t /etc/security/keytab/nm.service.keytab
+ Keytab name: FILE:/etc/security/keytab/nm.service.keytab
+ KVNO Timestamp Principal
+ 4 07/18/11 21:08:09 nm/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 nm/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 nm/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+
+#### MapReduce JobHistory Server
+
+The MapReduce JobHistory Server keytab file, on that host, should look like the following:
+
+ $ klist -e -k -t /etc/security/keytab/jhs.service.keytab
+ Keytab name: FILE:/etc/security/keytab/jhs.service.keytab
+ KVNO Timestamp Principal
+ 4 07/18/11 21:08:09 jhs/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 jhs/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 jhs/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-256 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (AES-128 CTS mode with 96-bit SHA-1 HMAC)
+ 4 07/18/11 21:08:09 host/full.qualified.domain.name@REALM.TLD (ArcFour with HMAC/md5)
+
+### Mapping from Kerberos principal to OS user account
+
+Hadoop maps Kerberos principal to OS user account using the rule specified by `hadoop.security.auth_to_local` which works in the same way as the `auth_to_local` in [Kerberos configuration file (krb5.conf)](http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html). In addition, Hadoop `auth_to_local` mapping supports the **/L** flag that lowercases the returned name.
+
+By default, it picks the first component of principal name as a user name if the realms matches to the `default_realm` (usually defined in /etc/krb5.conf). For example, `host/full.qualified.domain.name@REALM.TLD` is mapped to `host` by default rule.
+
+### Mapping from user to group
+
+Though files on HDFS are associated to owner and group, Hadoop does not have the definition of group by itself. Mapping from user to group is done by OS or LDAP.
+
+You can change a way of mapping by specifying the name of mapping provider as a value of `hadoop.security.group.mapping` See [HDFS Permissions Guide](../hadoop-hdfs/HdfsPermissionsGuide.html) for details.
+
+Practically you need to manage SSO environment using Kerberos with LDAP for Hadoop in secure mode.
+
+### Proxy user
+
+Some products such as Apache Oozie which access the services of Hadoop on behalf of end users need to be able to impersonate end users. See [the doc of proxy user](./Superusers.html) for details.
+
+### Secure DataNode
+
+Because the data transfer protocol of DataNode does not use the RPC framework of Hadoop, DataNode must authenticate itself by using privileged ports which are specified by `dfs.datanode.address` and `dfs.datanode.http.address`. This authentication is based on the assumption that the attacker won't be able to get root privileges.
+
+When you execute `hdfs datanode` command as root, server process binds privileged port at first, then drops privilege and runs as the user account specified by `HADOOP_SECURE_DN_USER`. This startup process uses jsvc installed to `JSVC_HOME`. You must specify `HADOOP_SECURE_DN_USER` and `JSVC_HOME` as environment variables on start up (in hadoop-env.sh).
+
+As of version 2.6.0, SASL can be used to authenticate the data transfer protocol. In this configuration, it is no longer required for secured clusters to start the DataNode as root using jsvc and bind to privileged ports. To enable SASL on data transfer protocol, set `dfs.data.transfer.protection` in hdfs-site.xml, set a non-privileged port for `dfs.datanode.address`, set `dfs.http.policy` to *HTTPS\_ONLY* and make sure the `HADOOP_SECURE_DN_USER` environment variable is not defined. Note that it is not possible to use SASL on data transfer protocol if `dfs.datanode.address` is set to a privileged port. This is required for backwards-compatibility reasons.
+
+In order to migrate an existing cluster that used root authentication to start using SASL instead, first ensure that version 2.6.0 or later has been deployed to all cluster nodes as well as any external applications that need to connect to the cluster. Only versions 2.6.0 and later of the HDFS client can connect to a DataNode that uses SASL for authentication of data transfer protocol, so it is vital that all callers have the correct version before migrating. After version 2.6.0 or later has been deployed everywhere, update configuration of any external applications to enable SASL. If an HDFS client is enabled for SASL, then it can connect successfully to a DataNode running with either root authentication or SASL authentication. Changing configuration for all clients guarantees that subsequent configuration changes on DataNodes will not disrupt the applications. Finally, each individual DataNode can be migrated by changing its configuration and restarting. It is acceptable to have a
mix of some DataNodes running with root authentication and some DataNodes running with SASL authentication temporarily during this migration period, because an HDFS client enabled for SASL can connect to both.
+
+Data confidentiality
+--------------------
+
+### Data Encryption on RPC
+
+The data transfered between hadoop services and clients. Setting `hadoop.rpc.protection` to `"privacy"` in the core-site.xml activate data encryption.
+
+### Data Encryption on Block data transfer.
+
+You need to set `dfs.encrypt.data.transfer` to `"true"` in the hdfs-site.xml in order to activate data encryption for data transfer protocol of DataNode.
+
+Optionally, you may set `dfs.encrypt.data.transfer.algorithm` to either "3des" or "rc4" to choose the specific encryption algorithm. If unspecified, then the configured JCE default on the system is used, which is usually 3DES.
+
+Setting `dfs.encrypt.data.transfer.cipher.suites` to `AES/CTR/NoPadding` activates AES encryption. By default, this is unspecified, so AES is not used. When AES is used, the algorithm specified in `dfs.encrypt.data.transfer.algorithm` is still used during an initial key exchange. The AES key bit length can be configured by setting `dfs.encrypt.data.transfer.cipher.key.bitlength` to 128, 192 or 256. The default is 128.
+
+AES offers the greatest cryptographic strength and the best performance. At this time, 3DES and RC4 have been used more often in Hadoop clusters.
+
+### Data Encryption on HTTP
+
+Data transfer between Web-console and clients are protected by using SSL(HTTPS).
+
+Configuration
+-------------
+
+### Permissions for both HDFS and local fileSystem paths
+
+The following table lists various paths on HDFS and local filesystems (on all nodes) and recommended permissions:
+
+| Filesystem | Path | User:Group | Permissions |
+|:---- |:---- |:---- |:---- |
+| local | `dfs.namenode.name.dir` | hdfs:hadoop | drwx------ |
+| local | `dfs.datanode.data.dir` | hdfs:hadoop | drwx------ |
+| local | $HADOOP\_LOG\_DIR | hdfs:hadoop | drwxrwxr-x |
+| local | $YARN\_LOG\_DIR | yarn:hadoop | drwxrwxr-x |
+| local | `yarn.nodemanager.local-dirs` | yarn:hadoop | drwxr-xr-x |
+| local | `yarn.nodemanager.log-dirs` | yarn:hadoop | drwxr-xr-x |
+| local | container-executor | root:hadoop | --Sr-s--* |
+| local | `conf/container-executor.cfg` | root:hadoop | r-------* |
+| hdfs | / | hdfs:hadoop | drwxr-xr-x |
+| hdfs | /tmp | hdfs:hadoop | drwxrwxrwxt |
+| hdfs | /user | hdfs:hadoop | drwxr-xr-x |
+| hdfs | `yarn.nodemanager.remote-app-log-dir` | yarn:hadoop | drwxrwxrwxt |
+| hdfs | `mapreduce.jobhistory.intermediate-done-dir` | mapred:hadoop | drwxrwxrwxt |
+| hdfs | `mapreduce.jobhistory.done-dir` | mapred:hadoop | drwxr-x--- |
+
+### Common Configurations
+
+In order to turn on RPC authentication in hadoop, set the value of `hadoop.security.authentication` property to `"kerberos"`, and set security related settings listed below appropriately.
+
+The following properties should be in the `core-site.xml` of all the nodes in the cluster.
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `hadoop.security.authentication` | *kerberos* | `simple` : No authentication. (default) `kerberos` : Enable authentication by Kerberos. |
+| `hadoop.security.authorization` | *true* | Enable [RPC service-level authorization](./ServiceLevelAuth.html). |
+| `hadoop.rpc.protection` | *authentication* | *authentication* : authentication only (default) *integrity* : integrity check in addition to authentication *privacy* : data encryption in addition to integrity |
+| `hadoop.security.auth_to_local` | `RULE:`*exp1* `RULE:`*exp2* *...* DEFAULT | The value is string containing new line characters. See [Kerberos documentation](http://web.mit.edu/Kerberos/krb5-latest/doc/admin/conf_files/krb5_conf.html) for format for *exp*. |
+| `hadoop.proxyuser.`*superuser*`.hosts` | | comma separated hosts from which *superuser* access are allowd to impersonation. `*` means wildcard. |
+| `hadoop.proxyuser.`*superuser*`.groups` | | comma separated groups to which users impersonated by *superuser* belongs. `*` means wildcard. |
+
+### NameNode
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `dfs.block.access.token.enable` | *true* | Enable HDFS block access tokens for secure operations. |
+| `dfs.https.enable` | *true* | This value is deprecated. Use dfs.http.policy |
+| `dfs.http.policy` | *HTTP\_ONLY* or *HTTPS\_ONLY* or *HTTP\_AND\_HTTPS* | HTTPS\_ONLY turns off http access. This option takes precedence over the deprecated configuration dfs.https.enable and hadoop.ssl.enabled. If using SASL to authenticate data transfer protocol instead of running DataNode as root and using privileged ports, then this property must be set to *HTTPS\_ONLY* to guarantee authentication of HTTP servers. (See `dfs.data.transfer.protection`.) |
+| `dfs.namenode.https-address` | *nn\_host\_fqdn:50470* | |
+| `dfs.https.port` | *50470* | |
+| `dfs.namenode.keytab.file` | */etc/security/keytab/nn.service.keytab* | Kerberos keytab file for the NameNode. |
+| `dfs.namenode.kerberos.principal` | nn/\_HOST@REALM.TLD | Kerberos principal name for the NameNode. |
+| `dfs.namenode.kerberos.internal.spnego.principal` | HTTP/\_HOST@REALM.TLD | HTTP Kerberos principal name for the NameNode. |
+
+### Secondary NameNode
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `dfs.namenode.secondary.http-address` | *c\_nn\_host\_fqdn:50090* | |
+| `dfs.namenode.secondary.https-port` | *50470* | |
+| `dfs.secondary.namenode.keytab.file` | */etc/security/keytab/sn.service.keytab* | Kerberos keytab file for the Secondary NameNode. |
+| `dfs.secondary.namenode.kerberos.principal` | sn/\_HOST@REALM.TLD | Kerberos principal name for the Secondary NameNode. |
+| `dfs.secondary.namenode.kerberos.internal.spnego.principal` | HTTP/\_HOST@REALM.TLD | HTTP Kerberos principal name for the Secondary NameNode. |
+
+### DataNode
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `dfs.datanode.data.dir.perm` | 700 | |
+| `dfs.datanode.address` | *0.0.0.0:1004* | Secure DataNode must use privileged port in order to assure that the server was started securely. This means that the server must be started via jsvc. Alternatively, this must be set to a non-privileged port if using SASL to authenticate data transfer protocol. (See `dfs.data.transfer.protection`.) |
+| `dfs.datanode.http.address` | *0.0.0.0:1006* | Secure DataNode must use privileged port in order to assure that the server was started securely. This means that the server must be started via jsvc. |
+| `dfs.datanode.https.address` | *0.0.0.0:50470* | |
+| `dfs.datanode.keytab.file` | */etc/security/keytab/dn.service.keytab* | Kerberos keytab file for the DataNode. |
+| `dfs.datanode.kerberos.principal` | dn/\_HOST@REALM.TLD | Kerberos principal name for the DataNode. |
+| `dfs.encrypt.data.transfer` | *false* | set to `true` when using data encryption |
+| `dfs.encrypt.data.transfer.algorithm` | | optionally set to `3des` or `rc4` when using data encryption to control encryption algorithm |
+| `dfs.encrypt.data.transfer.cipher.suites` | | optionally set to `AES/CTR/NoPadding` to activate AES encryption when using data encryption |
+| `dfs.encrypt.data.transfer.cipher.key.bitlength` | | optionally set to `128`, `192` or `256` to control key bit length when using AES with data encryption |
+| `dfs.data.transfer.protection` | | *authentication* : authentication only *integrity* : integrity check in addition to authentication *privacy* : data encryption in addition to integrity This property is unspecified by default. Setting this property enables SASL for authentication of data transfer protocol. If this is enabled, then `dfs.datanode.address` must use a non-privileged port, `dfs.http.policy` must be set to *HTTPS\_ONLY* and the `HADOOP_SECURE_DN_USER` environment variable must be undefined when starting the DataNode process. |
+
+### WebHDFS
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `dfs.web.authentication.kerberos.principal` | http/\_HOST@REALM.TLD | Kerberos keytab file for the WebHDFS. |
+| `dfs.web.authentication.kerberos.keytab` | */etc/security/keytab/http.service.keytab* | Kerberos principal name for WebHDFS. |
+
+### ResourceManager
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.resourcemanager.keytab` | */etc/security/keytab/rm.service.keytab* | Kerberos keytab file for the ResourceManager. |
+| `yarn.resourcemanager.principal` | rm/\_HOST@REALM.TLD | Kerberos principal name for the ResourceManager. |
+
+### NodeManager
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.nodemanager.keytab` | */etc/security/keytab/nm.service.keytab* | Kerberos keytab file for the NodeManager. |
+| `yarn.nodemanager.principal` | nm/\_HOST@REALM.TLD | Kerberos principal name for the NodeManager. |
+| `yarn.nodemanager.container-executor.class` | `org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor` | Use LinuxContainerExecutor. |
+| `yarn.nodemanager.linux-container-executor.group` | *hadoop* | Unix group of the NodeManager. |
+| `yarn.nodemanager.linux-container-executor.path` | */path/to/bin/container-executor* | The path to the executable of Linux container executor. |
+
+### Configuration for WebAppProxy
+
+The `WebAppProxy` provides a proxy between the web applications exported by an application and an end user. If security is enabled it will warn users before accessing a potentially unsafe web application. Authentication and authorization using the proxy is handled just like any other privileged web application.
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.web-proxy.address` | `WebAppProxy` host:port for proxy to AM web apps. | *host:port* if this is the same as `yarn.resourcemanager.webapp.address` or it is not defined then the `ResourceManager` will run the proxy otherwise a standalone proxy server will need to be launched. |
+| `yarn.web-proxy.keytab` | */etc/security/keytab/web-app.service.keytab* | Kerberos keytab file for the WebAppProxy. |
+| `yarn.web-proxy.principal` | wap/\_HOST@REALM.TLD | Kerberos principal name for the WebAppProxy. |
+
+### LinuxContainerExecutor
+
+A `ContainerExecutor` used by YARN framework which define how any *container* launched and controlled.
+
+The following are the available in Hadoop YARN:
+
+| ContainerExecutor | Description |
+|:---- |:---- |
+| `DefaultContainerExecutor` | The default executor which YARN uses to manage container execution. The container process has the same Unix user as the NodeManager. |
+| `LinuxContainerExecutor` | Supported only on GNU/Linux, this executor runs the containers as either the YARN user who submitted the application (when full security is enabled) or as a dedicated user (defaults to nobody) when full security is not enabled. When full security is enabled, this executor requires all user accounts to be created on the cluster nodes where the containers are launched. It uses a *setuid* executable that is included in the Hadoop distribution. The NodeManager uses this executable to launch and kill containers. The setuid executable switches to the user who has submitted the application and launches or kills the containers. For maximum security, this executor sets up restricted permissions and user/group ownership of local files and directories used by the containers such as the shared objects, jars, intermediate files, log files etc. Particularly note that, because of this, except the application owner and NodeManager, no other user can access any of the lo
cal files/directories including those localized as part of the distributed cache. |
+
+To build the LinuxContainerExecutor executable run:
+
+ $ mvn package -Dcontainer-executor.conf.dir=/etc/hadoop/
+
+The path passed in `-Dcontainer-executor.conf.dir` should be the path on the cluster nodes where a configuration file for the setuid executable should be located. The executable should be installed in $HADOOP\_YARN\_HOME/bin.
+
+The executable must have specific permissions: 6050 or --Sr-s--- permissions user-owned by *root* (super-user) and group-owned by a special group (e.g. `hadoop`) of which the NodeManager Unix user is the group member and no ordinary application user is. If any application user belongs to this special group, security will be compromised. This special group name should be specified for the configuration property `yarn.nodemanager.linux-container-executor.group` in both `conf/yarn-site.xml` and `conf/container-executor.cfg`.
+
+For example, let's say that the NodeManager is run as user *yarn* who is part of the groups users and *hadoop*, any of them being the primary group. Let also be that *users* has both *yarn* and another user (application submitter) *alice* as its members, and *alice* does not belong to *hadoop*. Going by the above description, the setuid/setgid executable should be set 6050 or --Sr-s--- with user-owner as *yarn* and group-owner as *hadoop* which has *yarn* as its member (and not *users* which has *alice* also as its member besides *yarn*).
+
+The LinuxTaskController requires that paths including and leading up to the directories specified in `yarn.nodemanager.local-dirs` and `yarn.nodemanager.log-dirs` to be set 755 permissions as described above in the table on permissions on directories.
+
+* `conf/container-executor.cfg`
+
+The executable requires a configuration file called `container-executor.cfg` to be present in the configuration directory passed to the mvn target mentioned above.
+
+The configuration file must be owned by the user running NodeManager (user `yarn` in the above example), group-owned by anyone and should have the permissions 0400 or r--------.
+
+The executable requires following configuration items to be present in the `conf/container-executor.cfg` file. The items should be mentioned as simple key=value pairs, one per-line:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.nodemanager.linux-container-executor.group` | *hadoop* | Unix group of the NodeManager. The group owner of the *container-executor* binary should be this group. Should be same as the value with which the NodeManager is configured. This configuration is required for validating the secure access of the *container-executor* binary. |
+| `banned.users` | hdfs,yarn,mapred,bin | Banned users. |
+| `allowed.system.users` | foo,bar | Allowed system users. |
+| `min.user.id` | 1000 | Prevent other super-users. |
+
+To re-cap, here are the local file-sysytem permissions required for the various paths related to the `LinuxContainerExecutor`:
+
+| Filesystem | Path | User:Group | Permissions |
+|:---- |:---- |:---- |:---- |
+| local | container-executor | root:hadoop | --Sr-s--* |
+| local | `conf/container-executor.cfg` | root:hadoop | r-------* |
+| local | `yarn.nodemanager.local-dirs` | yarn:hadoop | drwxr-xr-x |
+| local | `yarn.nodemanager.log-dirs` | yarn:hadoop | drwxr-xr-x |
+
+### MapReduce JobHistory Server
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `mapreduce.jobhistory.address` | MapReduce JobHistory Server *host:port* | Default port is 10020. |
+| `mapreduce.jobhistory.keytab` | */etc/security/keytab/jhs.service.keytab* | Kerberos keytab file for the MapReduce JobHistory Server. |
+| `mapreduce.jobhistory.principal` | jhs/\_HOST@REALM.TLD | Kerberos principal name for the MapReduce JobHistory Server. |
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md b/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
new file mode 100644
index 0000000..8b4a10f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ServiceLevelAuth.md
@@ -0,0 +1,144 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Service Level Authorization Guide
+=================================
+
+* [Service Level Authorization Guide](#Service_Level_Authorization_Guide)
+ * [Purpose](#Purpose)
+ * [Prerequisites](#Prerequisites)
+ * [Overview](#Overview)
+ * [Configuration](#Configuration)
+ * [Enable Service Level Authorization](#Enable_Service_Level_Authorization)
+ * [Hadoop Services and Configuration Properties](#Hadoop_Services_and_Configuration_Properties)
+ * [Access Control Lists](#Access_Control_Lists)
+ * [Refreshing Service Level Authorization Configuration](#Refreshing_Service_Level_Authorization_Configuration)
+ * [Examples](#Examples)
+
+Purpose
+-------
+
+This document describes how to configure and manage Service Level Authorization for Hadoop.
+
+Prerequisites
+-------------
+
+Make sure Hadoop is installed, configured and setup correctly. For more information see:
+
+* [Single Node Setup](./SingleCluster.html) for first-time users.
+* [Cluster Setup](./ClusterSetup.html) for large, distributed clusters.
+
+Overview
+--------
+
+Service Level Authorization is the initial authorization mechanism to ensure clients connecting to a particular Hadoop service have the necessary, pre-configured, permissions and are authorized to access the given service. For example, a MapReduce cluster can use this mechanism to allow a configured list of users/groups to submit jobs.
+
+The `$HADOOP_CONF_DIR/hadoop-policy.xml` configuration file is used to define the access control lists for various Hadoop services.
+
+Service Level Authorization is performed much before to other access control checks such as file-permission checks, access control on job queues etc.
+
+Configuration
+-------------
+
+This section describes how to configure service-level authorization via the configuration file `$HADOOP_CONF_DIR/hadoop-policy.xml`.
+
+### Enable Service Level Authorization
+
+By default, service-level authorization is disabled for Hadoop. To enable it set the configuration property hadoop.security.authorization to true in `$HADOOP_CONF_DIR/core-site.xml`.
+
+### Hadoop Services and Configuration Properties
+
+This section lists the various Hadoop services and their configuration knobs:
+
+| Property | Service |
+|:---- |:---- |
+| security.client.protocol.acl | ACL for ClientProtocol, which is used by user code via the DistributedFileSystem. |
+| security.client.datanode.protocol.acl | ACL for ClientDatanodeProtocol, the client-to-datanode protocol for block recovery. |
+| security.datanode.protocol.acl | ACL for DatanodeProtocol, which is used by datanodes to communicate with the namenode. |
+| security.inter.datanode.protocol.acl | ACL for InterDatanodeProtocol, the inter-datanode protocol for updating generation timestamp. |
+| security.namenode.protocol.acl | ACL for NamenodeProtocol, the protocol used by the secondary namenode to communicate with the namenode. |
+| security.inter.tracker.protocol.acl | ACL for InterTrackerProtocol, used by the tasktrackers to communicate with the jobtracker. |
+| security.job.submission.protocol.acl | ACL for JobSubmissionProtocol, used by job clients to communciate with the jobtracker for job submission, querying job status etc. |
+| security.task.umbilical.protocol.acl | ACL for TaskUmbilicalProtocol, used by the map and reduce tasks to communicate with the parent tasktracker. |
+| security.refresh.policy.protocol.acl | ACL for RefreshAuthorizationPolicyProtocol, used by the dfsadmin and mradmin commands to refresh the security policy in-effect. |
+| security.ha.service.protocol.acl | ACL for HAService protocol used by HAAdmin to manage the active and stand-by states of namenode. |
+
+### Access Control Lists
+
+`$HADOOP_CONF_DIR/hadoop-policy.xml` defines an access control list for each Hadoop service. Every access control list has a simple format:
+
+The list of users and groups are both comma separated list of names. The two lists are separated by a space.
+
+Example: `user1,user2 group1,group2`.
+
+Add a blank at the beginning of the line if only a list of groups is to be provided, equivalently a comma-separated list of users followed by a space or nothing implies only a set of given users.
+
+A special value of `*` implies that all users are allowed to access the service.
+
+If access control list is not defined for a service, the value of `security.service.authorization.default.acl` is applied. If `security.service.authorization.default.acl` is not defined, `*` is applied.
+
+* Blocked Access Control ListsIn some cases, it is required to specify blocked access control list for a service. This specifies the list of users and groups who are not authorized to access the service. The format of the blocked access control list is same as that of access control list. The blocked access control list can be specified via `$HADOOP_CONF_DIR/hadoop-policy.xml`. The property name is derived by suffixing with ".blocked".
+
+ Example: The property name of blocked access control list for `security.client.protocol.acl>> will be <<<security.client.protocol.acl.blocked`
+
+ For a service, it is possible to specify both an access control list and a blocked control list. A user is authorized to access the service if the user is in the access control and not in the blocked access control list.
+
+ If blocked access control list is not defined for a service, the value of `security.service.authorization.default.acl.blocked` is applied. If `security.service.authorization.default.acl.blocked` is not defined, empty blocked access control list is applied.
+
+### Refreshing Service Level Authorization Configuration
+
+The service-level authorization configuration for the NameNode and JobTracker can be changed without restarting either of the Hadoop master daemons. The cluster administrator can change `$HADOOP_CONF_DIR/hadoop-policy.xml` on the master nodes and instruct the NameNode and JobTracker to reload their respective configurations via the `-refreshServiceAcl` switch to `dfsadmin` and `mradmin` commands respectively.
+
+Refresh the service-level authorization configuration for the NameNode:
+
+ $ bin/hadoop dfsadmin -refreshServiceAcl
+
+Refresh the service-level authorization configuration for the JobTracker:
+
+ $ bin/hadoop mradmin -refreshServiceAcl
+
+Of course, one can use the `security.refresh.policy.protocol.acl` property in `$HADOOP_CONF_DIR/hadoop-policy.xml` to restrict access to the ability to refresh the service-level authorization configuration to certain users/groups.
+
+* Access Control using list of ip addresses, host names and ip rangesAccess to a service can be controlled based on the ip address of the client accessing the service. It is possible to restrict access to a service from a set of machines by specifying a list of ip addresses, host names and ip ranges. The property name for each service is derived from the corresponding acl's property name. If the property name of acl is security.client.protocol.acl, property name for the hosts list will be security.client.protocol.hosts.
+
+ If hosts list is not defined for a service, the value of `security.service.authorization.default.hosts` is applied. If `security.service.authorization.default.hosts` is not defined, `*` is applied.
+
+ It is possible to specify a blocked list of hosts. Only those machines which are in the hosts list, but not in the blocked hosts list will be granted access to the service. The property name is derived by suffixing with ".blocked".
+
+ Example: The property name of blocked hosts list for `security.client.protocol.hosts>> will be <<<security.client.protocol.hosts.blocked`
+
+ If blocked hosts list is not defined for a service, the value of `security.service.authorization.default.hosts.blocked` is applied. If `security.service.authorization.default.hosts.blocked` is not defined, empty blocked hosts list is applied.
+
+### Examples
+
+Allow only users `alice`, `bob` and users in the `mapreduce` group to submit jobs to the MapReduce cluster:
+
+ <property>
+ <name>security.job.submission.protocol.acl</name>
+ <value>alice,bob mapreduce</value>
+ </property>
+
+Allow only DataNodes running as the users who belong to the group datanodes to communicate with the NameNode:
+
+ <property>
+ <name>security.datanode.protocol.acl</name>
+ <value>datanodes</value>
+ </property>
+
+Allow any user to talk to the HDFS cluster as a DFSClient:
+
+ <property>
+ <name>security.client.protocol.acl</name>
+ <value>*</value>
+ </property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
new file mode 100644
index 0000000..ca5b48c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleCluster.md.vm
@@ -0,0 +1,232 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+#set ( $H3 = '###' )
+#set ( $H4 = '####' )
+#set ( $H5 = '#####' )
+
+Hadoop: Setting up a Single Node Cluster.
+=========================================
+
+* [Hadoop: Setting up a Single Node Cluster.](#Hadoop:_Setting_up_a_Single_Node_Cluster.)
+ * [Purpose](#Purpose)
+ * [Prerequisites](#Prerequisites)
+ * [Supported Platforms](#Supported_Platforms)
+ * [Required Software](#Required_Software)
+ * [Installing Software](#Installing_Software)
+ * [Download](#Download)
+ * [Prepare to Start the Hadoop Cluster](#Prepare_to_Start_the_Hadoop_Cluster)
+ * [Standalone Operation](#Standalone_Operation)
+ * [Pseudo-Distributed Operation](#Pseudo-Distributed_Operation)
+ * [Configuration](#Configuration)
+ * [Setup passphraseless ssh](#Setup_passphraseless_ssh)
+ * [Execution](#Execution)
+ * [YARN on a Single Node](#YARN_on_a_Single_Node)
+ * [Fully-Distributed Operation](#Fully-Distributed_Operation)
+
+Purpose
+-------
+
+This document describes how to set up and configure a single-node Hadoop installation so that you can quickly perform simple operations using Hadoop MapReduce and the Hadoop Distributed File System (HDFS).
+
+Prerequisites
+-------------
+
+$H3 Supported Platforms
+
+* GNU/Linux is supported as a development and production platform. Hadoop has been demonstrated on GNU/Linux clusters with 2000 nodes.
+
+* Windows is also a supported platform but the followings steps are for Linux only. To set up Hadoop on Windows, see [wiki page](http://wiki.apache.org/hadoop/Hadoop2OnWindows).
+
+$H3 Required Software
+
+Required software for Linux include:
+
+1. Java™ must be installed. Recommended Java versions are described at [HadoopJavaVersions](http://wiki.apache.org/hadoop/HadoopJavaVersions).
+
+2. ssh must be installed and sshd must be running to use the Hadoop scripts that manage remote Hadoop daemons if the optional start and stop scripts are to be used. Additionally, it is recommmended that pdsh also be installed for better ssh resource management.
+
+$H3 Installing Software
+
+If your cluster doesn't have the requisite software you will need to install it.
+
+For example on Ubuntu Linux:
+
+ $ sudo apt-get install ssh
+ $ sudo apt-get install pdsh
+
+Download
+--------
+
+To get a Hadoop distribution, download a recent stable release from one of the [Apache Download Mirrors](http://www.apache.org/dyn/closer.cgi/hadoop/common/).
+
+Prepare to Start the Hadoop Cluster
+-----------------------------------
+
+Unpack the downloaded Hadoop distribution. In the distribution, edit the file `etc/hadoop/hadoop-env.sh` to define some parameters as follows:
+
+ # set to the root of your Java installation
+ export JAVA_HOME=/usr/java/latest
+
+Try the following command:
+
+ $ bin/hadoop
+
+This will display the usage documentation for the hadoop script.
+
+Now you are ready to start your Hadoop cluster in one of the three supported modes:
+
+* [Local (Standalone) Mode](#Standalone_Operation)
+* [Pseudo-Distributed Mode](#Pseudo-Distributed_Operation)
+* [Fully-Distributed Mode](#Fully-Distributed_Operation)
+
+Standalone Operation
+--------------------
+
+By default, Hadoop is configured to run in a non-distributed mode, as a single Java process. This is useful for debugging.
+
+The following example copies the unpacked conf directory to use as input and then finds and displays every match of the given regular expression. Output is written to the given output directory.
+
+ $ mkdir input
+ $ cp etc/hadoop/*.xml input
+ $ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar grep input output 'dfs[a-z.]+'
+ $ cat output/*
+
+Pseudo-Distributed Operation
+----------------------------
+
+Hadoop can also be run on a single-node in a pseudo-distributed mode where each Hadoop daemon runs in a separate Java process.
+
+$H3 Configuration
+
+Use the following:
+
+etc/hadoop/core-site.xml:
+
+ <configuration>
+ <property>
+ <name>fs.defaultFS</name>
+ <value>hdfs://localhost:9000</value>
+ </property>
+ </configuration>
+
+etc/hadoop/hdfs-site.xml:
+
+ <configuration>
+ <property>
+ <name>dfs.replication</name>
+ <value>1</value>
+ </property>
+ </configuration>
+
+$H3 Setup passphraseless ssh
+
+Now check that you can ssh to the localhost without a passphrase:
+
+ $ ssh localhost
+
+If you cannot ssh to localhost without a passphrase, execute the following commands:
+
+ $ ssh-keygen -t dsa -P '' -f ~/.ssh/id_dsa
+ $ cat ~/.ssh/id_dsa.pub >> ~/.ssh/authorized_keys
+ $ chmod 0700 ~/.ssh/authorized_keys
+
+$H3 Execution
+
+The following instructions are to run a MapReduce job locally. If you want to execute a job on YARN, see [YARN on Single Node](#YARN_on_Single_Node).
+
+1. Format the filesystem:
+
+ $ bin/hdfs namenode -format
+
+2. Start NameNode daemon and DataNode daemon:
+
+ $ sbin/start-dfs.sh
+
+ The hadoop daemon log output is written to the `$HADOOP_LOG_DIR` directory (defaults to `$HADOOP_HOME/logs`).
+
+3. Browse the web interface for the NameNode; by default it is available at:
+
+ * NameNode - `http://localhost:50070/`
+
+4. Make the HDFS directories required to execute MapReduce jobs:
+
+ $ bin/hdfs dfs -mkdir /user
+ $ bin/hdfs dfs -mkdir /user/<username>
+
+5. Copy the input files into the distributed filesystem:
+
+ $ bin/hdfs dfs -put etc/hadoop input
+
+6. Run some of the examples provided:
+
+ $ bin/hadoop jar share/hadoop/mapreduce/hadoop-mapreduce-examples-${project.version}.jar grep input output 'dfs[a-z.]+'
+
+7. Examine the output files: Copy the output files from the distributed filesystem to the local filesystem and examine them:
+
+ $ bin/hdfs dfs -get output output
+ $ cat output/*
+
+ or
+
+ View the output files on the distributed filesystem:
+
+ $ bin/hdfs dfs -cat output/*
+
+8. When you're done, stop the daemons with:
+
+ $ sbin/stop-dfs.sh
+
+$H3 YARN on a Single Node
+
+You can run a MapReduce job on YARN in a pseudo-distributed mode by setting a few parameters and running ResourceManager daemon and NodeManager daemon in addition.
+
+The following instructions assume that 1. ~ 4. steps of [the above instructions](#Execution) are already executed.
+
+1. Configure parameters as follows:`etc/hadoop/mapred-site.xml`:
+
+ <configuration>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+ </configuration>
+
+ `etc/hadoop/yarn-site.xml`:
+
+ <configuration>
+ <property>
+ <name>yarn.nodemanager.aux-services</name>
+ <value>mapreduce_shuffle</value>
+ </property>
+ </configuration>
+
+2. Start ResourceManager daemon and NodeManager daemon:
+
+ $ sbin/start-yarn.sh
+
+3. Browse the web interface for the ResourceManager; by default it is available at:
+
+ * ResourceManager - `http://localhost:8088/`
+
+4. Run a MapReduce job.
+
+5. When you're done, stop the daemons with:
+
+ $ sbin/stop-yarn.sh
+
+Fully-Distributed Operation
+---------------------------
+
+For information on setting up fully-distributed, non-trivial clusters see [Cluster Setup](./ClusterSetup.html).
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md
new file mode 100644
index 0000000..fae8b5c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md
@@ -0,0 +1,20 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Single Node Setup
+=================
+
+This page will be removed in the next major release.
+
+See [Single Cluster Setup](./SingleCluster.html) to set up and configure a single-node Hadoop installation.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/Superusers.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Superusers.md b/hadoop-common-project/hadoop-common/src/site/markdown/Superusers.md
new file mode 100644
index 0000000..8c9fb72
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Superusers.md
@@ -0,0 +1,106 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Proxy user - Superusers Acting On Behalf Of Other Users
+=======================================================
+
+* [Proxy user - Superusers Acting On Behalf Of Other Users](#Proxy_user_-_Superusers_Acting_On_Behalf_Of_Other_Users)
+ * [Introduction](#Introduction)
+ * [Use Case](#Use_Case)
+ * [Code example](#Code_example)
+ * [Configurations](#Configurations)
+ * [Caveats](#Caveats)
+
+Introduction
+------------
+
+This document describes how a superuser can submit jobs or access hdfs on behalf of another user.
+
+Use Case
+--------
+
+The code example described in the next section is applicable for the following use case.
+
+A superuser with username 'super' wants to submit job and access hdfs on behalf of a user joe. The superuser has kerberos credentials but user joe doesn't have any. The tasks are required to run as user joe and any file accesses on namenode are required to be done as user joe. It is required that user joe can connect to the namenode or job tracker on a connection authenticated with super's kerberos credentials. In other words super is impersonating the user joe.
+
+Some products such as Apache Oozie need this.
+
+Code example
+------------
+
+In this example super's credentials are used for login and a proxy user ugi object is created for joe. The operations are performed within the doAs method of this proxy user ugi object.
+
+ ...
+ //Create ugi for joe. The login user is 'super'.
+ UserGroupInformation ugi =
+ UserGroupInformation.createProxyUser("joe", UserGroupInformation.getLoginUser());
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ public Void run() throws Exception {
+ //Submit a job
+ JobClient jc = new JobClient(conf);
+ jc.submitJob(conf);
+ //OR access hdfs
+ FileSystem fs = FileSystem.get(conf);
+ fs.mkdir(someFilePath);
+ }
+ }
+
+Configurations
+--------------
+
+You can configure proxy user using properties `hadoop.proxyuser.$superuser.hosts` along with either or both of `hadoop.proxyuser.$superuser.groups` and `hadoop.proxyuser.$superuser.users`.
+
+By specifying as below in core-site.xml, the superuser named `super` can connect only from `host1` and `host2` to impersonate a user belonging to `group1` and `group2`.
+
+ <property>
+ <name>hadoop.proxyuser.super.hosts</name>
+ <value>host1,host2</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.super.groups</name>
+ <value>group1,group2</value>
+ </property>
+
+If these configurations are not present, impersonation will not be allowed and connection will fail.
+
+If more lax security is preferred, the wildcard value \* may be used to allow impersonation from any host or of any user. For example, by specifying as below in core-site.xml, user named `oozie` accessing from any host can impersonate any user belonging to any group.
+
+ <property>
+ <name>hadoop.proxyuser.oozie.hosts</name>
+ <value>*</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.oozie.groups</name>
+ <value>*</value>
+ </property>
+
+The `hadoop.proxyuser.$superuser.hosts` accepts list of ip addresses, ip address ranges in CIDR format and/or host names. For example, by specifying as below, user named `super` accessing from hosts in the range `10.222.0.0-15` and `10.113.221.221` can impersonate `user1` and `user2`.
+
+ <property>
+ <name>hadoop.proxyuser.super.hosts</name>
+ <value>10.222.0.0/16,10.113.221.221</value>
+ </property>
+ <property>
+ <name>hadoop.proxyuser.super.users</name>
+ <value>user1,user2</value>
+ </property>
+
+Caveats
+-------
+
+If the cluster is running in [Secure Mode](./SecureMode.html), the superuser must have kerberos credentials to be able to impersonate another user.
+
+It cannot use delegation tokens for this feature. It would be wrong if superuser adds its own delegation token to the proxy user ugi, as it will allow the proxy user to connect to the service with the privileges of the superuser.
+
+However, if the superuser does want to give a delegation token to joe, it must first impersonate joe and get a delegation token for joe, in the same way as the code example above, and add it to the ugi of joe. In this way the delegation token will have the owner as joe.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
new file mode 100644
index 0000000..84c95e0
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Tracing.md
@@ -0,0 +1,182 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Enabling Dapper-like Tracing in Hadoop
+======================================
+
+* [Enabling Dapper-like Tracing in Hadoop](#Enabling_Dapper-like_Tracing_in_Hadoop)
+ * [Dapper-like Tracing in Hadoop](#Dapper-like_Tracing_in_Hadoop)
+ * [HTrace](#HTrace)
+ * [Samplers Configure the samplers in core-site.xml property: hadoop.htrace.sampler. The value can be NeverSampler, AlwaysSampler or ProbabilitySampler. NeverSampler: HTrace is OFF for all spans; AlwaysSampler: HTrace is ON for all spans; ProbabilitySampler: HTrace is ON for some percentage% of top-level spans.](#Samplers_Configure_the_samplers_in_core-site.xml_property:_hadoop.htrace.sampler._The_value_can_be_NeverSampler_AlwaysSampler_or_ProbabilitySampler._NeverSampler:_HTrace_is_OFF_for_all_spans_AlwaysSampler:_HTrace_is_ON_for_all_spans_ProbabilitySampler:_HTrace_is_ON_for_some_percentage_of_top-level_spans.)
+ * [SpanReceivers](#SpanReceivers)
+ * [Setting up ZipkinSpanReceiver](#Setting_up_ZipkinSpanReceiver)
+ * [Dynamic update of tracing configuration](#Dynamic_update_of_tracing_configuration)
+ * [Starting tracing spans by HTrace API](#Starting_tracing_spans_by_HTrace_API)
+ * [Sample code for tracing](#Sample_code_for_tracing)
+
+Dapper-like Tracing in Hadoop
+-----------------------------
+
+### HTrace
+
+[HDFS-5274](https://issues.apache.org/jira/browse/HDFS-5274) added support for tracing requests through HDFS, using the open source tracing library, [Apache HTrace](https://git-wip-us.apache.org/repos/asf/incubator-htrace.git). Setting up tracing is quite simple, however it requires some very minor changes to your client code.
+
+### Samplers Configure the samplers in `core-site.xml` property: `hadoop.htrace.sampler`. The value can be NeverSampler, AlwaysSampler or ProbabilitySampler. NeverSampler: HTrace is OFF for all spans; AlwaysSampler: HTrace is ON for all spans; ProbabilitySampler: HTrace is ON for some percentage% of top-level spans.
+
+ <property>
+ <name>hadoop.htrace.sampler</name>
+ <value>NeverSampler</value>
+ </property>
+
+### SpanReceivers
+
+The tracing system works by collecting information in structs called 'Spans'. It is up to you to choose how you want to receive this information by implementing the SpanReceiver interface, which defines one method:
+
+ public void receiveSpan(Span span);
+
+Configure what SpanReceivers you'd like to use by putting a comma separated list of the fully-qualified class name of classes implementing SpanReceiver in `core-site.xml` property: `hadoop.htrace.spanreceiver.classes`.
+
+ <property>
+ <name>hadoop.htrace.spanreceiver.classes</name>
+ <value>org.apache.htrace.impl.LocalFileSpanReceiver</value>
+ </property>
+ <property>
+ <name>hadoop.htrace.local-file-span-receiver.path</name>
+ <value>/var/log/hadoop/htrace.out</value>
+ </property>
+
+You can omit package name prefix if you use span receiver bundled with HTrace.
+
+ <property>
+ <name>hadoop.htrace.spanreceiver.classes</name>
+ <value>LocalFileSpanReceiver</value>
+ </property>
+
+### Setting up ZipkinSpanReceiver
+
+Instead of implementing SpanReceiver by yourself, you can use `ZipkinSpanReceiver` which uses [Zipkin](https://github.com/twitter/zipkin) for collecting and displaying tracing data.
+
+In order to use `ZipkinSpanReceiver`, you need to download and setup [Zipkin](https://github.com/twitter/zipkin) first.
+
+you also need to add the jar of `htrace-zipkin` to the classpath of Hadoop on each node. Here is example setup procedure.
+
+ $ git clone https://github.com/cloudera/htrace
+ $ cd htrace/htrace-zipkin
+ $ mvn compile assembly:single
+ $ cp target/htrace-zipkin-*-jar-with-dependencies.jar $HADOOP_HOME/share/hadoop/common/lib/
+
+The sample configuration for `ZipkinSpanReceiver` is shown below. By adding these to `core-site.xml` of NameNode and DataNodes, `ZipkinSpanReceiver` is initialized on the startup. You also need this configuration on the client node in addition to the servers.
+
+ <property>
+ <name>hadoop.htrace.spanreceiver.classes</name>
+ <value>ZipkinSpanReceiver</value>
+ </property>
+ <property>
+ <name>hadoop.htrace.zipkin.collector-hostname</name>
+ <value>192.168.1.2</value>
+ </property>
+ <property>
+ <name>hadoop.htrace.zipkin.collector-port</name>
+ <value>9410</value>
+ </property>
+
+### Dynamic update of tracing configuration
+
+You can use `hadoop trace` command to see and update the tracing configuration of each servers. You must specify IPC server address of namenode or datanode by `-host` option. You need to run the command against all servers if you want to update the configuration of all servers.
+
+`hadoop trace -list` shows list of loaded span receivers associated with the id.
+
+ $ hadoop trace -list -host 192.168.56.2:9000
+ ID CLASS
+ 1 org.apache.htrace.impl.LocalFileSpanReceiver
+
+ $ hadoop trace -list -host 192.168.56.2:50020
+ ID CLASS
+ 1 org.apache.htrace.impl.LocalFileSpanReceiver
+
+`hadoop trace -remove` removes span receiver from server. `-remove` options takes id of span receiver as argument.
+
+ $ hadoop trace -remove 1 -host 192.168.56.2:9000
+ Removed trace span receiver 1
+
+`hadoop trace -add` adds span receiver to server. You need to specify the class name of span receiver as argument of `-class` option. You can specify the configuration associated with span receiver by `-Ckey=value` options.
+
+ $ hadoop trace -add -class LocalFileSpanReceiver -Chadoop.htrace.local-file-span-receiver.path=/tmp/htrace.out -host 192.168.56.2:9000
+ Added trace span receiver 2 with configuration hadoop.htrace.local-file-span-receiver.path = /tmp/htrace.out
+
+ $ hadoop trace -list -host 192.168.56.2:9000
+ ID CLASS
+ 2 org.apache.htrace.impl.LocalFileSpanReceiver
+
+### Starting tracing spans by HTrace API
+
+In order to trace, you will need to wrap the traced logic with **tracing span** as shown below. When there is running tracing spans, the tracing information is propagated to servers along with RPC requests.
+
+In addition, you need to initialize `SpanReceiver` once per process.
+
+ import org.apache.hadoop.hdfs.HdfsConfiguration;
+ import org.apache.hadoop.tracing.SpanReceiverHost;
+ import org.apache.htrace.Sampler;
+ import org.apache.htrace.Trace;
+ import org.apache.htrace.TraceScope;
+
+ ...
+
+ SpanReceiverHost.getInstance(new HdfsConfiguration());
+
+ ...
+
+ TraceScope ts = Trace.startSpan("Gets", Sampler.ALWAYS);
+ try {
+ ... // traced logic
+ } finally {
+ if (ts != null) ts.close();
+ }
+
+### Sample code for tracing
+
+The `TracingFsShell.java` shown below is the wrapper of FsShell which start tracing span before invoking HDFS shell command.
+
+ import org.apache.hadoop.conf.Configuration;
+ import org.apache.hadoop.fs.FsShell;
+ import org.apache.hadoop.tracing.SpanReceiverHost;
+ import org.apache.hadoop.util.ToolRunner;
+ import org.apache.htrace.Sampler;
+ import org.apache.htrace.Trace;
+ import org.apache.htrace.TraceScope;
+
+ public class TracingFsShell {
+ public static void main(String argv[]) throws Exception {
+ Configuration conf = new Configuration();
+ FsShell shell = new FsShell();
+ conf.setQuietMode(false);
+ shell.setConf(conf);
+ SpanReceiverHost.getInstance(conf);
+ int res = 0;
+ TraceScope ts = null;
+ try {
+ ts = Trace.startSpan("FsShell", Sampler.ALWAYS);
+ res = ToolRunner.run(shell, argv);
+ } finally {
+ shell.close();
+ if (ts != null) ts.close();
+ }
+ System.exit(res);
+ }
+ }
+
+You can compile and execute this code as shown below.
+
+ $ javac -cp `hadoop classpath` TracingFsShell.java
+ $ java -cp .:`hadoop classpath` TracingFsShell -ls /
[28/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
deleted file mode 100644
index c4f3b1e..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
+++ /dev/null
@@ -1,552 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop ${project.version}
- ---
- ---
- ${maven.build.timestamp}
-
-Deprecated Properties
-
- The following table lists the configuration property names that are
- deprecated in this version of Hadoop, and their replacements.
-
-*-------------------------------+-----------------------+
-|| <<Deprecated property name>> || <<New property name>>|
-*-------------------------------+-----------------------+
-|create.empty.dir.if.nonexist | mapreduce.jobcontrol.createdir.ifnotexist
-*---+---+
-|dfs.access.time.precision | dfs.namenode.accesstime.precision
-*---+---+
-|dfs.backup.address | dfs.namenode.backup.address
-*---+---+
-|dfs.backup.http.address | dfs.namenode.backup.http-address
-*---+---+
-|dfs.balance.bandwidthPerSec | dfs.datanode.balance.bandwidthPerSec
-*---+---+
-|dfs.block.size | dfs.blocksize
-*---+---+
-|dfs.data.dir | dfs.datanode.data.dir
-*---+---+
-|dfs.datanode.max.xcievers | dfs.datanode.max.transfer.threads
-*---+---+
-|dfs.df.interval | fs.df.interval
-*---+---+
-|dfs.federation.nameservice.id | dfs.nameservice.id
-*---+---+
-|dfs.federation.nameservices | dfs.nameservices
-*---+---+
-|dfs.http.address | dfs.namenode.http-address
-*---+---+
-|dfs.https.address | dfs.namenode.https-address
-*---+---+
-|dfs.https.client.keystore.resource | dfs.client.https.keystore.resource
-*---+---+
-|dfs.https.need.client.auth | dfs.client.https.need-auth
-*---+---+
-|dfs.max.objects | dfs.namenode.max.objects
-*---+---+
-|dfs.max-repl-streams | dfs.namenode.replication.max-streams
-*---+---+
-|dfs.name.dir | dfs.namenode.name.dir
-*---+---+
-|dfs.name.dir.restore | dfs.namenode.name.dir.restore
-*---+---+
-|dfs.name.edits.dir | dfs.namenode.edits.dir
-*---+---+
-|dfs.permissions | dfs.permissions.enabled
-*---+---+
-|dfs.permissions.supergroup | dfs.permissions.superusergroup
-*---+---+
-|dfs.read.prefetch.size | dfs.client.read.prefetch.size
-*---+---+
-|dfs.replication.considerLoad | dfs.namenode.replication.considerLoad
-*---+---+
-|dfs.replication.interval | dfs.namenode.replication.interval
-*---+---+
-|dfs.replication.min | dfs.namenode.replication.min
-*---+---+
-|dfs.replication.pending.timeout.sec | dfs.namenode.replication.pending.timeout-sec
-*---+---+
-|dfs.safemode.extension | dfs.namenode.safemode.extension
-*---+---+
-|dfs.safemode.threshold.pct | dfs.namenode.safemode.threshold-pct
-*---+---+
-|dfs.secondary.http.address | dfs.namenode.secondary.http-address
-*---+---+
-|dfs.socket.timeout | dfs.client.socket-timeout
-*---+---+
-|dfs.umaskmode | fs.permissions.umask-mode
-*---+---+
-|dfs.write.packet.size | dfs.client-write-packet-size
-*---+---+
-|fs.checkpoint.dir | dfs.namenode.checkpoint.dir
-*---+---+
-|fs.checkpoint.edits.dir | dfs.namenode.checkpoint.edits.dir
-*---+---+
-|fs.checkpoint.period | dfs.namenode.checkpoint.period
-*---+---+
-|fs.default.name | fs.defaultFS
-*---+---+
-|hadoop.configured.node.mapping | net.topology.configured.node.mapping
-*---+---+
-|hadoop.job.history.location | mapreduce.jobtracker.jobhistory.location
-*---+---+
-|hadoop.native.lib | io.native.lib.available
-*---+---+
-|hadoop.net.static.resolutions | mapreduce.tasktracker.net.static.resolutions
-*---+---+
-|hadoop.pipes.command-file.keep | mapreduce.pipes.commandfile.preserve
-*---+---+
-|hadoop.pipes.executable.interpretor | mapreduce.pipes.executable.interpretor
-*---+---+
-|hadoop.pipes.executable | mapreduce.pipes.executable
-*---+---+
-|hadoop.pipes.java.mapper | mapreduce.pipes.isjavamapper
-*---+---+
-|hadoop.pipes.java.recordreader | mapreduce.pipes.isjavarecordreader
-*---+---+
-|hadoop.pipes.java.recordwriter | mapreduce.pipes.isjavarecordwriter
-*---+---+
-|hadoop.pipes.java.reducer | mapreduce.pipes.isjavareducer
-*---+---+
-|hadoop.pipes.partitioner | mapreduce.pipes.partitioner
-*---+---+
-|heartbeat.recheck.interval | dfs.namenode.heartbeat.recheck-interval
-*---+---+
-|io.bytes.per.checksum | dfs.bytes-per-checksum
-*---+---+
-|io.sort.factor | mapreduce.task.io.sort.factor
-*---+---+
-|io.sort.mb | mapreduce.task.io.sort.mb
-*---+---+
-|io.sort.spill.percent | mapreduce.map.sort.spill.percent
-*---+---+
-|jobclient.completion.poll.interval | mapreduce.client.completion.pollinterval
-*---+---+
-|jobclient.output.filter | mapreduce.client.output.filter
-*---+---+
-|jobclient.progress.monitor.poll.interval | mapreduce.client.progressmonitor.pollinterval
-*---+---+
-|job.end.notification.url | mapreduce.job.end-notification.url
-*---+---+
-|job.end.retry.attempts | mapreduce.job.end-notification.retry.attempts
-*---+---+
-|job.end.retry.interval | mapreduce.job.end-notification.retry.interval
-*---+---+
-|job.local.dir | mapreduce.job.local.dir
-*---+---+
-|keep.failed.task.files | mapreduce.task.files.preserve.failedtasks
-*---+---+
-|keep.task.files.pattern | mapreduce.task.files.preserve.filepattern
-*---+---+
-|key.value.separator.in.input.line | mapreduce.input.keyvaluelinerecordreader.key.value.separator
-*---+---+
-|local.cache.size | mapreduce.tasktracker.cache.local.size
-*---+---+
-|map.input.file | mapreduce.map.input.file
-*---+---+
-|map.input.length | mapreduce.map.input.length
-*---+---+
-|map.input.start | mapreduce.map.input.start
-*---+---+
-|map.output.key.field.separator | mapreduce.map.output.key.field.separator
-*---+---+
-|map.output.key.value.fields.spec | mapreduce.fieldsel.map.output.key.value.fields.spec
-*---+---+
-|mapred.acls.enabled | mapreduce.cluster.acls.enabled
-*---+---+
-|mapred.binary.partitioner.left.offset | mapreduce.partition.binarypartitioner.left.offset
-*---+---+
-|mapred.binary.partitioner.right.offset | mapreduce.partition.binarypartitioner.right.offset
-*---+---+
-|mapred.cache.archives | mapreduce.job.cache.archives
-*---+---+
-|mapred.cache.archives.timestamps | mapreduce.job.cache.archives.timestamps
-*---+---+
-|mapred.cache.files | mapreduce.job.cache.files
-*---+---+
-|mapred.cache.files.timestamps | mapreduce.job.cache.files.timestamps
-*---+---+
-|mapred.cache.localArchives | mapreduce.job.cache.local.archives
-*---+---+
-|mapred.cache.localFiles | mapreduce.job.cache.local.files
-*---+---+
-|mapred.child.tmp | mapreduce.task.tmp.dir
-*---+---+
-|mapred.cluster.average.blacklist.threshold | mapreduce.jobtracker.blacklist.average.threshold
-*---+---+
-|mapred.cluster.map.memory.mb | mapreduce.cluster.mapmemory.mb
-*---+---+
-|mapred.cluster.max.map.memory.mb | mapreduce.jobtracker.maxmapmemory.mb
-*---+---+
-|mapred.cluster.max.reduce.memory.mb | mapreduce.jobtracker.maxreducememory.mb
-*---+---+
-|mapred.cluster.reduce.memory.mb | mapreduce.cluster.reducememory.mb
-*---+---+
-|mapred.committer.job.setup.cleanup.needed | mapreduce.job.committer.setup.cleanup.needed
-*---+---+
-|mapred.compress.map.output | mapreduce.map.output.compress
-*---+---+
-|mapred.data.field.separator | mapreduce.fieldsel.data.field.separator
-*---+---+
-|mapred.debug.out.lines | mapreduce.task.debugout.lines
-*---+---+
-|mapred.healthChecker.interval | mapreduce.tasktracker.healthchecker.interval
-*---+---+
-|mapred.healthChecker.script.args | mapreduce.tasktracker.healthchecker.script.args
-*---+---+
-|mapred.healthChecker.script.path | mapreduce.tasktracker.healthchecker.script.path
-*---+---+
-|mapred.healthChecker.script.timeout | mapreduce.tasktracker.healthchecker.script.timeout
-*---+---+
-|mapred.heartbeats.in.second | mapreduce.jobtracker.heartbeats.in.second
-*---+---+
-|mapred.hosts.exclude | mapreduce.jobtracker.hosts.exclude.filename
-*---+---+
-|mapred.hosts | mapreduce.jobtracker.hosts.filename
-*---+---+
-|mapred.inmem.merge.threshold | mapreduce.reduce.merge.inmem.threshold
-*---+---+
-|mapred.input.dir.formats | mapreduce.input.multipleinputs.dir.formats
-*---+---+
-|mapred.input.dir.mappers | mapreduce.input.multipleinputs.dir.mappers
-*---+---+
-|mapred.input.dir | mapreduce.input.fileinputformat.inputdir
-*---+---+
-|mapred.input.pathFilter.class | mapreduce.input.pathFilter.class
-*---+---+
-|mapred.jar | mapreduce.job.jar
-*---+---+
-|mapred.job.classpath.archives | mapreduce.job.classpath.archives
-*---+---+
-|mapred.job.classpath.files | mapreduce.job.classpath.files
-*---+---+
-|mapred.job.id | mapreduce.job.id
-*---+---+
-|mapred.jobinit.threads | mapreduce.jobtracker.jobinit.threads
-*---+---+
-|mapred.job.map.memory.mb | mapreduce.map.memory.mb
-*---+---+
-|mapred.job.name | mapreduce.job.name
-*---+---+
-|mapred.job.priority | mapreduce.job.priority
-*---+---+
-|mapred.job.queue.name | mapreduce.job.queuename
-*---+---+
-|mapred.job.reduce.input.buffer.percent | mapreduce.reduce.input.buffer.percent
-*---+---+
-|mapred.job.reduce.markreset.buffer.percent | mapreduce.reduce.markreset.buffer.percent
-*---+---+
-|mapred.job.reduce.memory.mb | mapreduce.reduce.memory.mb
-*---+---+
-|mapred.job.reduce.total.mem.bytes | mapreduce.reduce.memory.totalbytes
-*---+---+
-|mapred.job.reuse.jvm.num.tasks | mapreduce.job.jvm.numtasks
-*---+---+
-|mapred.job.shuffle.input.buffer.percent | mapreduce.reduce.shuffle.input.buffer.percent
-*---+---+
-|mapred.job.shuffle.merge.percent | mapreduce.reduce.shuffle.merge.percent
-*---+---+
-|mapred.job.tracker.handler.count | mapreduce.jobtracker.handler.count
-*---+---+
-|mapred.job.tracker.history.completed.location | mapreduce.jobtracker.jobhistory.completed.location
-*---+---+
-|mapred.job.tracker.http.address | mapreduce.jobtracker.http.address
-*---+---+
-|mapred.jobtracker.instrumentation | mapreduce.jobtracker.instrumentation
-*---+---+
-|mapred.jobtracker.job.history.block.size | mapreduce.jobtracker.jobhistory.block.size
-*---+---+
-|mapred.job.tracker.jobhistory.lru.cache.size | mapreduce.jobtracker.jobhistory.lru.cache.size
-*---+---+
-|mapred.job.tracker | mapreduce.jobtracker.address
-*---+---+
-|mapred.jobtracker.maxtasks.per.job | mapreduce.jobtracker.maxtasks.perjob
-*---+---+
-|mapred.job.tracker.persist.jobstatus.active | mapreduce.jobtracker.persist.jobstatus.active
-*---+---+
-|mapred.job.tracker.persist.jobstatus.dir | mapreduce.jobtracker.persist.jobstatus.dir
-*---+---+
-|mapred.job.tracker.persist.jobstatus.hours | mapreduce.jobtracker.persist.jobstatus.hours
-*---+---+
-|mapred.jobtracker.restart.recover | mapreduce.jobtracker.restart.recover
-*---+---+
-|mapred.job.tracker.retiredjobs.cache.size | mapreduce.jobtracker.retiredjobs.cache.size
-*---+---+
-|mapred.job.tracker.retire.jobs | mapreduce.jobtracker.retirejobs
-*---+---+
-|mapred.jobtracker.taskalloc.capacitypad | mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad
-*---+---+
-|mapred.jobtracker.taskScheduler | mapreduce.jobtracker.taskscheduler
-*---+---+
-|mapred.jobtracker.taskScheduler.maxRunningTasksPerJob | mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob
-*---+---+
-|mapred.join.expr | mapreduce.join.expr
-*---+---+
-|mapred.join.keycomparator | mapreduce.join.keycomparator
-*---+---+
-|mapred.lazy.output.format | mapreduce.output.lazyoutputformat.outputformat
-*---+---+
-|mapred.line.input.format.linespermap | mapreduce.input.lineinputformat.linespermap
-*---+---+
-|mapred.linerecordreader.maxlength | mapreduce.input.linerecordreader.line.maxlength
-*---+---+
-|mapred.local.dir | mapreduce.cluster.local.dir
-*---+---+
-|mapred.local.dir.minspacekill | mapreduce.tasktracker.local.dir.minspacekill
-*---+---+
-|mapred.local.dir.minspacestart | mapreduce.tasktracker.local.dir.minspacestart
-*---+---+
-|mapred.map.child.env | mapreduce.map.env
-*---+---+
-|mapred.map.child.java.opts | mapreduce.map.java.opts
-*---+---+
-|mapred.map.child.log.level | mapreduce.map.log.level
-*---+---+
-|mapred.map.max.attempts | mapreduce.map.maxattempts
-*---+---+
-|mapred.map.output.compression.codec | mapreduce.map.output.compress.codec
-*---+---+
-|mapred.mapoutput.key.class | mapreduce.map.output.key.class
-*---+---+
-|mapred.mapoutput.value.class | mapreduce.map.output.value.class
-*---+---+
-|mapred.mapper.regex.group | mapreduce.mapper.regexmapper..group
-*---+---+
-|mapred.mapper.regex | mapreduce.mapper.regex
-*---+---+
-|mapred.map.task.debug.script | mapreduce.map.debug.script
-*---+---+
-|mapred.map.tasks | mapreduce.job.maps
-*---+---+
-|mapred.map.tasks.speculative.execution | mapreduce.map.speculative
-*---+---+
-|mapred.max.map.failures.percent | mapreduce.map.failures.maxpercent
-*---+---+
-|mapred.max.reduce.failures.percent | mapreduce.reduce.failures.maxpercent
-*---+---+
-|mapred.max.split.size | mapreduce.input.fileinputformat.split.maxsize
-*---+---+
-|mapred.max.tracker.blacklists | mapreduce.jobtracker.tasktracker.maxblacklists
-*---+---+
-|mapred.max.tracker.failures | mapreduce.job.maxtaskfailures.per.tracker
-*---+---+
-|mapred.merge.recordsBeforeProgress | mapreduce.task.merge.progress.records
-*---+---+
-|mapred.min.split.size | mapreduce.input.fileinputformat.split.minsize
-*---+---+
-|mapred.min.split.size.per.node | mapreduce.input.fileinputformat.split.minsize.per.node
-*---+---+
-|mapred.min.split.size.per.rack | mapreduce.input.fileinputformat.split.minsize.per.rack
-*---+---+
-|mapred.output.compression.codec | mapreduce.output.fileoutputformat.compress.codec
-*---+---+
-|mapred.output.compression.type | mapreduce.output.fileoutputformat.compress.type
-*---+---+
-|mapred.output.compress | mapreduce.output.fileoutputformat.compress
-*---+---+
-|mapred.output.dir | mapreduce.output.fileoutputformat.outputdir
-*---+---+
-|mapred.output.key.class | mapreduce.job.output.key.class
-*---+---+
-|mapred.output.key.comparator.class | mapreduce.job.output.key.comparator.class
-*---+---+
-|mapred.output.value.class | mapreduce.job.output.value.class
-*---+---+
-|mapred.output.value.groupfn.class | mapreduce.job.output.group.comparator.class
-*---+---+
-|mapred.permissions.supergroup | mapreduce.cluster.permissions.supergroup
-*---+---+
-|mapred.pipes.user.inputformat | mapreduce.pipes.inputformat
-*---+---+
-|mapred.reduce.child.env | mapreduce.reduce.env
-*---+---+
-|mapred.reduce.child.java.opts | mapreduce.reduce.java.opts
-*---+---+
-|mapred.reduce.child.log.level | mapreduce.reduce.log.level
-*---+---+
-|mapred.reduce.max.attempts | mapreduce.reduce.maxattempts
-*---+---+
-|mapred.reduce.parallel.copies | mapreduce.reduce.shuffle.parallelcopies
-*---+---+
-|mapred.reduce.slowstart.completed.maps | mapreduce.job.reduce.slowstart.completedmaps
-*---+---+
-|mapred.reduce.task.debug.script | mapreduce.reduce.debug.script
-*---+---+
-|mapred.reduce.tasks | mapreduce.job.reduces
-*---+---+
-|mapred.reduce.tasks.speculative.execution | mapreduce.reduce.speculative
-*---+---+
-|mapred.seqbinary.output.key.class | mapreduce.output.seqbinaryoutputformat.key.class
-*---+---+
-|mapred.seqbinary.output.value.class | mapreduce.output.seqbinaryoutputformat.value.class
-*---+---+
-|mapred.shuffle.connect.timeout | mapreduce.reduce.shuffle.connect.timeout
-*---+---+
-|mapred.shuffle.read.timeout | mapreduce.reduce.shuffle.read.timeout
-*---+---+
-|mapred.skip.attempts.to.start.skipping | mapreduce.task.skip.start.attempts
-*---+---+
-|mapred.skip.map.auto.incr.proc.count | mapreduce.map.skip.proc-count.auto-incr
-*---+---+
-|mapred.skip.map.max.skip.records | mapreduce.map.skip.maxrecords
-*---+---+
-|mapred.skip.on | mapreduce.job.skiprecords
-*---+---+
-|mapred.skip.out.dir | mapreduce.job.skip.outdir
-*---+---+
-|mapred.skip.reduce.auto.incr.proc.count | mapreduce.reduce.skip.proc-count.auto-incr
-*---+---+
-|mapred.skip.reduce.max.skip.groups | mapreduce.reduce.skip.maxgroups
-*---+---+
-|mapred.speculative.execution.slowNodeThreshold | mapreduce.job.speculative.slownodethreshold
-*---+---+
-|mapred.speculative.execution.slowTaskThreshold | mapreduce.job.speculative.slowtaskthreshold
-*---+---+
-|mapred.speculative.execution.speculativeCap | mapreduce.job.speculative.speculativecap
-*---+---+
-|mapred.submit.replication | mapreduce.client.submit.file.replication
-*---+---+
-|mapred.system.dir | mapreduce.jobtracker.system.dir
-*---+---+
-|mapred.task.cache.levels | mapreduce.jobtracker.taskcache.levels
-*---+---+
-|mapred.task.id | mapreduce.task.attempt.id
-*---+---+
-|mapred.task.is.map | mapreduce.task.ismap
-*---+---+
-|mapred.task.partition | mapreduce.task.partition
-*---+---+
-|mapred.task.profile | mapreduce.task.profile
-*---+---+
-|mapred.task.profile.maps | mapreduce.task.profile.maps
-*---+---+
-|mapred.task.profile.params | mapreduce.task.profile.params
-*---+---+
-|mapred.task.profile.reduces | mapreduce.task.profile.reduces
-*---+---+
-|mapred.task.timeout | mapreduce.task.timeout
-*---+---+
-|mapred.tasktracker.dns.interface | mapreduce.tasktracker.dns.interface
-*---+---+
-|mapred.tasktracker.dns.nameserver | mapreduce.tasktracker.dns.nameserver
-*---+---+
-|mapred.tasktracker.events.batchsize | mapreduce.tasktracker.events.batchsize
-*---+---+
-|mapred.tasktracker.expiry.interval | mapreduce.jobtracker.expire.trackers.interval
-*---+---+
-|mapred.task.tracker.http.address | mapreduce.tasktracker.http.address
-*---+---+
-|mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb
-*---+---+
-|mapred.tasktracker.instrumentation | mapreduce.tasktracker.instrumentation
-*---+---+
-|mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum
-*---+---+
-|mapred.tasktracker.memory_calculator_plugin | mapreduce.tasktracker.resourcecalculatorplugin
-*---+---+
-|mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin
-*---+---+
-|mapred.tasktracker.reduce.tasks.maximum | mapreduce.tasktracker.reduce.tasks.maximum
-*---+---+
-|mapred.task.tracker.report.address | mapreduce.tasktracker.report.address
-*---+---+
-|mapred.task.tracker.task-controller | mapreduce.tasktracker.taskcontroller
-*---+---+
-|mapred.tasktracker.taskmemorymanager.monitoring-interval | mapreduce.tasktracker.taskmemorymanager.monitoringinterval
-*---+---+
-|mapred.tasktracker.tasks.sleeptime-before-sigkill | mapreduce.tasktracker.tasks.sleeptimebeforesigkill
-*---+---+
-|mapred.temp.dir | mapreduce.cluster.temp.dir
-*---+---+
-|mapred.text.key.comparator.options | mapreduce.partition.keycomparator.options
-*---+---+
-|mapred.text.key.partitioner.options | mapreduce.partition.keypartitioner.options
-*---+---+
-|mapred.textoutputformat.separator | mapreduce.output.textoutputformat.separator
-*---+---+
-|mapred.tip.id | mapreduce.task.id
-*---+---+
-|mapreduce.combine.class | mapreduce.job.combine.class
-*---+---+
-|mapreduce.inputformat.class | mapreduce.job.inputformat.class
-*---+---+
-|mapreduce.job.counters.limit | mapreduce.job.counters.max
-*---+---+
-|mapreduce.jobtracker.permissions.supergroup | mapreduce.cluster.permissions.supergroup
-*---+---+
-|mapreduce.map.class | mapreduce.job.map.class
-*---+---+
-|mapreduce.outputformat.class | mapreduce.job.outputformat.class
-*---+---+
-|mapreduce.partitioner.class | mapreduce.job.partitioner.class
-*---+---+
-|mapreduce.reduce.class | mapreduce.job.reduce.class
-*---+---+
-|mapred.used.genericoptionsparser | mapreduce.client.genericoptionsparser.used
-*---+---+
-|mapred.userlog.limit.kb | mapreduce.task.userlog.limit.kb
-*---+---+
-|mapred.userlog.retain.hours | mapreduce.job.userlog.retain.hours
-*---+---+
-|mapred.working.dir | mapreduce.job.working.dir
-*---+---+
-|mapred.work.output.dir | mapreduce.task.output.dir
-*---+---+
-|min.num.spills.for.combine | mapreduce.map.combine.minspills
-*---+---+
-|reduce.output.key.value.fields.spec | mapreduce.fieldsel.reduce.output.key.value.fields.spec
-*---+---+
-|security.job.submission.protocol.acl | security.job.client.protocol.acl
-*---+---+
-|security.task.umbilical.protocol.acl | security.job.task.protocol.acl
-*---+---+
-|sequencefile.filter.class | mapreduce.input.sequencefileinputfilter.class
-*---+---+
-|sequencefile.filter.frequency | mapreduce.input.sequencefileinputfilter.frequency
-*---+---+
-|sequencefile.filter.regex | mapreduce.input.sequencefileinputfilter.regex
-*---+---+
-|session.id | dfs.metrics.session-id
-*---+---+
-|slave.host.name | dfs.datanode.hostname
-*---+---+
-|slave.host.name | mapreduce.tasktracker.host.name
-*---+---+
-|tasktracker.contention.tracking | mapreduce.tasktracker.contention.tracking
-*---+---+
-|tasktracker.http.threads | mapreduce.tasktracker.http.threads
-*---+---+
-|topology.node.switch.mapping.impl | net.topology.node.switch.mapping.impl
-*---+---+
-|topology.script.file.name | net.topology.script.file.name
-*---+---+
-|topology.script.number.args | net.topology.script.number.args
-*---+---+
-|user.name | mapreduce.job.user.name
-*---+---+
-|webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted
-*---+---+
-|yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts
-*---+---+
-
- The following table lists additional changes to some configuration properties:
-
-*-------------------------------+-----------------------+
-|| <<Deprecated property name>> || <<New property name>>|
-*-------------------------------+-----------------------+
-|mapred.create.symlink | NONE - symlinking is always on
-*---+---+
-|mapreduce.job.cache.symlink.create | NONE - symlinking is always on
-*---+---+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
deleted file mode 100644
index 6831ebf..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/FileSystemShell.apt.vm
+++ /dev/null
@@ -1,764 +0,0 @@
-~~ Licensed to the Apache Software Foundation (ASF) under one or more
-~~ contributor license agreements. See the NOTICE file distributed with
-~~ this work for additional information regarding copyright ownership.
-~~ The ASF licenses this file to You under the Apache License, Version 2.0
-~~ (the "License"); you may not use this file except in compliance with
-~~ the License. You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- File System Shell Guide
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc}
-
-Overview
-
- The File System (FS) shell includes various shell-like commands that
- directly interact with the Hadoop Distributed File System (HDFS) as well as
- other file systems that Hadoop supports, such as Local FS, HFTP FS, S3 FS,
- and others. The FS shell is invoked by:
-
-+---
-bin/hadoop fs <args>
-+---
-
- All FS shell commands take path URIs as arguments. The URI format is
- <<<scheme://authority/path>>>. For HDFS the scheme is <<<hdfs>>>, and for
- the Local FS the scheme is <<<file>>>. The scheme and authority are
- optional. If not specified, the default scheme specified in the
- configuration is used. An HDFS file or directory such as /parent/child can
- be specified as <<<hdfs://namenodehost/parent/child>>> or simply as
- <<</parent/child>>> (given that your configuration is set to point to
- <<<hdfs://namenodehost>>>).
-
- Most of the commands in FS shell behave like corresponding Unix commands.
- Differences are described with each of the commands. Error information is
- sent to stderr and the output is sent to stdout.
-
- If HDFS is being used, <<<hdfs dfs>>> is a synonym.
-
- See the {{{./CommandsManual.html}Commands Manual}} for generic shell options.
-
-* appendToFile
-
- Usage: <<<hadoop fs -appendToFile <localsrc> ... <dst> >>>
-
- Append single src, or multiple srcs from local file system to the
- destination file system. Also reads input from stdin and appends to
- destination file system.
-
- * <<<hadoop fs -appendToFile localfile /user/hadoop/hadoopfile>>>
-
- * <<<hadoop fs -appendToFile localfile1 localfile2 /user/hadoop/hadoopfile>>>
-
- * <<<hadoop fs -appendToFile localfile hdfs://nn.example.com/hadoop/hadoopfile>>>
-
- * <<<hadoop fs -appendToFile - hdfs://nn.example.com/hadoop/hadoopfile>>>
- Reads the input from stdin.
-
- Exit Code:
-
- Returns 0 on success and 1 on error.
-
-* cat
-
- Usage: <<<hadoop fs -cat URI [URI ...]>>>
-
- Copies source paths to stdout.
-
- Example:
-
- * <<<hadoop fs -cat hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
-
- * <<<hadoop fs -cat file:///file3 /user/hadoop/file4>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* checksum
-
- Usage: <<<hadoop fs -checksum URI>>>
-
- Returns the checksum information of a file.
-
- Example:
-
- * <<<hadoop fs -checksum hdfs://nn1.example.com/file1>>>
-
- * <<<hadoop fs -checksum file:///etc/hosts>>>
-
-* chgrp
-
- Usage: <<<hadoop fs -chgrp [-R] GROUP URI [URI ...]>>>
-
- Change group association of files. The user must be the owner of files, or
- else a super-user. Additional information is in the
- {{{../hadoop-hdfs/HdfsPermissionsGuide.html}Permissions Guide}}.
-
- Options
-
- * The -R option will make the change recursively through the directory structure.
-
-* chmod
-
- Usage: <<<hadoop fs -chmod [-R] <MODE[,MODE]... | OCTALMODE> URI [URI ...]>>>
-
- Change the permissions of files. With -R, make the change recursively
- through the directory structure. The user must be the owner of the file, or
- else a super-user. Additional information is in the
- {{{../hadoop-hdfs/HdfsPermissionsGuide.html}Permissions Guide}}.
-
- Options
-
- * The -R option will make the change recursively through the directory structure.
-
-* chown
-
- Usage: <<<hadoop fs -chown [-R] [OWNER][:[GROUP]] URI [URI ]>>>
-
- Change the owner of files. The user must be a super-user. Additional information
- is in the {{{../hadoop-hdfs/HdfsPermissionsGuide.html}Permissions Guide}}.
-
- Options
-
- * The -R option will make the change recursively through the directory structure.
-
-* copyFromLocal
-
- Usage: <<<hadoop fs -copyFromLocal <localsrc> URI>>>
-
- Similar to put command, except that the source is restricted to a local
- file reference.
-
- Options:
-
- * The -f option will overwrite the destination if it already exists.
-
-* copyToLocal
-
- Usage: <<<hadoop fs -copyToLocal [-ignorecrc] [-crc] URI <localdst> >>>
-
- Similar to get command, except that the destination is restricted to a
- local file reference.
-
-* count
-
- Usage: <<<hadoop fs -count [-q] [-h] [-v] <paths> >>>
-
- Count the number of directories, files and bytes under the paths that match
- the specified file pattern. The output columns with -count are: DIR_COUNT,
- FILE_COUNT, CONTENT_SIZE PATHNAME
-
- The output columns with -count -q are: QUOTA, REMAINING_QUATA, SPACE_QUOTA,
- REMAINING_SPACE_QUOTA, DIR_COUNT, FILE_COUNT, CONTENT_SIZE, PATHNAME
-
- The -h option shows sizes in human readable format.
-
- The -v option displays a header line.
-
- Example:
-
- * <<<hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2>>>
-
- * <<<hadoop fs -count -q hdfs://nn1.example.com/file1>>>
-
- * <<<hadoop fs -count -q -h hdfs://nn1.example.com/file1>>>
-
- * <<<hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* cp
-
- Usage: <<<hadoop fs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> >>>
-
- Copy files from source to destination. This command allows multiple sources
- as well in which case the destination must be a directory.
-
- 'raw.*' namespace extended attributes are preserved if (1) the source and
- destination filesystems support them (HDFS only), and (2) all source and
- destination pathnames are in the /.reserved/raw hierarchy. Determination of
- whether raw.* namespace xattrs are preserved is independent of the
- -p (preserve) flag.
-
- Options:
-
- * The -f option will overwrite the destination if it already exists.
-
- * The -p option will preserve file attributes [topx] (timestamps,
- ownership, permission, ACL, XAttr). If -p is specified with no <arg>,
- then preserves timestamps, ownership, permission. If -pa is specified,
- then preserves permission also because ACL is a super-set of
- permission. Determination of whether raw namespace extended attributes
- are preserved is independent of the -p flag.
-
- Example:
-
- * <<<hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2>>>
-
- * <<<hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* createSnapshot
-
- See {{{../hadoop-hdfs/HdfsSnapshots.html}HDFS Snapshots Guide}}.
-
-
-* deleteSnapshot
-
- See {{{../hadoop-hdfs/HdfsSnapshots.html}HDFS Snapshots Guide}}.
-
-* df
-
- Usage: <<<hadoop fs -df [-h] URI [URI ...]>>>
-
- Displays free space.
-
- Options:
-
- * The -h option will format file sizes in a "human-readable" fashion (e.g
- 64.0m instead of 67108864)
-
- Example:
-
- * <<<hadoop dfs -df /user/hadoop/dir1>>>
-
-* du
-
- Usage: <<<hadoop fs -du [-s] [-h] URI [URI ...]>>>
-
- Displays sizes of files and directories contained in the given directory or
- the length of a file in case its just a file.
-
- Options:
-
- * The -s option will result in an aggregate summary of file lengths being
- displayed, rather than the individual files.
-
- * The -h option will format file sizes in a "human-readable" fashion (e.g
- 64.0m instead of 67108864)
-
- Example:
-
- * <<<hadoop fs -du /user/hadoop/dir1 /user/hadoop/file1 hdfs://nn.example.com/user/hadoop/dir1>>>
-
- Exit Code:
- Returns 0 on success and -1 on error.
-
-* dus
-
- Usage: <<<hadoop fs -dus <args> >>>
-
- Displays a summary of file lengths.
-
- <<Note:>> This command is deprecated. Instead use <<<hadoop fs -du -s>>>.
-
-* expunge
-
- Usage: <<<hadoop fs -expunge>>>
-
- Empty the Trash. Refer to the {{{../hadoop-hdfs/HdfsDesign.html}
- HDFS Architecture Guide}} for more information on the Trash feature.
-
-* find
-
- Usage: <<<hadoop fs -find <path> ... <expression> ... >>>
-
- Finds all files that match the specified expression and applies selected
- actions to them. If no <path> is specified then defaults to the current
- working directory. If no expression is specified then defaults to -print.
-
- The following primary expressions are recognised:
-
- * -name pattern \
- -iname pattern
-
- Evaluates as true if the basename of the file matches the pattern using
- standard file system globbing. If -iname is used then the match is case
- insensitive.
-
- * -print \
- -print0
-
- Always evaluates to true. Causes the current pathname to be written to
- standard output. If the -print0 expression is used then an ASCII NULL
- character is appended.
-
- The following operators are recognised:
-
- * expression -a expression \
- expression -and expression \
- expression expression
-
- Logical AND operator for joining two expressions. Returns true if both
- child expressions return true. Implied by the juxtaposition of two
- expressions and so does not need to be explicitly specified. The second
- expression will not be applied if the first fails.
-
- Example:
-
- <<<hadoop fs -find / -name test -print>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* get
-
- Usage: <<<hadoop fs -get [-ignorecrc] [-crc] <src> <localdst> >>>
-
- Copy files to the local file system. Files that fail the CRC check may be
- copied with the -ignorecrc option. Files and CRCs may be copied using the
- -crc option.
-
- Example:
-
- * <<<hadoop fs -get /user/hadoop/file localfile>>>
-
- * <<<hadoop fs -get hdfs://nn.example.com/user/hadoop/file localfile>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* getfacl
-
- Usage: <<<hadoop fs -getfacl [-R] <path> >>>
-
- Displays the Access Control Lists (ACLs) of files and directories. If a
- directory has a default ACL, then getfacl also displays the default ACL.
-
- Options:
-
- * -R: List the ACLs of all files and directories recursively.
-
- * <path>: File or directory to list.
-
- Examples:
-
- * <<<hadoop fs -getfacl /file>>>
-
- * <<<hadoop fs -getfacl -R /dir>>>
-
- Exit Code:
-
- Returns 0 on success and non-zero on error.
-
-* getfattr
-
- Usage: <<<hadoop fs -getfattr [-R] {-n name | -d} [-e en] <path> >>>
-
- Displays the extended attribute names and values (if any) for a file or
- directory.
-
- Options:
-
- * -R: Recursively list the attributes for all files and directories.
-
- * -n name: Dump the named extended attribute value.
-
- * -d: Dump all extended attribute values associated with pathname.
-
- * -e <encoding>: Encode values after retrieving them. Valid encodings are "text", "hex", and "base64". Values encoded as text strings are enclosed in double quotes ("), and values encoded as hexadecimal and base64 are prefixed with 0x and 0s, respectively.
-
- * <path>: The file or directory.
-
- Examples:
-
- * <<<hadoop fs -getfattr -d /file>>>
-
- * <<<hadoop fs -getfattr -R -n user.myAttr /dir>>>
-
- Exit Code:
-
- Returns 0 on success and non-zero on error.
-
-* getmerge
-
- Usage: <<<hadoop fs -getmerge <src> <localdst> [addnl]>>>
-
- Takes a source directory and a destination file as input and concatenates
- files in src into the destination local file. Optionally addnl can be set to
- enable adding a newline character at the
- end of each file.
-
-* help
-
- Usage: <<<hadoop fs -help>>>
-
- Return usage output.
-
-* ls
-
- Usage: <<<hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> >>>
-
- Options:
-
- * -d: Directories are listed as plain files.
-
- * -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
-
- * -R: Recursively list subdirectories encountered.
-
- * -t: Sort output by modification time (most recent first).
-
- * -S: Sort output by file size.
-
- * -r: Reverse the sort order.
-
- * -u: Use access time rather than modification time for display and sorting.
-
- For a file ls returns stat on the file with the following format:
-
-+---+
-permissions number_of_replicas userid groupid filesize modification_date modification_time filename
-+---+
-
- For a directory it returns list of its direct children as in Unix. A directory is listed as:
-
-+---+
-permissions userid groupid modification_date modification_time dirname
-+---+
-
- Files within a directory are order by filename by default.
-
-
- Example:
-
- * <<<hadoop fs -ls /user/hadoop/file1>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* lsr
-
- Usage: <<<hadoop fs -lsr <args> >>>
-
- Recursive version of ls.
-
- <<Note:>> This command is deprecated. Instead use <<<hadoop fs -ls -R>>>
-
-* mkdir
-
- Usage: <<<hadoop fs -mkdir [-p] <paths> >>>
-
- Takes path uri's as argument and creates directories.
-
- Options:
-
- * The -p option behavior is much like Unix mkdir -p, creating parent directories along the path.
-
- Example:
-
- * <<<hadoop fs -mkdir /user/hadoop/dir1 /user/hadoop/dir2>>>
-
- * <<<hadoop fs -mkdir hdfs://nn1.example.com/user/hadoop/dir hdfs://nn2.example.com/user/hadoop/dir>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* moveFromLocal
-
- Usage: <<<hadoop fs -moveFromLocal <localsrc> <dst> >>>
-
- Similar to put command, except that the source localsrc is deleted after
- it's copied.
-
-* moveToLocal
-
- Usage: <<<hadoop fs -moveToLocal [-crc] <src> <dst> >>>
-
- Displays a "Not implemented yet" message.
-
-* mv
-
- Usage: <<<hadoop fs -mv URI [URI ...] <dest> >>>
-
- Moves files from source to destination. This command allows multiple sources
- as well in which case the destination needs to be a directory. Moving files
- across file systems is not permitted.
-
- Example:
-
- * <<<hadoop fs -mv /user/hadoop/file1 /user/hadoop/file2>>>
-
- * <<<hadoop fs -mv hdfs://nn.example.com/file1 hdfs://nn.example.com/file2 hdfs://nn.example.com/file3 hdfs://nn.example.com/dir1>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* put
-
- Usage: <<<hadoop fs -put <localsrc> ... <dst> >>>
-
- Copy single src, or multiple srcs from local file system to the destination
- file system. Also reads input from stdin and writes to destination file
- system.
-
- * <<<hadoop fs -put localfile /user/hadoop/hadoopfile>>>
-
- * <<<hadoop fs -put localfile1 localfile2 /user/hadoop/hadoopdir>>>
-
- * <<<hadoop fs -put localfile hdfs://nn.example.com/hadoop/hadoopfile>>>
-
- * <<<hadoop fs -put - hdfs://nn.example.com/hadoop/hadoopfile>>>
- Reads the input from stdin.
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* renameSnapshot
-
- See {{{../hadoop-hdfs/HdfsSnapshots.html}HDFS Snapshots Guide}}.
-
-* rm
-
- Usage: <<<hadoop fs -rm [-f] [-r|-R] [-skipTrash] URI [URI ...]>>>
-
- Delete files specified as args.
-
- Options:
-
- * The -f option will not display a diagnostic message or modify the exit
- status to reflect an error if the file does not exist.
-
- * The -R option deletes the directory and any content under it recursively.
-
- * The -r option is equivalent to -R.
-
- * The -skipTrash option will bypass trash, if enabled, and delete the
- specified file(s) immediately. This can be useful when it is necessary
- to delete files from an over-quota directory.
-
- Example:
-
- * <<<hadoop fs -rm hdfs://nn.example.com/file /user/hadoop/emptydir>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* rmdir
-
- Usage: <<<hadoop fs -rmdir [--ignore-fail-on-non-empty] URI [URI ...]>>>
-
- Delete a directory.
-
- Options:
-
- * --ignore-fail-on-non-empty: When using wildcards, do not fail if a directory still contains files.
-
- Example:
-
- * <<<hadoop fs -rmdir /user/hadoop/emptydir>>>
-
-* rmr
-
- Usage: <<<hadoop fs -rmr [-skipTrash] URI [URI ...]>>>
-
- Recursive version of delete.
-
- <<Note:>> This command is deprecated. Instead use <<<hadoop fs -rm -r>>>
-
-* setfacl
-
- Usage: <<<hadoop fs -setfacl [-R] [{-b|-k} {-m|-x <acl_spec>} <path>]|[--set <acl_spec> <path>] >>>
-
- Sets Access Control Lists (ACLs) of files and directories.
-
- Options:
-
- * -b: Remove all but the base ACL entries. The entries for user, group and
- others are retained for compatibility with permission bits.
-
- * -k: Remove the default ACL.
-
- * -R: Apply operations to all files and directories recursively.
-
- * -m: Modify ACL. New entries are added to the ACL, and existing entries
- are retained.
-
- * -x: Remove specified ACL entries. Other ACL entries are retained.
-
- * --set: Fully replace the ACL, discarding all existing entries. The
- <acl_spec> must include entries for user, group, and others for
- compatibility with permission bits.
-
- * <acl_spec>: Comma separated list of ACL entries.
-
- * <path>: File or directory to modify.
-
- Examples:
-
- * <<<hadoop fs -setfacl -m user:hadoop:rw- /file>>>
-
- * <<<hadoop fs -setfacl -x user:hadoop /file>>>
-
- * <<<hadoop fs -setfacl -b /file>>>
-
- * <<<hadoop fs -setfacl -k /dir>>>
-
- * <<<hadoop fs -setfacl --set user::rw-,user:hadoop:rw-,group::r--,other::r-- /file>>>
-
- * <<<hadoop fs -setfacl -R -m user:hadoop:r-x /dir>>>
-
- * <<<hadoop fs -setfacl -m default:user:hadoop:r-x /dir>>>
-
- Exit Code:
-
- Returns 0 on success and non-zero on error.
-
-* setfattr
-
- Usage: <<<hadoop fs -setfattr {-n name [-v value] | -x name} <path> >>>
-
- Sets an extended attribute name and value for a file or directory.
-
- Options:
-
- * -b: Remove all but the base ACL entries. The entries for user, group and others are retained for compatibility with permission bits.
-
- * -n name: The extended attribute name.
-
- * -v value: The extended attribute value. There are three different encoding methods for the value. If the argument is enclosed in double quotes, then the value is the string inside the quotes. If the argument is prefixed with 0x or 0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 0S, then it is taken as a base64 encoding.
-
- * -x name: Remove the extended attribute.
-
- * <path>: The file or directory.
-
- Examples:
-
- * <<<hadoop fs -setfattr -n user.myAttr -v myValue /file>>>
-
- * <<<hadoop fs -setfattr -n user.noValue /file>>>
-
- * <<<hadoop fs -setfattr -x user.myAttr /file>>>
-
- Exit Code:
-
- Returns 0 on success and non-zero on error.
-
-* setrep
-
- Usage: <<<hadoop fs -setrep [-R] [-w] <numReplicas> <path> >>>
-
- Changes the replication factor of a file. If <path> is a directory then
- the command recursively changes the replication factor of all files under
- the directory tree rooted at <path>.
-
- Options:
-
- * The -w flag requests that the command wait for the replication
- to complete. This can potentially take a very long time.
-
- * The -R flag is accepted for backwards compatibility. It has no effect.
-
- Example:
-
- * <<<hadoop fs -setrep -w 3 /user/hadoop/dir1>>>
-
- Exit Code:
-
- Returns 0 on success and -1 on error.
-
-* stat
-
- Usage: <<<hadoop fs -stat [format] \<path\> ...>>>
-
- Print statistics about the file/directory at \<path\> in the specified
- format. Format accepts filesize in blocks (%b), type (%F), group name of
- owner (%g), name (%n), block size (%o), replication (%r), user name of
- owner(%u), and modification date (%y, %Y). %y shows UTC date as
- "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC.
- If the format is not specified, %y is used by default.
-
- Example:
-
- * <<<hadoop fs -stat "%F %u:%g %b %y %n" /file>>>
-
- Exit Code:
- Returns 0 on success and -1 on error.
-
-* tail
-
- Usage: <<<hadoop fs -tail [-f] URI>>>
-
- Displays last kilobyte of the file to stdout.
-
- Options:
-
- * The -f option will output appended data as the file grows, as in Unix.
-
- Example:
-
- * <<<hadoop fs -tail pathname>>>
-
- Exit Code:
- Returns 0 on success and -1 on error.
-
-* test
-
- Usage: <<<hadoop fs -test -[defsz] URI>>>
-
- Options:
-
- * -d: f the path is a directory, return 0.
-
- * -e: if the path exists, return 0.
-
- * -f: if the path is a file, return 0.
-
- * -s: if the path is not empty, return 0.
-
- * -z: if the file is zero length, return 0.
-
- Example:
-
- * <<<hadoop fs -test -e filename>>>
-
-* text
-
- Usage: <<<hadoop fs -text <src> >>>
-
- Takes a source file and outputs the file in text format. The allowed formats
- are zip and TextRecordInputStream.
-
-* touchz
-
- Usage: <<<hadoop fs -touchz URI [URI ...]>>>
-
- Create a file of zero length.
-
- Example:
-
- * <<<hadoop fs -touchz pathname>>>
-
- Exit Code:
- Returns 0 on success and -1 on error.
-
-
-* usage
-
- Usage: <<<hadoop fs -usage command>>>
-
- Return the help for an individual command.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/HttpAuthentication.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/HttpAuthentication.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/HttpAuthentication.apt.vm
deleted file mode 100644
index 1f95da0..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/HttpAuthentication.apt.vm
+++ /dev/null
@@ -1,98 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Authentication for Hadoop HTTP web-consoles
- ---
- ---
- ${maven.build.timestamp}
-
-Authentication for Hadoop HTTP web-consoles
-
-%{toc|section=1|fromDepth=0}
-
-* Introduction
-
- This document describes how to configure Hadoop HTTP web-consoles to
- require user authentication.
-
- By default Hadoop HTTP web-consoles (JobTracker, NameNode, TaskTrackers
- and DataNodes) allow access without any form of authentication.
-
- Similarly to Hadoop RPC, Hadoop HTTP web-consoles can be configured to
- require Kerberos authentication using HTTP SPNEGO protocol (supported
- by browsers like Firefox and Internet Explorer).
-
- In addition, Hadoop HTTP web-consoles support the equivalent of
- Hadoop's Pseudo/Simple authentication. If this option is enabled, user
- must specify their user name in the first browser interaction using the
- user.name query string parameter. For example:
- <<<http://localhost:50030/jobtracker.jsp?user.name=babu>>>.
-
- If a custom authentication mechanism is required for the HTTP
- web-consoles, it is possible to implement a plugin to support the
- alternate authentication mechanism (refer to Hadoop hadoop-auth for details
- on writing an <<<AuthenticatorHandler>>>).
-
- The next section describes how to configure Hadoop HTTP web-consoles to
- require user authentication.
-
-* Configuration
-
- The following properties should be in the <<<core-site.xml>>> of all the
- nodes in the cluster.
-
- <<<hadoop.http.filter.initializers>>>: add to this property the
- <<<org.apache.hadoop.security.AuthenticationFilterInitializer>>> initializer
- class.
-
- <<<hadoop.http.authentication.type>>>: Defines authentication used for the
- HTTP web-consoles. The supported values are: <<<simple>>> | <<<kerberos>>> |
- <<<#AUTHENTICATION_HANDLER_CLASSNAME#>>>. The dfeault value is <<<simple>>>.
-
- <<<hadoop.http.authentication.token.validity>>>: Indicates how long (in
- seconds) an authentication token is valid before it has to be renewed.
- The default value is <<<36000>>>.
-
- <<<hadoop.http.authentication.signature.secret.file>>>: The signature secret
- file for signing the authentication tokens. The same secret should be used
- for all nodes in the cluster, JobTracker, NameNode, DataNode and TastTracker.
- The default value is <<<${user.home}/hadoop-http-auth-signature-secret>>>.
- IMPORTANT: This file should be readable only by the Unix user running the
- daemons.
-
- <<<hadoop.http.authentication.cookie.domain>>>: The domain to use for the
- HTTP cookie that stores the authentication token. In order to
- authentiation to work correctly across all nodes in the cluster the
- domain must be correctly set. There is no default value, the HTTP
- cookie will not have a domain working only with the hostname issuing
- the HTTP cookie.
-
- IMPORTANT: when using IP addresses, browsers ignore cookies with domain
- settings. For this setting to work properly all nodes in the cluster
- must be configured to generate URLs with <<<hostname.domain>>> names on it.
-
- <<<hadoop.http.authentication.simple.anonymous.allowed>>>: Indicates if
- anonymous requests are allowed when using 'simple' authentication. The
- default value is <<<true>>>
-
- <<<hadoop.http.authentication.kerberos.principal>>>: Indicates the Kerberos
- principal to be used for HTTP endpoint when using 'kerberos'
- authentication. The principal short name must be <<<HTTP>>> per Kerberos HTTP
- SPNEGO specification. The default value is <<<HTTP/_HOST@$LOCALHOST>>>,
- where <<<_HOST>>> -if present- is replaced with bind address of the HTTP
- server.
-
- <<<hadoop.http.authentication.kerberos.keytab>>>: Location of the keytab file
- with the credentials for the Kerberos principal used for the HTTP
- endpoint. The default value is <<<${user.home}/hadoop.keytab>>>.i
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/InterfaceClassification.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/InterfaceClassification.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/InterfaceClassification.apt.vm
deleted file mode 100644
index 85e66bd..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/InterfaceClassification.apt.vm
+++ /dev/null
@@ -1,239 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop Interface Taxonomy: Audience and Stability Classification
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop Interface Taxonomy: Audience and Stability Classification
-
-%{toc|section=1|fromDepth=0}
-
-* Motivation
-
- The interface taxonomy classification provided here is for guidance to
- developers and users of interfaces. The classification guides a developer
- to declare the targeted audience or users of an interface and also its
- stability.
-
- * Benefits to the user of an interface: Knows which interfaces to use or not
- use and their stability.
-
- * Benefits to the developer: to prevent accidental changes of interfaces and
- hence accidental impact on users or other components or system. This is
- particularly useful in large systems with many developers who may not all
- have a shared state/history of the project.
-
-* Interface Classification
-
- Hadoop adopts the following interface classification,
- this classification was derived from the
- {{{http://www.opensolaris.org/os/community/arc/policies/interface-taxonomy/#Advice}OpenSolaris taxonomy}}
- and, to some extent, from taxonomy used inside Yahoo. Interfaces have two main
- attributes: Audience and Stability
-
-** Audience
-
- Audience denotes the potential consumers of the interface. While many
- interfaces are internal/private to the implementation,
- other are public/external interfaces are meant for wider consumption by
- applications and/or clients. For example, in posix, libc is an external or
- public interface, while large parts of the kernel are internal or private
- interfaces. Also, some interfaces are targeted towards other specific
- subsystems.
-
- Identifying the audience of an interface helps define the impact of
- breaking it. For instance, it might be okay to break the compatibility of
- an interface whose audience is a small number of specific subsystems. On
- the other hand, it is probably not okay to break a protocol interfaces
- that millions of Internet users depend on.
-
- Hadoop uses the following kinds of audience in order of
- increasing/wider visibility:
-
- * Private:
-
- * The interface is for internal use within the project (such as HDFS or
- MapReduce) and should not be used by applications or by other projects. It
- is subject to change at anytime without notice. Most interfaces of a
- project are Private (also referred to as project-private).
-
- * Limited-Private:
-
- * The interface is used by a specified set of projects or systems
- (typically closely related projects). Other projects or systems should not
- use the interface. Changes to the interface will be communicated/
- negotiated with the specified projects. For example, in the Hadoop project,
- some interfaces are LimitedPrivate\{HDFS, MapReduce\} in that they
- are private to the HDFS and MapReduce projects.
-
- * Public
-
- * The interface is for general use by any application.
-
- Hadoop doesn't have a Company-Private classification,
- which is meant for APIs which are intended to be used by other projects
- within the company, since it doesn't apply to opensource projects. Also,
- certain APIs are annotated as @VisibleForTesting (from com.google.common
- .annotations.VisibleForTesting) - these are meant to be used strictly for
- unit tests and should be treated as "Private" APIs.
-
-** Stability
-
- Stability denotes how stable an interface is, as in when incompatible
- changes to the interface are allowed. Hadoop APIs have the following
- levels of stability.
-
- * Stable
-
- * Can evolve while retaining compatibility for minor release boundaries;
- in other words, incompatible changes to APIs marked Stable are allowed
- only at major releases (i.e. at m.0).
-
- * Evolving
-
- * Evolving, but incompatible changes are allowed at minor release (i.e. m
- .x)
-
- * Unstable
-
- * Incompatible changes to Unstable APIs are allowed any time. This
- usually makes sense for only private interfaces.
-
- * However one may call this out for a supposedly public interface to
- highlight that it should not be used as an interface; for public
- interfaces, labeling it as Not-an-interface is probably more appropriate
- than "Unstable".
-
- * Examples of publicly visible interfaces that are unstable (i.e.
- not-an-interface): GUI, CLIs whose output format will change
-
- * Deprecated
-
- * APIs that could potentially removed in the future and should not be
- used.
-
-* How are the Classifications Recorded?
-
- How will the classification be recorded for Hadoop APIs?
-
- * Each interface or class will have the audience and stability recorded
- using annotations in org.apache.hadoop.classification package.
-
- * The javadoc generated by the maven target javadoc:javadoc lists only the
- public API.
-
- * One can derive the audience of java classes and java interfaces by the
- audience of the package in which they are contained. Hence it is useful to
- declare the audience of each java package as public or private (along with
- the private audience variations).
-
-* FAQ
-
- * Why aren’t the java scopes (private, package private and public) good
- enough?
-
- * Java’s scoping is not very complete. One is often forced to make a class
- public in order for other internal components to use it. It does not have
- friends or sub-package-private like C++.
-
- * But I can easily access a private implementation interface if it is Java
- public. Where is the protection and control?
-
- * The purpose of this is not providing absolute access control. Its purpose
- is to communicate to users and developers. One can access private
- implementation functions in libc; however if they change the internal
- implementation details, your application will break and you will have little
- sympathy from the folks who are supplying libc. If you use a non-public
- interface you understand the risks.
-
- * Why bother declaring the stability of a private interface? Aren’t private
- interfaces always unstable?
-
- * Private interfaces are not always unstable. In the cases where they are
- stable they capture internal properties of the system and can communicate
- these properties to its internal users and to developers of the interface.
-
- * e.g. In HDFS, NN-DN protocol is private but stable and can help
- implement rolling upgrades. It communicates that this interface should not
- be changed in incompatible ways even though it is private.
-
- * e.g. In HDFS, FSImage stability can help provide more flexible roll
- backs.
-
- * What is the harm in applications using a private interface that is
- stable? How is it different than a public stable interface?
-
- * While a private interface marked as stable is targeted to change only at
- major releases, it may break at other times if the providers of that
- interface are willing to changes the internal users of that interface.
- Further, a public stable interface is less likely to break even at major
- releases (even though it is allowed to break compatibility) because the
- impact of the change is larger. If you use a private interface (regardless
- of its stability) you run the risk of incompatibility.
-
- * Why bother with Limited-private? Isn’t it giving special treatment to some
- projects? That is not fair.
-
- * First, most interfaces should be public or private; actually let us state
- it even stronger: make it private unless you really want to expose it to
- public for general use.
-
- * Limited-private is for interfaces that are not intended for general use.
- They are exposed to related projects that need special hooks. Such a
- classification has a cost to both the supplier and consumer of the limited
- interface. Both will have to work together if ever there is a need to break
- the interface in the future; for example the supplier and the consumers will
- have to work together to get coordinated releases of their respective
- projects. This should not be taken lightly – if you can get away with
- private then do so; if the interface is really for general use for all
- applications then do so. But remember that making an interface public has
- huge responsibility. Sometimes Limited-private is just right.
-
- * A good example of a limited-private interface is BlockLocations, This is
- fairly low-level interface that we are willing to expose to MR and perhaps
- HBase. We are likely to change it down the road and at that time we will
- have get a coordinated effort with the MR team to release matching releases.
- While MR and HDFS are always released in sync today, they may change down
- the road.
-
- * If you have a limited-private interface with many projects listed then
- you are fooling yourself. It is practically public.
-
- * It might be worth declaring a special audience classification called
- Hadoop-Private for the Hadoop family.
-
- * Lets treat all private interfaces as Hadoop-private. What is the harm in
- projects in the Hadoop family have access to private classes?
-
- * Do we want MR accessing class files that are implementation details
- inside HDFS. There used to be many such layer violations in the code that
- we have been cleaning up over the last few years. We don’t want such
- layer violations to creep back in by no separating between the major
- components like HDFS and MR.
-
- * Aren't all public interfaces stable?
-
- * One may mark a public interface as evolving in its early days.
- Here one is promising to make an effort to make compatible changes but may
- need to break it at minor releases.
-
- * One example of a public interface that is unstable is where one is providing
- an implementation of a standards-body based interface that is still under development.
- For example, many companies, in an attampt to be first to market,
- have provided implementations of a new NFS protocol even when the protocol was not
- fully completed by IETF.
- The implementor cannot evolve the interface in a fashion that causes least distruption
- because the stability is controlled by the standards body. Hence it is appropriate to
- label the interface as unstable.
[37/50] [abbrv] hadoop git commit: HADOOP-11554. Expose
HadoopKerberosName as a hadoop subcommand (aw)
Posted by zj...@apache.org.
HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cfd8a217
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cfd8a217
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cfd8a217
Branch: refs/heads/YARN-2928
Commit: cfd8a2174afb1140b5042d913782b24615f32bfe
Parents: c3da2db
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Feb 11 07:40:39 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Feb 11 07:40:39 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
hadoop-common-project/hadoop-common/src/main/bin/hadoop | 4 ++++
.../hadoop-common/src/main/bin/hadoop.cmd | 6 +++++-
.../hadoop-common/src/site/markdown/CommandsManual.md | 10 ++++++++++
.../hadoop-common/src/site/markdown/SecureMode.md | 2 ++
5 files changed, 23 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd8a217/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 1536e44..e2f1761 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -27,6 +27,8 @@ Trunk (Unreleased)
HADOOP-8934. Shell command ls should include sort options (Jonathan Allen
via aw)
+ HADOOP-11554. Expose HadoopKerberosName as a hadoop subcommand (aw)
+
IMPROVEMENTS
HADOOP-8017. Configure hadoop-main pom to get rid of M2E plugin execution
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd8a217/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index d689bda..42e4e83 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -38,6 +38,7 @@ function hadoop_usage()
echo " note: please use \"yarn jar\" to launch"
echo " YARN applications, not this command."
echo " jnipath prints the java.library.path"
+ echo " kerbname show auth_to_local principal conversion"
echo " key manage keys via the KeyProvider"
echo " trace view and modify Hadoop tracing settings"
echo " version print the version"
@@ -156,6 +157,9 @@ case ${COMMAND} in
echo "${JAVA_LIBRARY_PATH}"
exit 0
;;
+ kerbname)
+ CLASS=org.apache.hadoop.security.HadoopKerberosName
+ ;;
key)
CLASS=org.apache.hadoop.crypto.key.KeyShell
;;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd8a217/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
index f612ace..c600fd1 100644
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop.cmd
@@ -146,7 +146,7 @@ call :updatepath %HADOOP_BIN_PATH%
)
)
- set corecommands=fs version jar checknative distcp daemonlog archive classpath credential key
+ set corecommands=fs version jar checknative distcp daemonlog archive classpath credential kerbname key
for %%i in ( %corecommands% ) do (
if %hadoop-command% == %%i set corecommand=true
)
@@ -215,6 +215,10 @@ call :updatepath %HADOOP_BIN_PATH%
set CLASS=org.apache.hadoop.security.alias.CredentialShell
goto :eof
+:kerbname
+ set CLASS=org.apache.hadoop.security.HadoopKerberosName
+ goto :eof
+
:key
set CLASS=org.apache.hadoop.crypto.key.KeyShell
goto :eof
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd8a217/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
index b9c92c4..f0c0367 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -27,6 +27,7 @@
* [fs](#fs)
* [jar](#jar)
* [jnipath](#jnipath)
+ * [kerbname](#kerbname)
* [key](#key)
* [trace](#trace)
* [version](#version)
@@ -175,6 +176,15 @@ Usage: `hadoop jnipath`
Print the computed java.library.path.
+### `kerbname`
+
+ Usage: <<<hadoop kerbname principal>>>
+
+ Convert the named principal via the auth_to_local rules to the Hadoop
+ user name.
+
+ Example: <<<hadoop kerbname user@EXAMPLE.COM>>>
+
### `key`
Manage keys via the KeyProvider.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cfd8a217/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
index ae057bc..f0e1f1f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/SecureMode.md
@@ -162,6 +162,8 @@ Hadoop maps Kerberos principal to OS user account using the rule specified by `h
By default, it picks the first component of principal name as a user name if the realms matches to the `default_realm` (usually defined in /etc/krb5.conf). For example, `host/full.qualified.domain.name@REALM.TLD` is mapped to `host` by default rule.
+Custom rules can be tested using the <<<hadoop kerbname>>> command. This command allows one to specify a principal and apply Hadoop's current auth_to_local ruleset. The output will be what identity Hadoop will use for its usage.
+
### Mapping from user to group
Though files on HDFS are associated to owner and group, Hadoop does not have the definition of group by itself. Mapping from user to group is done by OS or LDAP.
[25/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
new file mode 100644
index 0000000..80c7d58
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -0,0 +1,339 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Hadoop Cluster Setup](#Hadoop_Cluster_Setup)
+ * [Purpose](#Purpose)
+ * [Prerequisites](#Prerequisites)
+ * [Installation](#Installation)
+ * [Configuring Hadoop in Non-Secure Mode](#Configuring_Hadoop_in_Non-Secure_Mode)
+ * [Configuring Environment of Hadoop Daemons](#Configuring_Environment_of_Hadoop_Daemons)
+ * [Configuring the Hadoop Daemons](#Configuring_the_Hadoop_Daemons)
+ * [Monitoring Health of NodeManagers](#Monitoring_Health_of_NodeManagers)
+ * [Slaves File](#Slaves_File)
+ * [Hadoop Rack Awareness](#Hadoop_Rack_Awareness)
+ * [Logging](#Logging)
+ * [Operating the Hadoop Cluster](#Operating_the_Hadoop_Cluster)
+ * [Hadoop Startup](#Hadoop_Startup)
+ * [Hadoop Shutdown](#Hadoop_Shutdown)
+ * [Web Interfaces](#Web_Interfaces)
+
+Hadoop Cluster Setup
+====================
+
+Purpose
+-------
+
+This document describes how to install and configure Hadoop clusters ranging from a few nodes to extremely large clusters with thousands of nodes. To play with Hadoop, you may first want to install it on a single machine (see [Single Node Setup](./SingleCluster.html)).
+
+This document does not cover advanced topics such as [Security](./SecureMode.html) or High Availability.
+
+Prerequisites
+-------------
+
+* Install Java. See the [Hadoop Wiki](http://wiki.apache.org/hadoop/HadoopJavaVersions) for known good versions.
+* Download a stable version of Hadoop from Apache mirrors.
+
+Installation
+------------
+
+Installing a Hadoop cluster typically involves unpacking the software on all the machines in the cluster or installing it via a packaging system as appropriate for your operating system. It is important to divide up the hardware into functions.
+
+Typically one machine in the cluster is designated as the NameNode and another machine the as ResourceManager, exclusively. These are the masters. Other services (such as Web App Proxy Server and MapReduce Job History server) are usually run either on dedicated hardware or on shared infrastrucutre, depending upon the load.
+
+The rest of the machines in the cluster act as both DataNode and NodeManager. These are the slaves.
+
+Configuring Hadoop in Non-Secure Mode
+-------------------------------------
+
+Hadoop's Java configuration is driven by two types of important configuration files:
+
+* Read-only default configuration - `core-default.xml`, `hdfs-default.xml`, `yarn-default.xml` and `mapred-default.xml`.
+
+* Site-specific configuration - `etc/hadoop/core-site.xml`, `etc/hadoop/hdfs-site.xml`, `etc/hadoop/yarn-site.xml` and `etc/hadoop/mapred-site.xml`.
+
+Additionally, you can control the Hadoop scripts found in the bin/ directory of the distribution, by setting site-specific values via the `etc/hadoop/hadoop-env.sh` and `etc/hadoop/yarn-env.sh`.
+
+To configure the Hadoop cluster you will need to configure the `environment` in which the Hadoop daemons execute as well as the `configuration parameters` for the Hadoop daemons.
+
+HDFS daemons are NameNode, SecondaryNameNode, and DataNode. YARN damones are ResourceManager, NodeManager, and WebAppProxy. If MapReduce is to be used, then the MapReduce Job History Server will also be running. For large installations, these are generally running on separate hosts.
+
+### Configuring Environment of Hadoop Daemons
+
+Administrators should use the `etc/hadoop/hadoop-env.sh` and optionally the `etc/hadoop/mapred-env.sh` and `etc/hadoop/yarn-env.sh` scripts to do site-specific customization of the Hadoop daemons' process environment.
+
+At the very least, you must specify the `JAVA_HOME` so that it is correctly defined on each remote node.
+
+Administrators can configure individual daemons using the configuration options shown below in the table:
+
+| Daemon | Environment Variable |
+|:---- |:---- |
+| NameNode | HADOOP\_NAMENODE\_OPTS |
+| DataNode | HADOOP\_DATANODE\_OPTS |
+| Secondary NameNode | HADOOP\_SECONDARYNAMENODE\_OPTS |
+| ResourceManager | YARN\_RESOURCEMANAGER\_OPTS |
+| NodeManager | YARN\_NODEMANAGER\_OPTS |
+| WebAppProxy | YARN\_PROXYSERVER\_OPTS |
+| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_OPTS |
+
+For example, To configure Namenode to use parallelGC, the following statement should be added in hadoop-env.sh :
+
+ export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC"
+
+See `etc/hadoop/hadoop-env.sh` for other examples.
+
+Other useful configuration parameters that you can customize include:
+
+* `HADOOP_PID_DIR` - The directory where the daemons' process id files are stored.
+* `HADOOP_LOG_DIR` - The directory where the daemons' log files are stored. Log files are automatically created if they don't exist.
+* `HADOOP_HEAPSIZE_MAX` - The maximum amount of memory to use for the Java heapsize. Units supported by the JVM are also supported here. If no unit is present, it will be assumed the number is in megabytes. By default, Hadoop will let the JVM determine how much to use. This value can be overriden on a per-daemon basis using the appropriate `_OPTS` variable listed above. For example, setting `HADOOP_HEAPSIZE_MAX=1g` and `HADOOP_NAMENODE_OPTS="-Xmx5g"` will configure the NameNode with 5GB heap.
+
+In most cases, you should specify the `HADOOP_PID_DIR` and `HADOOP_LOG_DIR` directories such that they can only be written to by the users that are going to run the hadoop daemons. Otherwise there is the potential for a symlink attack.
+
+It is also traditional to configure `HADOOP_PREFIX` in the system-wide shell environment configuration. For example, a simple script inside `/etc/profile.d`:
+
+ HADOOP_PREFIX=/path/to/hadoop
+ export HADOOP_PREFIX
+
+| Daemon | Environment Variable |
+|:---- |:---- |
+| ResourceManager | YARN\_RESOURCEMANAGER\_HEAPSIZE |
+| NodeManager | YARN\_NODEMANAGER\_HEAPSIZE |
+| WebAppProxy | YARN\_PROXYSERVER\_HEAPSIZE |
+| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_HEAPSIZE |
+
+### Configuring the Hadoop Daemons
+
+This section deals with important parameters to be specified in the given configuration files:
+
+* `etc/hadoop/core-site.xml`
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `fs.defaultFS` | NameNode URI | hdfs://host:port/ |
+| `io.file.buffer.size` | 131072 | Size of read/write buffer used in SequenceFiles. |
+
+* `etc/hadoop/hdfs-site.xml`
+
+ * Configurations for NameNode:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `dfs.namenode.name.dir` | Path on the local filesystem where the NameNode stores the namespace and transactions logs persistently. | If this is a comma-delimited list of directories then the name table is replicated in all of the directories, for redundancy. |
+| `dfs.namenode.hosts` / `dfs.namenode.hosts.exclude` | List of permitted/excluded DataNodes. | If necessary, use these files to control the list of allowable datanodes. |
+| `dfs.blocksize` | 268435456 | HDFS blocksize of 256MB for large file-systems. |
+| `dfs.namenode.handler.count` | 100 | More NameNode server threads to handle RPCs from large number of DataNodes. |
+
+ * Configurations for DataNode:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `dfs.datanode.data.dir` | Comma separated list of paths on the local filesystem of a `DataNode` where it should store its blocks. | If this is a comma-delimited list of directories, then data will be stored in all named directories, typically on different devices. |
+
+* `etc/hadoop/yarn-site.xml`
+
+ * Configurations for ResourceManager and NodeManager:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.acl.enable` | `true` / `false` | Enable ACLs? Defaults to *false*. |
+| `yarn.admin.acl` | Admin ACL | ACL to set admins on the cluster. ACLs are of for *comma-separated-usersspacecomma-separated-groups*. Defaults to special value of **\*** which means *anyone*. Special value of just *space* means no one has access. |
+| `yarn.log-aggregation-enable` | *false* | Configuration to enable or disable log aggregation |
+
+ * Configurations for ResourceManager:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.resourcemanager.address` | `ResourceManager` host:port for clients to submit jobs. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
+| `yarn.resourcemanager.scheduler.address` | `ResourceManager` host:port for ApplicationMasters to talk to Scheduler to obtain resources. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
+| `yarn.resourcemanager.resource-tracker.address` | `ResourceManager` host:port for NodeManagers. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
+| `yarn.resourcemanager.admin.address` | `ResourceManager` host:port for administrative commands. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
+| `yarn.resourcemanager.webapp.address` | `ResourceManager` web-ui host:port. | *host:port* If set, overrides the hostname set in `yarn.resourcemanager.hostname`. |
+| `yarn.resourcemanager.hostname` | `ResourceManager` host. | *host* Single hostname that can be set in place of setting all `yarn.resourcemanager*address` resources. Results in default ports for ResourceManager components. |
+| `yarn.resourcemanager.scheduler.class` | `ResourceManager` Scheduler class. | `CapacityScheduler` (recommended), `FairScheduler` (also recommended), or `FifoScheduler` |
+| `yarn.scheduler.minimum-allocation-mb` | Minimum limit of memory to allocate to each container request at the `Resource Manager`. | In MBs |
+| `yarn.scheduler.maximum-allocation-mb` | Maximum limit of memory to allocate to each container request at the `Resource Manager`. | In MBs |
+| `yarn.resourcemanager.nodes.include-path` / `yarn.resourcemanager.nodes.exclude-path` | List of permitted/excluded NodeManagers. | If necessary, use these files to control the list of allowable NodeManagers. |
+
+ * Configurations for NodeManager:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.nodemanager.resource.memory-mb` | Resource i.e. available physical memory, in MB, for given `NodeManager` | Defines total available resources on the `NodeManager` to be made available to running containers |
+| `yarn.nodemanager.vmem-pmem-ratio` | Maximum ratio by which virtual memory usage of tasks may exceed physical memory | The virtual memory usage of each task may exceed its physical memory limit by this ratio. The total amount of virtual memory used by tasks on the NodeManager may exceed its physical memory usage by this ratio. |
+| `yarn.nodemanager.local-dirs` | Comma-separated list of paths on the local filesystem where intermediate data is written. | Multiple paths help spread disk i/o. |
+| `yarn.nodemanager.log-dirs` | Comma-separated list of paths on the local filesystem where logs are written. | Multiple paths help spread disk i/o. |
+| `yarn.nodemanager.log.retain-seconds` | *10800* | Default time (in seconds) to retain log files on the NodeManager Only applicable if log-aggregation is disabled. |
+| `yarn.nodemanager.remote-app-log-dir` | */logs* | HDFS directory where the application logs are moved on application completion. Need to set appropriate permissions. Only applicable if log-aggregation is enabled. |
+| `yarn.nodemanager.remote-app-log-dir-suffix` | *logs* | Suffix appended to the remote log dir. Logs will be aggregated to ${yarn.nodemanager.remote-app-log-dir}/${user}/${thisParam} Only applicable if log-aggregation is enabled. |
+| `yarn.nodemanager.aux-services` | mapreduce\_shuffle | Shuffle service that needs to be set for Map Reduce applications. |
+
+ * Configurations for History Server (Needs to be moved elsewhere):
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.log-aggregation.retain-seconds` | *-1* | How long to keep aggregation logs before deleting them. -1 disables. Be careful, set this too small and you will spam the name node. |
+| `yarn.log-aggregation.retain-check-interval-seconds` | *-1* | Time between checks for aggregated log retention. If set to 0 or a negative value then the value is computed as one-tenth of the aggregated log retention time. Be careful, set this too small and you will spam the name node. |
+
+* `etc/hadoop/mapred-site.xml`
+
+ * Configurations for MapReduce Applications:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `mapreduce.framework.name` | yarn | Execution framework set to Hadoop YARN. |
+| `mapreduce.map.memory.mb` | 1536 | Larger resource limit for maps. |
+| `mapreduce.map.java.opts` | -Xmx1024M | Larger heap-size for child jvms of maps. |
+| `mapreduce.reduce.memory.mb` | 3072 | Larger resource limit for reduces. |
+| `mapreduce.reduce.java.opts` | -Xmx2560M | Larger heap-size for child jvms of reduces. |
+| `mapreduce.task.io.sort.mb` | 512 | Higher memory-limit while sorting data for efficiency. |
+| `mapreduce.task.io.sort.factor` | 100 | More streams merged at once while sorting files. |
+| `mapreduce.reduce.shuffle.parallelcopies` | 50 | Higher number of parallel copies run by reduces to fetch outputs from very large number of maps. |
+
+ * Configurations for MapReduce JobHistory Server:
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `mapreduce.jobhistory.address` | MapReduce JobHistory Server *host:port* | Default port is 10020. |
+| `mapreduce.jobhistory.webapp.address` | MapReduce JobHistory Server Web UI *host:port* | Default port is 19888. |
+| `mapreduce.jobhistory.intermediate-done-dir` | /mr-history/tmp | Directory where history files are written by MapReduce jobs. |
+| `mapreduce.jobhistory.done-dir` | /mr-history/done | Directory where history files are managed by the MR JobHistory Server. |
+
+Monitoring Health of NodeManagers
+---------------------------------
+
+Hadoop provides a mechanism by which administrators can configure the NodeManager to run an administrator supplied script periodically to determine if a node is healthy or not.
+
+Administrators can determine if the node is in a healthy state by performing any checks of their choice in the script. If the script detects the node to be in an unhealthy state, it must print a line to standard output beginning with the string ERROR. The NodeManager spawns the script periodically and checks its output. If the script's output contains the string ERROR, as described above, the node's status is reported as `unhealthy` and the node is black-listed by the ResourceManager. No further tasks will be assigned to this node. However, the NodeManager continues to run the script, so that if the node becomes healthy again, it will be removed from the blacklisted nodes on the ResourceManager automatically. The node's health along with the output of the script, if it is unhealthy, is available to the administrator in the ResourceManager web interface. The time since the node was healthy is also displayed on the web interface.
+
+The following parameters can be used to control the node health monitoring script in `etc/hadoop/yarn-site.xml`.
+
+| Parameter | Value | Notes |
+|:---- |:---- |:---- |
+| `yarn.nodemanager.health-checker.script.path` | Node health script | Script to check for node's health status. |
+| `yarn.nodemanager.health-checker.script.opts` | Node health script options | Options for script to check for node's health status. |
+| `yarn.nodemanager.health-checker.script.interval-ms` | Node health script interval | Time interval for running health script. |
+| `yarn.nodemanager.health-checker.script.timeout-ms` | Node health script timeout interval | Timeout for health script execution. |
+
+The health checker script is not supposed to give ERROR if only some of the local disks become bad. NodeManager has the ability to periodically check the health of the local disks (specifically checks nodemanager-local-dirs and nodemanager-log-dirs) and after reaching the threshold of number of bad directories based on the value set for the config property yarn.nodemanager.disk-health-checker.min-healthy-disks, the whole node is marked unhealthy and this info is sent to resource manager also. The boot disk is either raided or a failure in the boot disk is identified by the health checker script.
+
+Slaves File
+-----------
+
+List all slave hostnames or IP addresses in your `etc/hadoop/slaves` file, one per line. Helper scripts (described below) will use the `etc/hadoop/slaves` file to run commands on many hosts at once. It is not used for any of the Java-based Hadoop configuration. In order to use this functionality, ssh trusts (via either passphraseless ssh or some other means, such as Kerberos) must be established for the accounts used to run Hadoop.
+
+Hadoop Rack Awareness
+---------------------
+
+Many Hadoop components are rack-aware and take advantage of the network topology for performance and safety. Hadoop daemons obtain the rack information of the slaves in the cluster by invoking an administrator configured module. See the [Rack Awareness](./RackAwareness.html) documentation for more specific information.
+
+It is highly recommended configuring rack awareness prior to starting HDFS.
+
+Logging
+-------
+
+Hadoop uses the [Apache log4j](http://logging.apache.org/log4j/2.x/) via the Apache Commons Logging framework for logging. Edit the `etc/hadoop/log4j.properties` file to customize the Hadoop daemons' logging configuration (log-formats and so on).
+
+Operating the Hadoop Cluster
+----------------------------
+
+Once all the necessary configuration is complete, distribute the files to the `HADOOP_CONF_DIR` directory on all the machines. This should be the same directory on all machines.
+
+In general, it is recommended that HDFS and YARN run as separate users. In the majority of installations, HDFS processes execute as 'hdfs'. YARN is typically using the 'yarn' account.
+
+### Hadoop Startup
+
+To start a Hadoop cluster you will need to start both the HDFS and YARN cluster.
+
+The first time you bring up HDFS, it must be formatted. Format a new distributed filesystem as *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format <cluster_name>
+
+Start the HDFS NameNode with the following command on the designated node as *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode
+
+Start a HDFS DataNode with the following command on each designated node as *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start datanode
+
+If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes can be started with a utility script. As *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh
+
+Start the YARN with the following command, run on the designated ResourceManager as *yarn*:
+
+ [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start resourcemanager
+
+Run a script to start a NodeManager on each designated host as *yarn*:
+
+ [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start nodemanager
+
+Start a standalone WebAppProxy server. Run on the WebAppProxy server as *yarn*. If multiple servers are used with load balancing it should be run on each of them:
+
+ [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start proxyserver
+
+If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be started with a utility script. As *yarn*:
+
+ [yarn]$ $HADOOP_PREFIX/sbin/start-yarn.sh
+
+Start the MapReduce JobHistory Server with the following command, run on the designated server as *mapred*:
+
+ [mapred]$ $HADOOP_PREFIX/bin/mapred --daemon start historyserver
+
+### Hadoop Shutdown
+
+Stop the NameNode with the following command, run on the designated NameNode as *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop namenode
+
+Run a script to stop a DataNode as *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop datanode
+
+If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the HDFS processes may be stopped with a utility script. As *hdfs*:
+
+ [hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh
+
+Stop the ResourceManager with the following command, run on the designated ResourceManager as *yarn*:
+
+ [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop resourcemanager
+
+Run a script to stop a NodeManager on a slave as *yarn*:
+
+ [yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop nodemanager
+
+If `etc/hadoop/slaves` and ssh trusted access is configured (see [Single Node Setup](./SingleCluster.html)), all of the YARN processes can be stopped with a utility script. As *yarn*:
+
+ [yarn]$ $HADOOP_PREFIX/sbin/stop-yarn.sh
+
+Stop the WebAppProxy server. Run on the WebAppProxy server as *yarn*. If multiple servers are used with load balancing it should be run on each of them:
+
+ [yarn]$ $HADOOP_PREFIX/bin/yarn stop proxyserver
+
+Stop the MapReduce JobHistory Server with the following command, run on the designated server as *mapred*:
+
+ [mapred]$ $HADOOP_PREFIX/bin/mapred --daemon stop historyserver
+
+Web Interfaces
+--------------
+
+Once the Hadoop cluster is up and running check the web-ui of the components as described below:
+
+| Daemon | Web Interface | Notes |
+|:---- |:---- |:---- |
+| NameNode | http://nn_host:port/ | Default HTTP port is 50070. |
+| ResourceManager | http://rm_host:port/ | Default HTTP port is 8088. |
+| MapReduce JobHistory Server | http://jhs_host:port/ | Default HTTP port is 19888. |
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
new file mode 100644
index 0000000..b9c92c4
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/CommandsManual.md
@@ -0,0 +1,227 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Hadoop Commands Guide](#Hadoop_Commands_Guide)
+ * [Overview](#Overview)
+ * [Shell Options](#Shell_Options)
+ * [Generic Options](#Generic_Options)
+* [Hadoop Common Commands](#Hadoop_Common_Commands)
+ * [User Commands](#User_Commands)
+ * [archive](#archive)
+ * [checknative](#checknative)
+ * [classpath](#classpath)
+ * [credential](#credential)
+ * [distch](#distch)
+ * [distcp](#distcp)
+ * [fs](#fs)
+ * [jar](#jar)
+ * [jnipath](#jnipath)
+ * [key](#key)
+ * [trace](#trace)
+ * [version](#version)
+ * [CLASSNAME](#CLASSNAME)
+ * [Administration Commands](#Administration_Commands)
+ * [daemonlog](#daemonlog)
+ * [Files](#Files)
+ * [etc/hadoop/hadoop-env.sh](#etchadoophadoop-env.sh)
+ * [etc/hadoop/hadoop-user-functions.sh](#etchadoophadoop-user-functions.sh)
+ * [~/.hadooprc](#a.hadooprc)
+
+Hadoop Commands Guide
+=====================
+
+Overview
+--------
+
+All of the Hadoop commands and subprojects follow the same basic structure:
+
+Usage: `shellcommand [SHELL_OPTIONS] [COMMAND] [GENERIC_OPTIONS] [COMMAND_OPTIONS]`
+
+| FIELD | Description |
+|:---- |:---- |
+| shellcommand | The command of the project being invoked. For example, Hadoop common uses `hadoop`, HDFS uses `hdfs`, and YARN uses `yarn`. |
+| SHELL\_OPTIONS | Options that the shell processes prior to executing Java. |
+| COMMAND | Action to perform. |
+| GENERIC\_OPTIONS | The common set of options supported by multiple commands. |
+| COMMAND\_OPTIONS | Various commands with their options are described in this documention for the Hadoop common sub-project. HDFS and YARN are covered in other documents. |
+
+### Shell Options
+
+All of the shell commands will accept a common set of options. For some commands, these options are ignored. For example, passing `---hostnames` on a command that only executes on a single host will be ignored.
+
+| SHELL\_OPTION | Description |
+|:---- |:---- |
+| `--buildpaths` | Enables developer versions of jars. |
+| `--config confdir` | Overwrites the default Configuration directory. Default is `$HADOOP_PREFIX/conf`. |
+| `--daemon mode` | If the command supports daemonization (e.g., `hdfs namenode`), execute in the appropriate mode. Supported modes are `start` to start the process in daemon mode, `stop` to stop the process, and `status` to determine the active status of the process. `status` will return an [LSB-compliant](http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html) result code. If no option is provided, commands that support daemonization will run in the foreground. |
+| `--debug` | Enables shell level configuration debugging information |
+| `--help` | Shell script usage information. |
+| `--hostnames` | A space delimited list of hostnames where to execute a multi-host subcommand. By default, the content of the `slaves` file is used. |
+| `--hosts` | A file that contains a list of hostnames where to execute a multi-host subcommand. By default, the content of the `slaves` file is used. |
+| `--loglevel loglevel` | Overrides the log level. Valid log levels are FATAL, ERROR, WARN, INFO, DEBUG, and TRACE. Default is INFO. |
+
+### Generic Options
+
+Many subcommands honor a common set of configuration options to alter their behavior:
+
+| GENERIC\_OPTION | Description |
+|:---- |:---- |
+| `-archives <comma separated list of archives> ` | Specify comma separated archives to be unarchived on the compute machines. Applies only to job. |
+| `-conf <configuration file> ` | Specify an application configuration file. |
+| `-D <property>=<value> ` | Use value for given property. |
+| `-files <comma separated list of files> ` | Specify comma separated files to be copied to the map reduce cluster. Applies only to job. |
+| `-jt <local> or <resourcemanager:port>` | Specify a ResourceManager. Applies only to job. |
+| `-libjars <comma seperated list of jars> ` | Specify comma separated jar files to include in the classpath. Applies only to job. |
+
+Hadoop Common Commands
+======================
+
+All of these commands are executed from the `hadoop` shell command. They have been broken up into [User Commands](#User_Commands) and [Admininistration Commands](#Admininistration_Commands).
+
+User Commands
+-------------
+
+Commands useful for users of a hadoop cluster.
+
+### `archive`
+
+Creates a hadoop archive. More information can be found at [Hadoop Archives Guide](../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html).
+
+### `checknative`
+
+Usage: `hadoop checknative [-a] [-h] `
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| `-a` | Check all libraries are available. |
+| `-h` | print help |
+
+This command checks the availability of the Hadoop native code. See [\#NativeLibraries.html](#NativeLibraries.html) for more information. By default, this command only checks the availability of libhadoop.
+
+### `classpath`
+
+Usage: `hadoop classpath [--glob |--jar <path> |-h |--help]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| `--glob` | expand wildcards |
+| `--jar` *path* | write classpath as manifest in jar named *path* |
+| `-h`, `--help` | print help |
+
+Prints the class path needed to get the Hadoop jar and the required libraries. If called without arguments, then prints the classpath set up by the command scripts, which is likely to contain wildcards in the classpath entries. Additional options print the classpath after wildcard expansion or write the classpath into the manifest of a jar file. The latter is useful in environments where wildcards cannot be used and the expanded classpath exceeds the maximum supported command line length.
+
+### `credential`
+
+Usage: `hadoop credential <subcommand> [options]`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| create *alias* [-v *value*][-provider *provider-path*] | Prompts the user for a credential to be stored as the given alias when a value is not provided via `-v`. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. |
+| delete *alias* [-i][-provider *provider-path*] | Deletes the credential with the provided alias and optionally warns the user when `--interactive` is used. The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. |
+| list [-provider *provider-path*] | Lists all of the credential aliases The *hadoop.security.credential.provider.path* within the core-site.xml file will be used unless a `-provider` is indicated. |
+
+Command to manage credentials, passwords and secrets within credential providers.
+
+The CredentialProvider API in Hadoop allows for the separation of applications and how they store their required passwords/secrets. In order to indicate a particular provider type and location, the user must provide the *hadoop.security.credential.provider.path* configuration element in core-site.xml or use the command line option `-provider` on each of the following commands. This provider path is a comma-separated list of URLs that indicates the type and location of a list of providers that should be consulted. For example, the following path: `user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks`
+
+indicates that the current user's credentials file should be consulted through the User Provider, that the local file located at `/tmp/test.jceks` is a Java Keystore Provider and that the file located within HDFS at `nn1.example.com/my/path/test.jceks` is also a store for a Java Keystore Provider.
+
+When utilizing the credential command it will often be for provisioning a password or secret to a particular credential store provider. In order to explicitly indicate which provider store to use the `-provider` option should be used. Otherwise, given a path of multiple providers, the first non-transient provider will be used. This may or may not be the one that you intended.
+
+Example: `-provider jceks://file/tmp/test.jceks`
+
+### `distch`
+
+Usage: `hadoop distch [-f urilist_url] [-i] [-log logdir] path:owner:group:permissions`
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| `-f` | List of objects to change |
+| `-i` | Ignore failures |
+| `-log` | Directory to log output |
+
+Change the ownership and permissions on many files at once.
+
+### `distcp`
+
+Copy file or directories recursively. More information can be found at [Hadoop DistCp Guide](../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html).
+
+### `fs`
+
+This command is documented in the [File System Shell Guide](./FileSystemShell.html). It is a synonym for `hdfs dfs` when HDFS is in use.
+
+### `jar`
+
+Usage: `hadoop jar <jar> [mainClass] args...`
+
+Runs a jar file.
+
+Use [`yarn jar`](../../hadoop-yarn/hadoop-yarn-site/YarnCommands.html#jar) to launch YARN applications instead.
+
+### `jnipath`
+
+Usage: `hadoop jnipath`
+
+Print the computed java.library.path.
+
+### `key`
+
+Manage keys via the KeyProvider.
+
+### `trace`
+
+View and modify Hadoop tracing settings. See the [Tracing Guide](./Tracing.html).
+
+### `version`
+
+Usage: `hadoop version`
+
+Prints the version.
+
+### `CLASSNAME`
+
+Usage: `hadoop CLASSNAME`
+
+Runs the class named `CLASSNAME`. The class must be part of a package.
+
+Administration Commands
+-----------------------
+
+Commands useful for administrators of a hadoop cluster.
+
+### `daemonlog`
+
+Usage: `hadoop daemonlog -getlevel <host:port> <name> ` Usage: `hadoop daemonlog -setlevel <host:port> <name> <level> `
+
+| COMMAND\_OPTION | Description |
+|:---- |:---- |
+| `-getlevel` *host:port* *name* | Prints the log level of the daemon running at *host:port*. This command internally connects to http://host:port/logLevel?log=name |
+| `-setlevel` *host:port* *name* *level* | Sets the log level of the daemon running at *host:port*. This command internally connects to http://host:port/logLevel?log=name |
+
+Get/Set the log level for each daemon.
+
+Files
+-----
+
+### **etc/hadoop/hadoop-env.sh**
+
+This file stores the global settings used by all Hadoop shell commands.
+
+### **etc/hadoop/hadoop-user-functions.sh**
+
+This file allows for advanced users to override some shell functionality.
+
+### **~/.hadooprc**
+
+This stores the personal environment for an individual user. It is processed after the hadoop-env.sh and hadoop-user-functions.sh files and can contain the same settings.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md b/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md
new file mode 100644
index 0000000..c058021
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Compatibility.md
@@ -0,0 +1,313 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Apache Hadoop Compatibility
+===========================
+
+* [Apache Hadoop Compatibility](#Apache_Hadoop_Compatibility)
+ * [Purpose](#Purpose)
+ * [Compatibility types](#Compatibility_types)
+ * [Java API](#Java_API)
+ * [Use Cases](#Use_Cases)
+ * [Policy](#Policy)
+ * [Semantic compatibility](#Semantic_compatibility)
+ * [Policy](#Policy)
+ * [Wire compatibility](#Wire_compatibility)
+ * [Use Cases](#Use_Cases)
+ * [Policy](#Policy)
+ * [Java Binary compatibility for end-user applications i.e. Apache Hadoop ABI](#Java_Binary_compatibility_for_end-user_applications_i.e._Apache_Hadoop_ABI)
+ * [Use cases](#Use_cases)
+ * [Policy](#Policy)
+ * [REST APIs](#REST_APIs)
+ * [Policy](#Policy)
+ * [Metrics/JMX](#MetricsJMX)
+ * [Policy](#Policy)
+ * [File formats & Metadata](#File_formats__Metadata)
+ * [User-level file formats](#User-level_file_formats)
+ * [Policy](#Policy)
+ * [System-internal file formats](#System-internal_file_formats)
+ * [MapReduce](#MapReduce)
+ * [Policy](#Policy)
+ * [HDFS Metadata](#HDFS_Metadata)
+ * [Policy](#Policy)
+ * [Command Line Interface (CLI)](#Command_Line_Interface_CLI)
+ * [Policy](#Policy)
+ * [Web UI](#Web_UI)
+ * [Policy](#Policy)
+ * [Hadoop Configuration Files](#Hadoop_Configuration_Files)
+ * [Policy](#Policy)
+ * [Directory Structure](#Directory_Structure)
+ * [Policy](#Policy)
+ * [Java Classpath](#Java_Classpath)
+ * [Policy](#Policy)
+ * [Environment variables](#Environment_variables)
+ * [Policy](#Policy)
+ * [Build artifacts](#Build_artifacts)
+ * [Policy](#Policy)
+ * [Hardware/Software Requirements](#HardwareSoftware_Requirements)
+ * [Policies](#Policies)
+ * [References](#References)
+
+Purpose
+-------
+
+This document captures the compatibility goals of the Apache Hadoop project. The different types of compatibility between Hadoop releases that affects Hadoop developers, downstream projects, and end-users are enumerated. For each type of compatibility we:
+
+* describe the impact on downstream projects or end-users
+* where applicable, call out the policy adopted by the Hadoop developers when incompatible changes are permitted.
+
+Compatibility types
+-------------------
+
+### Java API
+
+Hadoop interfaces and classes are annotated to describe the intended audience and stability in order to maintain compatibility with previous releases. See [Hadoop Interface Classification](./InterfaceClassification.html) for details.
+
+* InterfaceAudience: captures the intended audience, possible values are Public (for end users and external projects), LimitedPrivate (for other Hadoop components, and closely related projects like YARN, MapReduce, HBase etc.), and Private (for intra component use).
+* InterfaceStability: describes what types of interface changes are permitted. Possible values are Stable, Evolving, Unstable, and Deprecated.
+
+#### Use Cases
+
+* Public-Stable API compatibility is required to ensure end-user programs and downstream projects continue to work without modification.
+* LimitedPrivate-Stable API compatibility is required to allow upgrade of individual components across minor releases.
+* Private-Stable API compatibility is required for rolling upgrades.
+
+#### Policy
+
+* Public-Stable APIs must be deprecated for at least one major release prior to their removal in a major release.
+* LimitedPrivate-Stable APIs can change across major releases, but not within a major release.
+* Private-Stable APIs can change across major releases, but not within a major release.
+* Classes not annotated are implicitly "Private". Class members not annotated inherit the annotations of the enclosing class.
+* Note: APIs generated from the proto files need to be compatible for rolling-upgrades. See the section on wire-compatibility for more details. The compatibility policies for APIs and wire-communication need to go hand-in-hand to address this.
+
+### Semantic compatibility
+
+Apache Hadoop strives to ensure that the behavior of APIs remains consistent over versions, though changes for correctness may result in changes in behavior. Tests and javadocs specify the API's behavior. The community is in the process of specifying some APIs more rigorously, and enhancing test suites to verify compliance with the specification, effectively creating a formal specification for the subset of behaviors that can be easily tested.
+
+#### Policy
+
+The behavior of API may be changed to fix incorrect behavior, such a change to be accompanied by updating existing buggy tests or adding tests in cases there were none prior to the change.
+
+### Wire compatibility
+
+Wire compatibility concerns data being transmitted over the wire between Hadoop processes. Hadoop uses Protocol Buffers for most RPC communication. Preserving compatibility requires prohibiting modification as described below. Non-RPC communication should be considered as well, for example using HTTP to transfer an HDFS image as part of snapshotting or transferring MapTask output. The potential communications can be categorized as follows:
+
+* Client-Server: communication between Hadoop clients and servers (e.g., the HDFS client to NameNode protocol, or the YARN client to ResourceManager protocol).
+* Client-Server (Admin): It is worth distinguishing a subset of the Client-Server protocols used solely by administrative commands (e.g., the HAAdmin protocol) as these protocols only impact administrators who can tolerate changes that end users (which use general Client-Server protocols) can not.
+* Server-Server: communication between servers (e.g., the protocol between the DataNode and NameNode, or NodeManager and ResourceManager)
+
+#### Use Cases
+
+* Client-Server compatibility is required to allow users to continue using the old clients even after upgrading the server (cluster) to a later version (or vice versa). For example, a Hadoop 2.1.0 client talking to a Hadoop 2.3.0 cluster.
+* Client-Server compatibility is also required to allow users to upgrade the client before upgrading the server (cluster). For example, a Hadoop 2.4.0 client talking to a Hadoop 2.3.0 cluster. This allows deployment of client-side bug fixes ahead of full cluster upgrades. Note that new cluster features invoked by new client APIs or shell commands will not be usable. YARN applications that attempt to use new APIs (including new fields in data structures) that have not yet deployed to the cluster can expect link exceptions.
+* Client-Server compatibility is also required to allow upgrading individual components without upgrading others. For example, upgrade HDFS from version 2.1.0 to 2.2.0 without upgrading MapReduce.
+* Server-Server compatibility is required to allow mixed versions within an active cluster so the cluster may be upgraded without downtime in a rolling fashion.
+
+#### Policy
+
+* Both Client-Server and Server-Server compatibility is preserved within a major release. (Different policies for different categories are yet to be considered.)
+* Compatibility can be broken only at a major release, though breaking compatibility even at major releases has grave consequences and should be discussed in the Hadoop community.
+* Hadoop protocols are defined in .proto (ProtocolBuffers) files. Client-Server protocols and Server-protocol .proto files are marked as stable. When a .proto file is marked as stable it means that changes should be made in a compatible fashion as described below:
+ * The following changes are compatible and are allowed at any time:
+ * Add an optional field, with the expectation that the code deals with the field missing due to communication with an older version of the code.
+ * Add a new rpc/method to the service
+ * Add a new optional request to a Message
+ * Rename a field
+ * Rename a .proto file
+ * Change .proto annotations that effect code generation (e.g. name of java package)
+ * The following changes are incompatible but can be considered only at a major release
+ * Change the rpc/method name
+ * Change the rpc/method parameter type or return type
+ * Remove an rpc/method
+ * Change the service name
+ * Change the name of a Message
+ * Modify a field type in an incompatible way (as defined recursively)
+ * Change an optional field to required
+ * Add or delete a required field
+ * Delete an optional field as long as the optional field has reasonable defaults to allow deletions
+ * The following changes are incompatible and hence never allowed
+ * Change a field id
+ * Reuse an old field that was previously deleted.
+ * Field numbers are cheap and changing and reusing is not a good idea.
+
+### Java Binary compatibility for end-user applications i.e. Apache Hadoop ABI
+
+As Apache Hadoop revisions are upgraded end-users reasonably expect that their applications should continue to work without any modifications. This is fulfilled as a result of support API compatibility, Semantic compatibility and Wire compatibility.
+
+However, Apache Hadoop is a very complex, distributed system and services a very wide variety of use-cases. In particular, Apache Hadoop MapReduce is a very, very wide API; in the sense that end-users may make wide-ranging assumptions such as layout of the local disk when their map/reduce tasks are executing, environment variables for their tasks etc. In such cases, it becomes very hard to fully specify, and support, absolute compatibility.
+
+#### Use cases
+
+* Existing MapReduce applications, including jars of existing packaged end-user applications and projects such as Apache Pig, Apache Hive, Cascading etc. should work unmodified when pointed to an upgraded Apache Hadoop cluster within a major release.
+* Existing YARN applications, including jars of existing packaged end-user applications and projects such as Apache Tez etc. should work unmodified when pointed to an upgraded Apache Hadoop cluster within a major release.
+* Existing applications which transfer data in/out of HDFS, including jars of existing packaged end-user applications and frameworks such as Apache Flume, should work unmodified when pointed to an upgraded Apache Hadoop cluster within a major release.
+
+#### Policy
+
+* Existing MapReduce, YARN & HDFS applications and frameworks should work unmodified within a major release i.e. Apache Hadoop ABI is supported.
+* A very minor fraction of applications maybe affected by changes to disk layouts etc., the developer community will strive to minimize these changes and will not make them within a minor version. In more egregious cases, we will consider strongly reverting these breaking changes and invalidating offending releases if necessary.
+* In particular for MapReduce applications, the developer community will try our best to support provide binary compatibility across major releases e.g. applications using org.apache.hadoop.mapred.
+* APIs are supported compatibly across hadoop-1.x and hadoop-2.x. See [Compatibility for MapReduce applications between hadoop-1.x and hadoop-2.x](../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html) for more details.
+
+### REST APIs
+
+REST API compatibility corresponds to both the request (URLs) and responses to each request (content, which may contain other URLs). Hadoop REST APIs are specifically meant for stable use by clients across releases, even major releases. The following are the exposed REST APIs:
+
+* [WebHDFS](../hadoop-hdfs/WebHDFS.html) - Stable
+* [ResourceManager](../../hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html)
+* [NodeManager](../../hadoop-yarn/hadoop-yarn-site/NodeManagerRest.html)
+* [MR Application Master](../../hadoop-yarn/hadoop-yarn-site/MapredAppMasterRest.html)
+* [History Server](../../hadoop-yarn/hadoop-yarn-site/HistoryServerRest.html)
+
+#### Policy
+
+The APIs annotated stable in the text above preserve compatibility across at least one major release, and maybe deprecated by a newer version of the REST API in a major release.
+
+### Metrics/JMX
+
+While the Metrics API compatibility is governed by Java API compatibility, the actual metrics exposed by Hadoop need to be compatible for users to be able to automate using them (scripts etc.). Adding additional metrics is compatible. Modifying (eg changing the unit or measurement) or removing existing metrics breaks compatibility. Similarly, changes to JMX MBean object names also break compatibility.
+
+#### Policy
+
+Metrics should preserve compatibility within the major release.
+
+### File formats & Metadata
+
+User and system level data (including metadata) is stored in files of different formats. Changes to the metadata or the file formats used to store data/metadata can lead to incompatibilities between versions.
+
+#### User-level file formats
+
+Changes to formats that end-users use to store their data can prevent them for accessing the data in later releases, and hence it is highly important to keep those file-formats compatible. One can always add a "new" format improving upon an existing format. Examples of these formats include har, war, SequenceFileFormat etc.
+
+##### Policy
+
+* Non-forward-compatible user-file format changes are restricted to major releases. When user-file formats change, new releases are expected to read existing formats, but may write data in formats incompatible with prior releases. Also, the community shall prefer to create a new format that programs must opt in to instead of making incompatible changes to existing formats.
+
+#### System-internal file formats
+
+Hadoop internal data is also stored in files and again changing these formats can lead to incompatibilities. While such changes are not as devastating as the user-level file formats, a policy on when the compatibility can be broken is important.
+
+##### MapReduce
+
+MapReduce uses formats like I-File to store MapReduce-specific data.
+
+##### Policy
+
+MapReduce-internal formats like IFile maintain compatibility within a major release. Changes to these formats can cause in-flight jobs to fail and hence we should ensure newer clients can fetch shuffle-data from old servers in a compatible manner.
+
+##### HDFS Metadata
+
+HDFS persists metadata (the image and edit logs) in a particular format. Incompatible changes to either the format or the metadata prevent subsequent releases from reading older metadata. Such incompatible changes might require an HDFS "upgrade" to convert the metadata to make it accessible. Some changes can require more than one such "upgrades".
+
+Depending on the degree of incompatibility in the changes, the following potential scenarios can arise:
+
+* Automatic: The image upgrades automatically, no need for an explicit "upgrade".
+* Direct: The image is upgradable, but might require one explicit release "upgrade".
+* Indirect: The image is upgradable, but might require upgrading to intermediate release(s) first.
+* Not upgradeable: The image is not upgradeable.
+
+##### Policy
+
+* A release upgrade must allow a cluster to roll-back to the older version and its older disk format. The rollback needs to restore the original data, but not required to restore the updated data.
+* HDFS metadata changes must be upgradeable via any of the upgrade paths - automatic, direct or indirect.
+* More detailed policies based on the kind of upgrade are yet to be considered.
+
+### Command Line Interface (CLI)
+
+The Hadoop command line programs may be use either directly via the system shell or via shell scripts. Changing the path of a command, removing or renaming command line options, the order of arguments, or the command return code and output break compatibility and may adversely affect users.
+
+#### Policy
+
+CLI commands are to be deprecated (warning when used) for one major release before they are removed or incompatibly modified in a subsequent major release.
+
+### Web UI
+
+Web UI, particularly the content and layout of web pages, changes could potentially interfere with attempts to screen scrape the web pages for information.
+
+#### Policy
+
+Web pages are not meant to be scraped and hence incompatible changes to them are allowed at any time. Users are expected to use REST APIs to get any information.
+
+### Hadoop Configuration Files
+
+Users use (1) Hadoop-defined properties to configure and provide hints to Hadoop and (2) custom properties to pass information to jobs. Hence, compatibility of config properties is two-fold:
+
+* Modifying key-names, units of values, and default values of Hadoop-defined properties.
+* Custom configuration property keys should not conflict with the namespace of Hadoop-defined properties. Typically, users should avoid using prefixes used by Hadoop: hadoop, io, ipc, fs, net, file, ftp, s3, kfs, ha, file, dfs, mapred, mapreduce, yarn.
+
+#### Policy
+
+* Hadoop-defined properties are to be deprecated at least for one major release before being removed. Modifying units for existing properties is not allowed.
+* The default values of Hadoop-defined properties can be changed across minor/major releases, but will remain the same across point releases within a minor release.
+* Currently, there is NO explicit policy regarding when new prefixes can be added/removed, and the list of prefixes to be avoided for custom configuration properties. However, as noted above, users should avoid using prefixes used by Hadoop: hadoop, io, ipc, fs, net, file, ftp, s3, kfs, ha, file, dfs, mapred, mapreduce, yarn.
+
+### Directory Structure
+
+Source code, artifacts (source and tests), user logs, configuration files, output and job history are all stored on disk either local file system or HDFS. Changing the directory structure of these user-accessible files break compatibility, even in cases where the original path is preserved via symbolic links (if, for example, the path is accessed by a servlet that is configured to not follow symbolic links).
+
+#### Policy
+
+* The layout of source code and build artifacts can change anytime, particularly so across major versions. Within a major version, the developers will attempt (no guarantees) to preserve the directory structure; however, individual files can be added/moved/deleted. The best way to ensure patches stay in sync with the code is to get them committed to the Apache source tree.
+* The directory structure of configuration files, user logs, and job history will be preserved across minor and point releases within a major release.
+
+### Java Classpath
+
+User applications built against Hadoop might add all Hadoop jars (including Hadoop's library dependencies) to the application's classpath. Adding new dependencies or updating the version of existing dependencies may interfere with those in applications' classpaths.
+
+#### Policy
+
+Currently, there is NO policy on when Hadoop's dependencies can change.
+
+### Environment variables
+
+Users and related projects often utilize the exported environment variables (eg HADOOP\_CONF\_DIR), therefore removing or renaming environment variables is an incompatible change.
+
+#### Policy
+
+Currently, there is NO policy on when the environment variables can change. Developers try to limit changes to major releases.
+
+### Build artifacts
+
+Hadoop uses maven for project management and changing the artifacts can affect existing user workflows.
+
+#### Policy
+
+* Test artifacts: The test jars generated are strictly for internal use and are not expected to be used outside of Hadoop, similar to APIs annotated @Private, @Unstable.
+* Built artifacts: The hadoop-client artifact (maven groupId:artifactId) stays compatible within a major release, while the other artifacts can change in incompatible ways.
+
+### Hardware/Software Requirements
+
+To keep up with the latest advances in hardware, operating systems, JVMs, and other software, new Hadoop releases or some of their features might require higher versions of the same. For a specific environment, upgrading Hadoop might require upgrading other dependent software components.
+
+#### Policies
+
+* Hardware
+ * Architecture: The community has no plans to restrict Hadoop to specific architectures, but can have family-specific optimizations.
+ * Minimum resources: While there are no guarantees on the minimum resources required by Hadoop daemons, the community attempts to not increase requirements within a minor release.
+* Operating Systems: The community will attempt to maintain the same OS requirements (OS kernel versions) within a minor release. Currently GNU/Linux and Microsoft Windows are the OSes officially supported by the community while Apache Hadoop is known to work reasonably well on other OSes such as Apple MacOSX, Solaris etc.
+* The JVM requirements will not change across point releases within the same minor release except if the JVM version under question becomes unsupported. Minor/major releases might require later versions of JVM for some/all of the supported operating systems.
+* Other software: The community tries to maintain the minimum versions of additional software required by Hadoop. For example, ssh, kerberos etc.
+
+References
+----------
+
+Here are some relevant JIRAs and pages related to the topic:
+
+* The evolution of this document - [HADOOP-9517](https://issues.apache.org/jira/browse/HADOOP-9517)
+* Binary compatibility for MapReduce end-user applications between hadoop-1.x and hadoop-2.x - [MapReduce Compatibility between hadoop-1.x and hadoop-2.x](../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html)
+* Annotations for interfaces as per interface classification schedule - [HADOOP-7391](https://issues.apache.org/jira/browse/HADOOP-7391) [Hadoop Interface Classification](./InterfaceClassification.html)
+* Compatibility for Hadoop 1.x releases - [HADOOP-5071](https://issues.apache.org/jira/browse/HADOOP-5071)
+* The [Hadoop Roadmap](http://wiki.apache.org/hadoop/Roadmap) page that captures other release policies
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
new file mode 100644
index 0000000..dae9928
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/DeprecatedProperties.md
@@ -0,0 +1,288 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Deprecated Properties
+=====================
+
+The following table lists the configuration property names that are deprecated in this version of Hadoop, and their replacements.
+
+| **Deprecated property name** | **New property name** |
+|:---- |:---- |
+| create.empty.dir.if.nonexist | mapreduce.jobcontrol.createdir.ifnotexist |
+| dfs.access.time.precision | dfs.namenode.accesstime.precision |
+| dfs.backup.address | dfs.namenode.backup.address |
+| dfs.backup.http.address | dfs.namenode.backup.http-address |
+| dfs.balance.bandwidthPerSec | dfs.datanode.balance.bandwidthPerSec |
+| dfs.block.size | dfs.blocksize |
+| dfs.data.dir | dfs.datanode.data.dir |
+| dfs.datanode.max.xcievers | dfs.datanode.max.transfer.threads |
+| dfs.df.interval | fs.df.interval |
+| dfs.federation.nameservice.id | dfs.nameservice.id |
+| dfs.federation.nameservices | dfs.nameservices |
+| dfs.http.address | dfs.namenode.http-address |
+| dfs.https.address | dfs.namenode.https-address |
+| dfs.https.client.keystore.resource | dfs.client.https.keystore.resource |
+| dfs.https.need.client.auth | dfs.client.https.need-auth |
+| dfs.max.objects | dfs.namenode.max.objects |
+| dfs.max-repl-streams | dfs.namenode.replication.max-streams |
+| dfs.name.dir | dfs.namenode.name.dir |
+| dfs.name.dir.restore | dfs.namenode.name.dir.restore |
+| dfs.name.edits.dir | dfs.namenode.edits.dir |
+| dfs.permissions | dfs.permissions.enabled |
+| dfs.permissions.supergroup | dfs.permissions.superusergroup |
+| dfs.read.prefetch.size | dfs.client.read.prefetch.size |
+| dfs.replication.considerLoad | dfs.namenode.replication.considerLoad |
+| dfs.replication.interval | dfs.namenode.replication.interval |
+| dfs.replication.min | dfs.namenode.replication.min |
+| dfs.replication.pending.timeout.sec | dfs.namenode.replication.pending.timeout-sec |
+| dfs.safemode.extension | dfs.namenode.safemode.extension |
+| dfs.safemode.threshold.pct | dfs.namenode.safemode.threshold-pct |
+| dfs.secondary.http.address | dfs.namenode.secondary.http-address |
+| dfs.socket.timeout | dfs.client.socket-timeout |
+| dfs.umaskmode | fs.permissions.umask-mode |
+| dfs.write.packet.size | dfs.client-write-packet-size |
+| fs.checkpoint.dir | dfs.namenode.checkpoint.dir |
+| fs.checkpoint.edits.dir | dfs.namenode.checkpoint.edits.dir |
+| fs.checkpoint.period | dfs.namenode.checkpoint.period |
+| fs.default.name | fs.defaultFS |
+| hadoop.configured.node.mapping | net.topology.configured.node.mapping |
+| hadoop.job.history.location | mapreduce.jobtracker.jobhistory.location |
+| hadoop.native.lib | io.native.lib.available |
+| hadoop.net.static.resolutions | mapreduce.tasktracker.net.static.resolutions |
+| hadoop.pipes.command-file.keep | mapreduce.pipes.commandfile.preserve |
+| hadoop.pipes.executable.interpretor | mapreduce.pipes.executable.interpretor |
+| hadoop.pipes.executable | mapreduce.pipes.executable |
+| hadoop.pipes.java.mapper | mapreduce.pipes.isjavamapper |
+| hadoop.pipes.java.recordreader | mapreduce.pipes.isjavarecordreader |
+| hadoop.pipes.java.recordwriter | mapreduce.pipes.isjavarecordwriter |
+| hadoop.pipes.java.reducer | mapreduce.pipes.isjavareducer |
+| hadoop.pipes.partitioner | mapreduce.pipes.partitioner |
+| heartbeat.recheck.interval | dfs.namenode.heartbeat.recheck-interval |
+| io.bytes.per.checksum | dfs.bytes-per-checksum |
+| io.sort.factor | mapreduce.task.io.sort.factor |
+| io.sort.mb | mapreduce.task.io.sort.mb |
+| io.sort.spill.percent | mapreduce.map.sort.spill.percent |
+| jobclient.completion.poll.interval | mapreduce.client.completion.pollinterval |
+| jobclient.output.filter | mapreduce.client.output.filter |
+| jobclient.progress.monitor.poll.interval | mapreduce.client.progressmonitor.pollinterval |
+| job.end.notification.url | mapreduce.job.end-notification.url |
+| job.end.retry.attempts | mapreduce.job.end-notification.retry.attempts |
+| job.end.retry.interval | mapreduce.job.end-notification.retry.interval |
+| job.local.dir | mapreduce.job.local.dir |
+| keep.failed.task.files | mapreduce.task.files.preserve.failedtasks |
+| keep.task.files.pattern | mapreduce.task.files.preserve.filepattern |
+| key.value.separator.in.input.line | mapreduce.input.keyvaluelinerecordreader.key.value.separator |
+| local.cache.size | mapreduce.tasktracker.cache.local.size |
+| map.input.file | mapreduce.map.input.file |
+| map.input.length | mapreduce.map.input.length |
+| map.input.start | mapreduce.map.input.start |
+| map.output.key.field.separator | mapreduce.map.output.key.field.separator |
+| map.output.key.value.fields.spec | mapreduce.fieldsel.map.output.key.value.fields.spec |
+| mapred.acls.enabled | mapreduce.cluster.acls.enabled |
+| mapred.binary.partitioner.left.offset | mapreduce.partition.binarypartitioner.left.offset |
+| mapred.binary.partitioner.right.offset | mapreduce.partition.binarypartitioner.right.offset |
+| mapred.cache.archives | mapreduce.job.cache.archives |
+| mapred.cache.archives.timestamps | mapreduce.job.cache.archives.timestamps |
+| mapred.cache.files | mapreduce.job.cache.files |
+| mapred.cache.files.timestamps | mapreduce.job.cache.files.timestamps |
+| mapred.cache.localArchives | mapreduce.job.cache.local.archives |
+| mapred.cache.localFiles | mapreduce.job.cache.local.files |
+| mapred.child.tmp | mapreduce.task.tmp.dir |
+| mapred.cluster.average.blacklist.threshold | mapreduce.jobtracker.blacklist.average.threshold |
+| mapred.cluster.map.memory.mb | mapreduce.cluster.mapmemory.mb |
+| mapred.cluster.max.map.memory.mb | mapreduce.jobtracker.maxmapmemory.mb |
+| mapred.cluster.max.reduce.memory.mb | mapreduce.jobtracker.maxreducememory.mb |
+| mapred.cluster.reduce.memory.mb | mapreduce.cluster.reducememory.mb |
+| mapred.committer.job.setup.cleanup.needed | mapreduce.job.committer.setup.cleanup.needed |
+| mapred.compress.map.output | mapreduce.map.output.compress |
+| mapred.data.field.separator | mapreduce.fieldsel.data.field.separator |
+| mapred.debug.out.lines | mapreduce.task.debugout.lines |
+| mapred.healthChecker.interval | mapreduce.tasktracker.healthchecker.interval |
+| mapred.healthChecker.script.args | mapreduce.tasktracker.healthchecker.script.args |
+| mapred.healthChecker.script.path | mapreduce.tasktracker.healthchecker.script.path |
+| mapred.healthChecker.script.timeout | mapreduce.tasktracker.healthchecker.script.timeout |
+| mapred.heartbeats.in.second | mapreduce.jobtracker.heartbeats.in.second |
+| mapred.hosts.exclude | mapreduce.jobtracker.hosts.exclude.filename |
+| mapred.hosts | mapreduce.jobtracker.hosts.filename |
+| mapred.inmem.merge.threshold | mapreduce.reduce.merge.inmem.threshold |
+| mapred.input.dir.formats | mapreduce.input.multipleinputs.dir.formats |
+| mapred.input.dir.mappers | mapreduce.input.multipleinputs.dir.mappers |
+| mapred.input.dir | mapreduce.input.fileinputformat.inputdir |
+| mapred.input.pathFilter.class | mapreduce.input.pathFilter.class |
+| mapred.jar | mapreduce.job.jar |
+| mapred.job.classpath.archives | mapreduce.job.classpath.archives |
+| mapred.job.classpath.files | mapreduce.job.classpath.files |
+| mapred.job.id | mapreduce.job.id |
+| mapred.jobinit.threads | mapreduce.jobtracker.jobinit.threads |
+| mapred.job.map.memory.mb | mapreduce.map.memory.mb |
+| mapred.job.name | mapreduce.job.name |
+| mapred.job.priority | mapreduce.job.priority |
+| mapred.job.queue.name | mapreduce.job.queuename |
+| mapred.job.reduce.input.buffer.percent | mapreduce.reduce.input.buffer.percent |
+| mapred.job.reduce.markreset.buffer.percent | mapreduce.reduce.markreset.buffer.percent |
+| mapred.job.reduce.memory.mb | mapreduce.reduce.memory.mb |
+| mapred.job.reduce.total.mem.bytes | mapreduce.reduce.memory.totalbytes |
+| mapred.job.reuse.jvm.num.tasks | mapreduce.job.jvm.numtasks |
+| mapred.job.shuffle.input.buffer.percent | mapreduce.reduce.shuffle.input.buffer.percent |
+| mapred.job.shuffle.merge.percent | mapreduce.reduce.shuffle.merge.percent |
+| mapred.job.tracker.handler.count | mapreduce.jobtracker.handler.count |
+| mapred.job.tracker.history.completed.location | mapreduce.jobtracker.jobhistory.completed.location |
+| mapred.job.tracker.http.address | mapreduce.jobtracker.http.address |
+| mapred.jobtracker.instrumentation | mapreduce.jobtracker.instrumentation |
+| mapred.jobtracker.job.history.block.size | mapreduce.jobtracker.jobhistory.block.size |
+| mapred.job.tracker.jobhistory.lru.cache.size | mapreduce.jobtracker.jobhistory.lru.cache.size |
+| mapred.job.tracker | mapreduce.jobtracker.address |
+| mapred.jobtracker.maxtasks.per.job | mapreduce.jobtracker.maxtasks.perjob |
+| mapred.job.tracker.persist.jobstatus.active | mapreduce.jobtracker.persist.jobstatus.active |
+| mapred.job.tracker.persist.jobstatus.dir | mapreduce.jobtracker.persist.jobstatus.dir |
+| mapred.job.tracker.persist.jobstatus.hours | mapreduce.jobtracker.persist.jobstatus.hours |
+| mapred.jobtracker.restart.recover | mapreduce.jobtracker.restart.recover |
+| mapred.job.tracker.retiredjobs.cache.size | mapreduce.jobtracker.retiredjobs.cache.size |
+| mapred.job.tracker.retire.jobs | mapreduce.jobtracker.retirejobs |
+| mapred.jobtracker.taskalloc.capacitypad | mapreduce.jobtracker.taskscheduler.taskalloc.capacitypad |
+| mapred.jobtracker.taskScheduler | mapreduce.jobtracker.taskscheduler |
+| mapred.jobtracker.taskScheduler.maxRunningTasksPerJob | mapreduce.jobtracker.taskscheduler.maxrunningtasks.perjob |
+| mapred.join.expr | mapreduce.join.expr |
+| mapred.join.keycomparator | mapreduce.join.keycomparator |
+| mapred.lazy.output.format | mapreduce.output.lazyoutputformat.outputformat |
+| mapred.line.input.format.linespermap | mapreduce.input.lineinputformat.linespermap |
+| mapred.linerecordreader.maxlength | mapreduce.input.linerecordreader.line.maxlength |
+| mapred.local.dir | mapreduce.cluster.local.dir |
+| mapred.local.dir.minspacekill | mapreduce.tasktracker.local.dir.minspacekill |
+| mapred.local.dir.minspacestart | mapreduce.tasktracker.local.dir.minspacestart |
+| mapred.map.child.env | mapreduce.map.env |
+| mapred.map.child.java.opts | mapreduce.map.java.opts |
+| mapred.map.child.log.level | mapreduce.map.log.level |
+| mapred.map.max.attempts | mapreduce.map.maxattempts |
+| mapred.map.output.compression.codec | mapreduce.map.output.compress.codec |
+| mapred.mapoutput.key.class | mapreduce.map.output.key.class |
+| mapred.mapoutput.value.class | mapreduce.map.output.value.class |
+| mapred.mapper.regex.group | mapreduce.mapper.regexmapper..group |
+| mapred.mapper.regex | mapreduce.mapper.regex |
+| mapred.map.task.debug.script | mapreduce.map.debug.script |
+| mapred.map.tasks | mapreduce.job.maps |
+| mapred.map.tasks.speculative.execution | mapreduce.map.speculative |
+| mapred.max.map.failures.percent | mapreduce.map.failures.maxpercent |
+| mapred.max.reduce.failures.percent | mapreduce.reduce.failures.maxpercent |
+| mapred.max.split.size | mapreduce.input.fileinputformat.split.maxsize |
+| mapred.max.tracker.blacklists | mapreduce.jobtracker.tasktracker.maxblacklists |
+| mapred.max.tracker.failures | mapreduce.job.maxtaskfailures.per.tracker |
+| mapred.merge.recordsBeforeProgress | mapreduce.task.merge.progress.records |
+| mapred.min.split.size | mapreduce.input.fileinputformat.split.minsize |
+| mapred.min.split.size.per.node | mapreduce.input.fileinputformat.split.minsize.per.node |
+| mapred.min.split.size.per.rack | mapreduce.input.fileinputformat.split.minsize.per.rack |
+| mapred.output.compression.codec | mapreduce.output.fileoutputformat.compress.codec |
+| mapred.output.compression.type | mapreduce.output.fileoutputformat.compress.type |
+| mapred.output.compress | mapreduce.output.fileoutputformat.compress |
+| mapred.output.dir | mapreduce.output.fileoutputformat.outputdir |
+| mapred.output.key.class | mapreduce.job.output.key.class |
+| mapred.output.key.comparator.class | mapreduce.job.output.key.comparator.class |
+| mapred.output.value.class | mapreduce.job.output.value.class |
+| mapred.output.value.groupfn.class | mapreduce.job.output.group.comparator.class |
+| mapred.permissions.supergroup | mapreduce.cluster.permissions.supergroup |
+| mapred.pipes.user.inputformat | mapreduce.pipes.inputformat |
+| mapred.reduce.child.env | mapreduce.reduce.env |
+| mapred.reduce.child.java.opts | mapreduce.reduce.java.opts |
+| mapred.reduce.child.log.level | mapreduce.reduce.log.level |
+| mapred.reduce.max.attempts | mapreduce.reduce.maxattempts |
+| mapred.reduce.parallel.copies | mapreduce.reduce.shuffle.parallelcopies |
+| mapred.reduce.slowstart.completed.maps | mapreduce.job.reduce.slowstart.completedmaps |
+| mapred.reduce.task.debug.script | mapreduce.reduce.debug.script |
+| mapred.reduce.tasks | mapreduce.job.reduces |
+| mapred.reduce.tasks.speculative.execution | mapreduce.reduce.speculative |
+| mapred.seqbinary.output.key.class | mapreduce.output.seqbinaryoutputformat.key.class |
+| mapred.seqbinary.output.value.class | mapreduce.output.seqbinaryoutputformat.value.class |
+| mapred.shuffle.connect.timeout | mapreduce.reduce.shuffle.connect.timeout |
+| mapred.shuffle.read.timeout | mapreduce.reduce.shuffle.read.timeout |
+| mapred.skip.attempts.to.start.skipping | mapreduce.task.skip.start.attempts |
+| mapred.skip.map.auto.incr.proc.count | mapreduce.map.skip.proc-count.auto-incr |
+| mapred.skip.map.max.skip.records | mapreduce.map.skip.maxrecords |
+| mapred.skip.on | mapreduce.job.skiprecords |
+| mapred.skip.out.dir | mapreduce.job.skip.outdir |
+| mapred.skip.reduce.auto.incr.proc.count | mapreduce.reduce.skip.proc-count.auto-incr |
+| mapred.skip.reduce.max.skip.groups | mapreduce.reduce.skip.maxgroups |
+| mapred.speculative.execution.slowNodeThreshold | mapreduce.job.speculative.slownodethreshold |
+| mapred.speculative.execution.slowTaskThreshold | mapreduce.job.speculative.slowtaskthreshold |
+| mapred.speculative.execution.speculativeCap | mapreduce.job.speculative.speculativecap |
+| mapred.submit.replication | mapreduce.client.submit.file.replication |
+| mapred.system.dir | mapreduce.jobtracker.system.dir |
+| mapred.task.cache.levels | mapreduce.jobtracker.taskcache.levels |
+| mapred.task.id | mapreduce.task.attempt.id |
+| mapred.task.is.map | mapreduce.task.ismap |
+| mapred.task.partition | mapreduce.task.partition |
+| mapred.task.profile | mapreduce.task.profile |
+| mapred.task.profile.maps | mapreduce.task.profile.maps |
+| mapred.task.profile.params | mapreduce.task.profile.params |
+| mapred.task.profile.reduces | mapreduce.task.profile.reduces |
+| mapred.task.timeout | mapreduce.task.timeout |
+| mapred.tasktracker.dns.interface | mapreduce.tasktracker.dns.interface |
+| mapred.tasktracker.dns.nameserver | mapreduce.tasktracker.dns.nameserver |
+| mapred.tasktracker.events.batchsize | mapreduce.tasktracker.events.batchsize |
+| mapred.tasktracker.expiry.interval | mapreduce.jobtracker.expire.trackers.interval |
+| mapred.task.tracker.http.address | mapreduce.tasktracker.http.address |
+| mapred.tasktracker.indexcache.mb | mapreduce.tasktracker.indexcache.mb |
+| mapred.tasktracker.instrumentation | mapreduce.tasktracker.instrumentation |
+| mapred.tasktracker.map.tasks.maximum | mapreduce.tasktracker.map.tasks.maximum |
+| mapred.tasktracker.memory\_calculator\_plugin | mapreduce.tasktracker.resourcecalculatorplugin |
+| mapred.tasktracker.memorycalculatorplugin | mapreduce.tasktracker.resourcecalculatorplugin |
+| mapred.tasktracker.reduce.tasks.maximum | mapreduce.tasktracker.reduce.tasks.maximum |
+| mapred.task.tracker.report.address | mapreduce.tasktracker.report.address |
+| mapred.task.tracker.task-controller | mapreduce.tasktracker.taskcontroller |
+| mapred.tasktracker.taskmemorymanager.monitoring-interval | mapreduce.tasktracker.taskmemorymanager.monitoringinterval |
+| mapred.tasktracker.tasks.sleeptime-before-sigkill | mapreduce.tasktracker.tasks.sleeptimebeforesigkill |
+| mapred.temp.dir | mapreduce.cluster.temp.dir |
+| mapred.text.key.comparator.options | mapreduce.partition.keycomparator.options |
+| mapred.text.key.partitioner.options | mapreduce.partition.keypartitioner.options |
+| mapred.textoutputformat.separator | mapreduce.output.textoutputformat.separator |
+| mapred.tip.id | mapreduce.task.id |
+| mapreduce.combine.class | mapreduce.job.combine.class |
+| mapreduce.inputformat.class | mapreduce.job.inputformat.class |
+| mapreduce.job.counters.limit | mapreduce.job.counters.max |
+| mapreduce.jobtracker.permissions.supergroup | mapreduce.cluster.permissions.supergroup |
+| mapreduce.map.class | mapreduce.job.map.class |
+| mapreduce.outputformat.class | mapreduce.job.outputformat.class |
+| mapreduce.partitioner.class | mapreduce.job.partitioner.class |
+| mapreduce.reduce.class | mapreduce.job.reduce.class |
+| mapred.used.genericoptionsparser | mapreduce.client.genericoptionsparser.used |
+| mapred.userlog.limit.kb | mapreduce.task.userlog.limit.kb |
+| mapred.userlog.retain.hours | mapreduce.job.userlog.retain.hours |
+| mapred.working.dir | mapreduce.job.working.dir |
+| mapred.work.output.dir | mapreduce.task.output.dir |
+| min.num.spills.for.combine | mapreduce.map.combine.minspills |
+| reduce.output.key.value.fields.spec | mapreduce.fieldsel.reduce.output.key.value.fields.spec |
+| security.job.submission.protocol.acl | security.job.client.protocol.acl |
+| security.task.umbilical.protocol.acl | security.job.task.protocol.acl |
+| sequencefile.filter.class | mapreduce.input.sequencefileinputfilter.class |
+| sequencefile.filter.frequency | mapreduce.input.sequencefileinputfilter.frequency |
+| sequencefile.filter.regex | mapreduce.input.sequencefileinputfilter.regex |
+| session.id | dfs.metrics.session-id |
+| slave.host.name | dfs.datanode.hostname |
+| slave.host.name | mapreduce.tasktracker.host.name |
+| tasktracker.contention.tracking | mapreduce.tasktracker.contention.tracking |
+| tasktracker.http.threads | mapreduce.tasktracker.http.threads |
+| topology.node.switch.mapping.impl | net.topology.node.switch.mapping.impl |
+| topology.script.file.name | net.topology.script.file.name |
+| topology.script.number.args | net.topology.script.number.args |
+| user.name | mapreduce.job.user.name |
+| webinterface.private.actions | mapreduce.jobtracker.webinterface.trusted |
+| yarn.app.mapreduce.yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts | yarn.app.mapreduce.client-am.ipc.max-retries-on-timeouts |
+
+The following table lists additional changes to some configuration properties:
+
+| **Deprecated property name** | **New property name** |
+|:---- |:---- |
+| mapred.create.symlink | NONE - symlinking is always on |
+| mapreduce.job.cache.symlink.create | NONE - symlinking is always on |
+
+
[38/50] [abbrv] hadoop git commit: HADOOP-11581. Multithreaded
correctness Warnings #org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via
aw)
Posted by zj...@apache.org.
HADOOP-11581. Multithreaded correctness Warnings #org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b015fec6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b015fec6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b015fec6
Branch: refs/heads/YARN-2928
Commit: b015fec6011586784fd6f4791a0c1c22e3902046
Parents: cfd8a21
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Feb 11 07:49:05 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Wed Feb 11 07:49:05 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/fs/shell/Ls.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b015fec6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e2f1761..26d84d8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -382,6 +382,9 @@ Trunk (Unreleased)
HADOOP-11559. Add links to RackAwareness and InterfaceClassification
to site index (Masatake Iwasaki via aw)
+ HADOOP-11581. Multithreaded correctness Warnings
+ #org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via aw)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b015fec6/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 0e46700..8ef6d5e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -84,7 +84,7 @@ class Ls extends FsCommand {
" Use time of last access instead of modification for\n" +
" display and sorting.";
- protected static final SimpleDateFormat dateFormat =
+ protected final SimpleDateFormat dateFormat =
new SimpleDateFormat("yyyy-MM-dd HH:mm");
protected int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
[42/50] [abbrv] hadoop git commit: HADOOP-11579. Documentation for
truncate. Contributed by Konstantin Shvachko.
Posted by zj...@apache.org.
HADOOP-11579. Documentation for truncate. Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f6737b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f6737b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f6737b3
Branch: refs/heads/YARN-2928
Commit: 6f6737b3bfda0f5c0b0a508dc8523f6636f3f4ab
Parents: 22441ab
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Wed Feb 11 09:50:19 2015 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Wed Feb 11 09:50:19 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/site/markdown/FileSystemShell.md | 20 +++++++++++
.../src/site/markdown/filesystem/filesystem.md | 35 ++++++++++++++++++++
3 files changed, 57 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6737b3/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 3035024..fca4b14 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -582,6 +582,8 @@ Release 2.7.0 - UNRELEASED
HADOOP-11520. Clean incomplete multi-part uploads in S3A tests.
(Thomas Demoor via stevel)
+ HADOOP-11579. Documentation for truncate. (shv)
+
OPTIMIZATIONS
HADOOP-11323. WritableComparator#compare keeps reference to byte array.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6737b3/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
index ae3bea8..305370d 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -54,6 +54,7 @@
* [test](#test)
* [text](#text)
* [touchz](#touchz)
+ * [truncate](#truncate)
* [usage](#usage)
Overview
@@ -681,6 +682,25 @@ Example:
Exit Code: Returns 0 on success and -1 on error.
+truncate
+--------
+
+Usage: `hadoop fs -truncate [-w] <length> <paths>`
+
+Truncate all files that match the specified file pattern
+to the specified length.
+
+Options:
+
+* The -w flag requests that the command waits for block recovery to complete, if necessary.
+Without -w flag the file may remain unclosed for some time while the recovery is in progress.
+During this time file cannot be reopened for append.
+
+Example:
+
+* `hadoop fs -truncate 55 /user/hadoop/file1 /user/hadoop/file2`
+* `hadoop fs -truncate -w 127 hdfs://nn1.example.com/user/hadoop/file1`
+
usage
-----
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f6737b3/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index e59fa1b..84e3755 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -831,3 +831,38 @@ HDFS's restrictions may be an implementation detail of how it implements
a sequence. As no other filesystem in the Hadoop core codebase
implements this method, there is no way to distinguish implementation detail.
from specification.
+
+
+### `boolean truncate(Path p, long newLength)`
+
+Truncate file `p` to the specified `newLength`.
+
+Implementations MAY throw `UnsupportedOperationException`.
+
+#### Preconditions
+
+ if not exists(FS, p) : raise FileNotFoundException
+
+ if isDir(FS, p) : raise [FileNotFoundException, IOException]
+
+ if newLength < 0 || newLength > len(FS.Files[p]) : raise HadoopIllegalArgumentException
+
+HDFS: The source file MUST be closed.
+Truncate cannot be performed on a file, which is open for writing or appending.
+
+#### Postconditions
+
+ FS' where:
+ len(FS.Files[p]) = newLength
+
+Return: `true`, if truncation is finished and the file can be immediately
+opened for appending, or `false` otherwise.
+
+HDFS: HDFS reutrns `false` to indicate that a background process of adjusting
+the length of the last block has been started, and clients should wait for it
+to complete before they can proceed with further file updates.
+
+#### Concurrency
+
+If an input stream is open when truncate() occurs, the outcome of read
+operations related to the part of the file being truncated is undefined.
[15/50] [abbrv] hadoop git commit: HDFS-7718. Store KeyProvider in
ClientContext to avoid leaking key provider threads when using FileContext
(Arun Suresh via Colin P. McCabe)
Posted by zj...@apache.org.
HDFS-7718. Store KeyProvider in ClientContext to avoid leaking key provider threads when using FileContext (Arun Suresh via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/02340a24
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/02340a24
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/02340a24
Branch: refs/heads/YARN-2928
Commit: 02340a24f211212b91dc7380c1e5b54ddb5e82eb
Parents: a9dc5cd
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Feb 9 20:23:23 2015 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Feb 9 20:23:23 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../org/apache/hadoop/hdfs/ClientContext.java | 14 ++-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 45 +++----
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 9 ++
.../apache/hadoop/hdfs/KeyProviderCache.java | 109 ++++++++++++++++
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +-
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 2 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 2 +-
.../hadoop/hdfs/TestKeyProviderCache.java | 124 +++++++++++++++++++
.../hadoop/hdfs/TestReservedRawPaths.java | 4 +-
....apache.hadoop.crypto.key.KeyProviderFactory | 16 +++
11 files changed, 304 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 446c6a3..4a6bc11 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -893,6 +893,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7744. Fix potential NPE in DFSInputStream after setDropBehind or
setReadahead is called (cmccabe)
+ HDFS-7718. Store KeyProvider in ClientContext to avoid leaking key provider
+ threads when using FileContext (Arun Suresh via Colin P. McCabe)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ClientContext.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
index e106fca..af7c095 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/ClientContext.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.shortcircuit.ShortCircuitCache;
import org.apache.hadoop.hdfs.util.ByteArrayManager;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.Cache;
/**
* ClientContext contains context information for a client.
@@ -72,6 +73,10 @@ public class ClientContext {
private final DomainSocketFactory domainSocketFactory;
/**
+ * Caches key Providers for the DFSClient
+ */
+ private final KeyProviderCache keyProviderCache;
+ /**
* True if we should use the legacy BlockReaderLocal.
*/
private final boolean useLegacyBlockReaderLocal;
@@ -107,6 +112,7 @@ public class ClientContext {
conf.shortCircuitSharedMemoryWatcherInterruptCheckMs);
this.peerCache =
new PeerCache(conf.socketCacheCapacity, conf.socketCacheExpiry);
+ this.keyProviderCache = new KeyProviderCache(conf.keyProviderCacheExpiryMs);
this.useLegacyBlockReaderLocal = conf.useLegacyBlockReaderLocal;
this.domainSocketFactory = new DomainSocketFactory(conf);
@@ -138,7 +144,9 @@ public class ClientContext {
append(", domainSocketDataTraffic = ").
append(conf.domainSocketDataTraffic).
append(", shortCircuitSharedMemoryWatcherInterruptCheckMs = ").
- append(conf.shortCircuitSharedMemoryWatcherInterruptCheckMs);
+ append(conf.shortCircuitSharedMemoryWatcherInterruptCheckMs).
+ append(", keyProviderCacheExpiryMs = ").
+ append(conf.keyProviderCacheExpiryMs);
return builder.toString();
}
@@ -195,6 +203,10 @@ public class ClientContext {
return peerCache;
}
+ public KeyProviderCache getKeyProviderCache() {
+ return keyProviderCache;
+ }
+
public boolean getUseLegacyBlockReaderLocal() {
return useLegacyBlockReaderLocal;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index e5bf98d..4869706 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -275,8 +275,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
private static final DFSHedgedReadMetrics HEDGED_READ_METRIC =
new DFSHedgedReadMetrics();
private static ThreadPoolExecutor HEDGED_READ_THREAD_POOL;
- @VisibleForTesting
- KeyProvider provider;
private final Sampler<?> traceSampler;
/**
@@ -336,6 +334,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
final long shortCircuitMmapCacheRetryTimeout;
final long shortCircuitCacheStaleThresholdMs;
+ final long keyProviderCacheExpiryMs;
+
public Conf(Configuration conf) {
// The hdfsTimeout is currently the same as the ipc timeout
hdfsTimeout = Client.getTimeout(conf);
@@ -499,6 +499,10 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
dfsclientSlowIoWarningThresholdMs = conf.getLong(
DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
DFSConfigKeys.DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
+
+ keyProviderCacheExpiryMs = conf.getLong(
+ DFSConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS,
+ DFSConfigKeys.DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT);
}
public boolean isUseLegacyBlockReaderLocal() {
@@ -638,14 +642,6 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
this.authority = nameNodeUri == null? "null": nameNodeUri.getAuthority();
this.clientName = "DFSClient_" + dfsClientConf.taskId + "_" +
DFSUtil.getRandom().nextInt() + "_" + Thread.currentThread().getId();
- provider = DFSUtil.createKeyProvider(conf);
- if (LOG.isDebugEnabled()) {
- if (provider == null) {
- LOG.debug("No KeyProvider found.");
- } else {
- LOG.debug("Found KeyProvider: " + provider.toString());
- }
- }
int numResponseToDrop = conf.getInt(
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_KEY,
DFSConfigKeys.DFS_CLIENT_TEST_DROP_NAMENODE_RESPONSE_NUM_DEFAULT);
@@ -961,18 +957,12 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*/
@Override
public synchronized void close() throws IOException {
- try {
- if(clientRunning) {
- closeAllFilesBeingWritten(false);
- clientRunning = false;
- getLeaseRenewer().closeClient(this);
- // close connections to the namenode
- closeConnectionToNamenode();
- }
- } finally {
- if (provider != null) {
- provider.close();
- }
+ if(clientRunning) {
+ closeAllFilesBeingWritten(false);
+ clientRunning = false;
+ getLeaseRenewer().closeClient(this);
+ // close connections to the namenode
+ closeConnectionToNamenode();
}
}
@@ -1382,6 +1372,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
feInfo) throws IOException {
TraceScope scope = Trace.startSpan("decryptEDEK", traceSampler);
try {
+ KeyProvider provider = getKeyProvider();
if (provider == null) {
throw new IOException("No KeyProvider is configured, cannot access" +
" an encrypted file");
@@ -3499,12 +3490,16 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
public KeyProvider getKeyProvider() {
- return provider;
+ return clientContext.getKeyProviderCache().get(conf);
}
@VisibleForTesting
- public void setKeyProvider(KeyProviderCryptoExtension provider) {
- this.provider = provider;
+ public void setKeyProvider(KeyProvider provider) {
+ try {
+ clientContext.getKeyProviderCache().setKeyProvider(conf, provider);
+ } catch (IOException e) {
+ LOG.error("Could not set KeyProvider !!", e);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index beb3e38..e4343bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.hdfs;
+import java.util.concurrent.TimeUnit;
+
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicyDefault;
@@ -769,4 +771,11 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String[] NNTOP_WINDOWS_MINUTES_DEFAULT = {"1","5","25"};
public static final String DFS_PIPELINE_ECN_ENABLED = "dfs.pipeline.ecn";
public static final boolean DFS_PIPELINE_ECN_ENABLED_DEFAULT = false;
+
+ // Key Provider Cache Expiry
+ public static final String DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_MS =
+ "dfs.client.key.provider.cache.expiry";
+ // 10 days
+ public static final long DFS_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT =
+ TimeUnit.DAYS.toMillis(10);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
new file mode 100644
index 0000000..68ff554
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/KeyProviderCache.java
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.cache.Cache;
+import com.google.common.cache.CacheBuilder;
+import com.google.common.cache.RemovalListener;
+import com.google.common.cache.RemovalNotification;
+
+public class KeyProviderCache {
+
+ public static final Log LOG = LogFactory.getLog(KeyProviderCache.class);
+
+ private final Cache<URI, KeyProvider> cache;
+
+ public KeyProviderCache(long expiryMs) {
+ cache = CacheBuilder.newBuilder()
+ .expireAfterAccess(expiryMs, TimeUnit.MILLISECONDS)
+ .removalListener(new RemovalListener<URI, KeyProvider>() {
+ @Override
+ public void onRemoval(
+ RemovalNotification<URI, KeyProvider> notification) {
+ try {
+ notification.getValue().close();
+ } catch (Throwable e) {
+ LOG.error(
+ "Error closing KeyProvider with uri ["
+ + notification.getKey() + "]", e);
+ ;
+ }
+ }
+ })
+ .build();
+ }
+
+ public KeyProvider get(final Configuration conf) {
+ URI kpURI = createKeyProviderURI(conf);
+ if (kpURI == null) {
+ return null;
+ }
+ try {
+ return cache.get(kpURI, new Callable<KeyProvider>() {
+ @Override
+ public KeyProvider call() throws Exception {
+ return DFSUtil.createKeyProvider(conf);
+ }
+ });
+ } catch (Exception e) {
+ LOG.error("Could not create KeyProvider for DFSClient !!", e.getCause());
+ return null;
+ }
+ }
+
+ private URI createKeyProviderURI(Configuration conf) {
+ final String providerUriStr =
+ conf.get(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, null);
+ // No provider set in conf
+ if (providerUriStr == null) {
+ LOG.error("Could not find uri with key ["
+ + DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI
+ + "] to create a keyProvider !!");
+ return null;
+ }
+ final URI providerUri;
+ try {
+ providerUri = new URI(providerUriStr);
+ } catch (URISyntaxException e) {
+ LOG.error("KeyProvider URI string is invalid [" + providerUriStr
+ + "]!!", e.getCause());
+ return null;
+ }
+ return providerUri;
+ }
+
+ @VisibleForTesting
+ public void setKeyProvider(Configuration conf, KeyProvider keyProvider)
+ throws IOException {
+ URI uri = createKeyProviderURI(conf);
+ cache.put(uri, keyProvider);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
index 13eb4cf..dbb7ea5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZones.java
@@ -38,11 +38,13 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import com.google.common.collect.Lists;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.CreateFlag;
@@ -96,7 +98,6 @@ import static org.mockito.Matchers.anyShort;
import static org.mockito.Mockito.withSettings;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.anyString;
-
import static org.apache.hadoop.hdfs.DFSTestUtil.verifyFilesEqual;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.assertEquals;
@@ -105,6 +106,7 @@ import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+
import org.xml.sax.InputSource;
import org.xml.sax.helpers.DefaultHandler;
@@ -157,8 +159,8 @@ public class TestEncryptionZones {
protected void setProvider() {
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
- fs.getClient().provider = cluster.getNameNode().getNamesystem()
- .getProvider();
+ fs.getClient().setKeyProvider(cluster.getNameNode().getNamesystem()
+ .getProvider());
}
@After
@@ -1072,7 +1074,7 @@ public class TestEncryptionZones {
addDelegationTokens(anyString(), (Credentials)any())).
thenReturn(new Token<?>[] { testToken });
- dfs.getClient().provider = keyProvider;
+ dfs.getClient().setKeyProvider(keyProvider);
Credentials creds = new Credentials();
final Token<?> tokens[] = dfs.addDelegationTokens("JobTracker", creds);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
index 3339f16..e61a02b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithHA.java
@@ -80,7 +80,7 @@ public class TestEncryptionZonesWithHA {
dfsAdmin1 = new HdfsAdmin(cluster.getURI(1), conf);
KeyProviderCryptoExtension nn0Provider =
cluster.getNameNode(0).getNamesystem().getProvider();
- fs.getClient().provider = nn0Provider;
+ fs.getClient().setKeyProvider(nn0Provider);
}
@After
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
index b165c56..0040d75 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestEncryptionZonesWithKMS.java
@@ -71,7 +71,7 @@ public class TestEncryptionZonesWithKMS extends TestEncryptionZones {
final Path zonePath = new Path("/TestEncryptionZone");
fsWrapper.mkdir(zonePath, FsPermission.getDirDefault(), false);
dfsAdmin.createEncryptionZone(zonePath, TEST_KEY);
- assertTrue(((KMSClientProvider)fs.getClient().provider).
+ assertTrue(((KMSClientProvider)fs.getClient().getKeyProvider()).
getEncKeyQueueSize(TEST_KEY) > 0);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
new file mode 100644
index 0000000..4cbe871
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestKeyProviderCache.java
@@ -0,0 +1,124 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestKeyProviderCache {
+
+ public static class DummyKeyProvider extends KeyProvider {
+
+ public DummyKeyProvider(Configuration conf) {
+ super(conf);
+ }
+
+ @Override
+ public KeyVersion getKeyVersion(String versionName) throws IOException {
+ return null;
+ }
+
+ @Override
+ public List<String> getKeys() throws IOException {
+ return null;
+ }
+
+ @Override
+ public List<KeyVersion> getKeyVersions(String name) throws IOException {
+ return null;
+ }
+
+ @Override
+ public Metadata getMetadata(String name) throws IOException {
+ return null;
+ }
+
+ @Override
+ public KeyVersion createKey(String name, byte[] material, Options options)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public void deleteKey(String name) throws IOException {
+ }
+
+ @Override
+ public KeyVersion rollNewVersion(String name, byte[] material)
+ throws IOException {
+ return null;
+ }
+
+ @Override
+ public void flush() throws IOException {
+ }
+
+ }
+
+ public static class Factory extends KeyProviderFactory {
+
+ @Override
+ public KeyProvider createProvider(URI providerName, Configuration conf)
+ throws IOException {
+ if ("dummy".equals(providerName.getScheme())) {
+ return new DummyKeyProvider(conf);
+ }
+ return null;
+ }
+ }
+
+ @Test
+ public void testCache() throws Exception {
+ KeyProviderCache kpCache = new KeyProviderCache(10000);
+ Configuration conf = new Configuration();
+ conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ "dummy://foo:bar@test_provider1");
+ KeyProvider keyProvider1 = kpCache.get(conf);
+ Assert.assertNotNull("Returned Key Provider is null !!", keyProvider1);
+
+ conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ "dummy://foo:bar@test_provider1");
+ KeyProvider keyProvider2 = kpCache.get(conf);
+
+ Assert.assertTrue("Different KeyProviders returned !!",
+ keyProvider1 == keyProvider2);
+
+ conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ "dummy://test_provider3");
+ KeyProvider keyProvider3 = kpCache.get(conf);
+
+ Assert.assertFalse("Same KeyProviders returned !!",
+ keyProvider1 == keyProvider3);
+
+ conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI,
+ "dummy://hello:there@test_provider1");
+ KeyProvider keyProvider4 = kpCache.get(conf);
+
+ Assert.assertFalse("Same KeyProviders returned !!",
+ keyProvider1 == keyProvider4);
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
index 1338133..9720dd6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestReservedRawPaths.java
@@ -81,8 +81,8 @@ public class TestReservedRawPaths {
dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
// Need to set the client's KeyProvider to the NN's for JKS,
// else the updates do not get flushed properly
- fs.getClient().provider = cluster.getNameNode().getNamesystem()
- .getProvider();
+ fs.getClient().setKeyProvider(cluster.getNameNode().getNamesystem()
+ .getProvider());
DFSTestUtil.createKey(TEST_KEY, cluster, conf);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/02340a24/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
new file mode 100644
index 0000000..f7b0912
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/META-INF/services/org.apache.hadoop.crypto.key.KeyProviderFactory
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+org.apache.hadoop.hdfs.TestKeyProviderCache$Factory
\ No newline at end of file
[21/50] [abbrv] hadoop git commit: YARN-2809. Implement workaround
for linux kernel panic when removing cgroup. Contributed by Nathan Roberts
Posted by zj...@apache.org.
YARN-2809. Implement workaround for linux kernel panic when removing cgroup. Contributed by Nathan Roberts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3f5431a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3f5431a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3f5431a2
Branch: refs/heads/YARN-2928
Commit: 3f5431a22fcef7e3eb9aceeefe324e5b7ac84049
Parents: 4eb5f7f
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Feb 10 17:27:21 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Feb 10 17:27:21 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 11 ++-
.../util/CgroupsLCEResourcesHandler.java | 73 +++++++++++++++---
.../util/TestCgroupsLCEResourcesHandler.java | 80 +++++++++++++++-----
4 files changed, 136 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f5431a2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5a3a505..ee58b24 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -530,6 +530,9 @@ Release 2.7.0 - UNRELEASED
YARN-3090. DeletionService can silently ignore deletion task failures
(Varun Saxena via jlowe)
+ YARN-2809. Implement workaround for linux kernel panic when removing
+ cgroup (Nathan Roberts via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f5431a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 6904543..05c6cbf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1002,7 +1002,16 @@ public class YarnConfiguration extends Configuration {
public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT =
1000;
-
+
+ /**
+ * Delay between attempts to remove linux cgroup.
+ */
+ public static final String NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY =
+ NM_PREFIX + "linux-container-executor.cgroups.delete-delay-ms";
+
+ public static final long DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY =
+ 20;
+
/**
/* The Windows group that the windows-secure-container-executor should run as.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f5431a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index a832a7a..ffa17ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -22,6 +22,7 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
+import java.io.FileReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
@@ -37,6 +38,7 @@ import java.util.regex.Matcher;
import java.util.regex.Pattern;
import com.google.common.annotations.VisibleForTesting;
+
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -75,6 +77,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
private final Map<String, String> controllerPaths; // Controller -> path
private long deleteCgroupTimeout;
+ private long deleteCgroupDelay;
// package private for testing purposes
Clock clock;
@@ -108,6 +111,9 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
this.deleteCgroupTimeout = conf.getLong(
YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT,
YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_TIMEOUT);
+ this.deleteCgroupDelay =
+ conf.getLong(YarnConfiguration.NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY,
+ YarnConfiguration.DEFAULT_NM_LINUX_CONTAINER_CGROUPS_DELETE_DELAY);
// remove extra /'s at end or start of cgroupPrefix
if (cgroupPrefix.charAt(0) == '/') {
cgroupPrefix = cgroupPrefix.substring(1);
@@ -271,23 +277,71 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
}
}
+ /*
+ * Utility routine to print first line from cgroup tasks file
+ */
+ private void logLineFromTasksFile(File cgf) {
+ String str;
+ if (LOG.isDebugEnabled()) {
+ try (BufferedReader inl =
+ new BufferedReader(new InputStreamReader(new FileInputStream(cgf
+ + "/tasks"), "UTF-8"))) {
+ if ((str = inl.readLine()) != null) {
+ LOG.debug("First line in cgroup tasks file: " + cgf + " " + str);
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to read cgroup tasks file. ", e);
+ }
+ }
+ }
+
+ /**
+ * If tasks file is empty, delete the cgroup.
+ *
+ * @param file object referring to the cgroup to be deleted
+ * @return Boolean indicating whether cgroup was deleted
+ */
+ @VisibleForTesting
+ boolean checkAndDeleteCgroup(File cgf) throws InterruptedException {
+ boolean deleted = false;
+ // FileInputStream in = null;
+ try (FileInputStream in = new FileInputStream(cgf + "/tasks")) {
+ if (in.read() == -1) {
+ /*
+ * "tasks" file is empty, sleep a bit more and then try to delete the
+ * cgroup. Some versions of linux will occasionally panic due to a race
+ * condition in this area, hence the paranoia.
+ */
+ Thread.sleep(deleteCgroupDelay);
+ deleted = cgf.delete();
+ if (!deleted) {
+ LOG.warn("Failed attempt to delete cgroup: " + cgf);
+ }
+ } else {
+ logLineFromTasksFile(cgf);
+ }
+ } catch (IOException e) {
+ LOG.warn("Failed to read cgroup tasks file. ", e);
+ }
+ return deleted;
+ }
+
@VisibleForTesting
boolean deleteCgroup(String cgroupPath) {
- boolean deleted;
-
+ boolean deleted = false;
+
if (LOG.isDebugEnabled()) {
LOG.debug("deleteCgroup: " + cgroupPath);
}
-
long start = clock.getTime();
do {
- deleted = new File(cgroupPath).delete();
- if (!deleted) {
- try {
- Thread.sleep(20);
- } catch (InterruptedException ex) {
- // NOP
+ try {
+ deleted = checkAndDeleteCgroup(new File(cgroupPath));
+ if (!deleted) {
+ Thread.sleep(deleteCgroupDelay);
}
+ } catch (InterruptedException ex) {
+ // NOP
}
} while (!deleted && (clock.getTime() - start) < deleteCgroupTimeout);
@@ -295,7 +349,6 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
LOG.warn("Unable to delete cgroup at: " + cgroupPath +
", tried to delete for " + deleteCgroupTimeout + "ms");
}
-
return deleted;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3f5431a2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
index d0bceee..4e35169 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/util/TestCgroupsLCEResourcesHandler.java
@@ -26,6 +26,8 @@ import org.junit.Assert;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Clock;
import org.junit.Test;
+import org.junit.After;
+import org.junit.Before;
import org.mockito.Mockito;
import java.io.*;
@@ -35,6 +37,7 @@ import java.util.UUID;
import java.util.concurrent.CountDownLatch;
public class TestCgroupsLCEResourcesHandler {
+ static File cgroupDir = null;
static class MockClock implements Clock {
long time;
@@ -43,6 +46,51 @@ public class TestCgroupsLCEResourcesHandler {
return time;
}
}
+
+ @Before
+ public void setUp() throws Exception {
+ cgroupDir =
+ new File(System.getProperty("test.build.data",
+ System.getProperty("java.io.tmpdir", "target")), this.getClass()
+ .getName());
+ FileUtils.deleteQuietly(cgroupDir);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ FileUtils.deleteQuietly(cgroupDir);
+ }
+
+ @Test
+ public void testcheckAndDeleteCgroup() throws Exception {
+ CgroupsLCEResourcesHandler handler = new CgroupsLCEResourcesHandler();
+ handler.setConf(new YarnConfiguration());
+ handler.initConfig();
+
+ FileUtils.deleteQuietly(cgroupDir);
+ // Test 0
+ // tasks file not present, should return false
+ Assert.assertFalse(handler.checkAndDeleteCgroup(cgroupDir));
+
+ File tfile = new File(cgroupDir.getAbsolutePath(), "tasks");
+ FileOutputStream fos = FileUtils.openOutputStream(tfile);
+ File fspy = Mockito.spy(cgroupDir);
+
+ // Test 1, tasks file is empty
+ // tasks file has no data, should return true
+ Mockito.stub(fspy.delete()).toReturn(true);
+ Assert.assertTrue(handler.checkAndDeleteCgroup(fspy));
+
+ // Test 2, tasks file has data
+ fos.write("1234".getBytes());
+ fos.close();
+ // tasks has data, would not be able to delete, should return false
+ Assert.assertFalse(handler.checkAndDeleteCgroup(fspy));
+ FileUtils.deleteQuietly(cgroupDir);
+
+ }
+
+ // Verify DeleteCgroup times out if "tasks" file contains data
@Test
public void testDeleteCgroup() throws Exception {
final MockClock clock = new MockClock();
@@ -51,13 +99,15 @@ public class TestCgroupsLCEResourcesHandler {
handler.setConf(new YarnConfiguration());
handler.initConfig();
handler.clock = clock;
-
- //file exists
- File file = new File("target", UUID.randomUUID().toString());
- new FileOutputStream(file).close();
- Assert.assertTrue(handler.deleteCgroup(file.getPath()));
- //file does not exists, timing out
+ FileUtils.deleteQuietly(cgroupDir);
+
+ // Create a non-empty tasks file
+ File tfile = new File(cgroupDir.getAbsolutePath(), "tasks");
+ FileOutputStream fos = FileUtils.openOutputStream(tfile);
+ fos.write("1234".getBytes());
+ fos.close();
+
final CountDownLatch latch = new CountDownLatch(1);
new Thread() {
@Override
@@ -73,8 +123,8 @@ public class TestCgroupsLCEResourcesHandler {
}
}.start();
latch.await();
- file = new File("target", UUID.randomUUID().toString());
- Assert.assertFalse(handler.deleteCgroup(file.getPath()));
+ Assert.assertFalse(handler.deleteCgroup(cgroupDir.getAbsolutePath()));
+ FileUtils.deleteQuietly(cgroupDir);
}
static class MockLinuxContainerExecutor extends LinuxContainerExecutor {
@@ -122,7 +172,6 @@ public class TestCgroupsLCEResourcesHandler {
handler.initConfig();
// create mock cgroup
- File cgroupDir = createMockCgroup();
File cgroupMountDir = createMockCgroupMount(cgroupDir);
// create mock mtab
@@ -202,18 +251,10 @@ public class TestCgroupsLCEResourcesHandler {
Assert.assertEquals(-1, ret[1]);
}
- private File createMockCgroup() throws IOException {
- File cgroupDir = new File("target", UUID.randomUUID().toString());
- if (!cgroupDir.mkdir()) {
- String message = "Could not create dir " + cgroupDir.getAbsolutePath();
- throw new IOException(message);
- }
- return cgroupDir;
- }
-
private File createMockCgroupMount(File cgroupDir) throws IOException {
File cgroupMountDir = new File(cgroupDir.getAbsolutePath(), "hadoop-yarn");
- if (!cgroupMountDir.mkdir()) {
+ FileUtils.deleteQuietly(cgroupDir);
+ if (!cgroupMountDir.mkdirs()) {
String message =
"Could not create dir " + cgroupMountDir.getAbsolutePath();
throw new IOException(message);
@@ -253,7 +294,6 @@ public class TestCgroupsLCEResourcesHandler {
handler.initConfig();
// create mock cgroup
- File cgroupDir = createMockCgroup();
File cgroupMountDir = createMockCgroupMount(cgroupDir);
// create mock mtab
[22/50] [abbrv] hadoop git commit: HADOOP-10953. NetworkTopology#add
calls NetworkTopology#toString without holding the netlock (Liang Xie via
Colin P. McCabe)
Posted by zj...@apache.org.
HADOOP-10953. NetworkTopology#add calls NetworkTopology#toString without holding the netlock (Liang Xie via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6338ce3a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6338ce3a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6338ce3a
Branch: refs/heads/YARN-2928
Commit: 6338ce3ae8870548cac5abe2f685748b5efb13c1
Parents: 3f5431a
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Feb 10 13:24:40 2015 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Feb 10 13:24:40 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/net/NetworkTopology.java | 10 +++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6338ce3a/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 8b80998..fadc744 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -889,6 +889,9 @@ Release 2.7.0 - UNRELEASED
HADOOP-11535 TableMapping related tests failed due to 'successful'
resolving of invalid test hostname. (Kai Zheng via stevel)
+ HADOOP-10953. NetworkTopology#add calls NetworkTopology#toString without
+ holding the netlock (Liang Xie via Colin P. McCabe)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6338ce3a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index fc8bf52..c60cc0b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -393,14 +393,14 @@ public class NetworkTopology {
*/
public void add(Node node) {
if (node==null) return;
- String oldTopoStr = this.toString();
- if( node instanceof InnerNode ) {
- throw new IllegalArgumentException(
- "Not allow to add an inner node: "+NodeBase.getPath(node));
- }
int newDepth = NodeBase.locationToDepth(node.getNetworkLocation()) + 1;
netlock.writeLock().lock();
try {
+ String oldTopoStr = this.toString();
+ if( node instanceof InnerNode ) {
+ throw new IllegalArgumentException(
+ "Not allow to add an inner node: "+NodeBase.getPath(node));
+ }
if ((depthOfAllLeaves != -1) && (depthOfAllLeaves != newDepth)) {
LOG.error("Error: can't add leaf node " + NodeBase.getPath(node) +
" at depth " + newDepth + " to topology:\n" + oldTopoStr);
[24/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
new file mode 100644
index 0000000..ae3bea8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/FileSystemShell.md
@@ -0,0 +1,689 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Overview](#Overview)
+ * [appendToFile](#appendToFile)
+ * [cat](#cat)
+ * [checksum](#checksum)
+ * [chgrp](#chgrp)
+ * [chmod](#chmod)
+ * [chown](#chown)
+ * [copyFromLocal](#copyFromLocal)
+ * [copyToLocal](#copyToLocal)
+ * [count](#count)
+ * [cp](#cp)
+ * [createSnapshot](#createSnapshot)
+ * [deleteSnapshot](#deleteSnapshot)
+ * [df](#df)
+ * [du](#du)
+ * [dus](#dus)
+ * [expunge](#expunge)
+ * [find](#find)
+ * [get](#get)
+ * [getfacl](#getfacl)
+ * [getfattr](#getfattr)
+ * [getmerge](#getmerge)
+ * [help](#help)
+ * [ls](#ls)
+ * [lsr](#lsr)
+ * [mkdir](#mkdir)
+ * [moveFromLocal](#moveFromLocal)
+ * [moveToLocal](#moveToLocal)
+ * [mv](#mv)
+ * [put](#put)
+ * [renameSnapshot](#renameSnapshot)
+ * [rm](#rm)
+ * [rmdir](#rmdir)
+ * [rmr](#rmr)
+ * [setfacl](#setfacl)
+ * [setfattr](#setfattr)
+ * [setrep](#setrep)
+ * [stat](#stat)
+ * [tail](#tail)
+ * [test](#test)
+ * [text](#text)
+ * [touchz](#touchz)
+ * [usage](#usage)
+
+Overview
+========
+
+The File System (FS) shell includes various shell-like commands that directly interact with the Hadoop Distributed File System (HDFS) as well as other file systems that Hadoop supports, such as Local FS, HFTP FS, S3 FS, and others. The FS shell is invoked by:
+
+ bin/hadoop fs <args>
+
+All FS shell commands take path URIs as arguments. The URI format is `scheme://authority/path`. For HDFS the scheme is `hdfs`, and for the Local FS the scheme is `file`. The scheme and authority are optional. If not specified, the default scheme specified in the configuration is used. An HDFS file or directory such as /parent/child can be specified as `hdfs://namenodehost/parent/child` or simply as `/parent/child` (given that your configuration is set to point to `hdfs://namenodehost`).
+
+Most of the commands in FS shell behave like corresponding Unix commands. Differences are described with each of the commands. Error information is sent to stderr and the output is sent to stdout.
+
+If HDFS is being used, `hdfs dfs` is a synonym.
+
+See the [Commands Manual](./CommandsManual.html) for generic shell options.
+
+appendToFile
+------------
+
+Usage: `hadoop fs -appendToFile <localsrc> ... <dst> `
+
+Append single src, or multiple srcs from local file system to the destination file system. Also reads input from stdin and appends to destination file system.
+
+* `hadoop fs -appendToFile localfile /user/hadoop/hadoopfile`
+* `hadoop fs -appendToFile localfile1 localfile2 /user/hadoop/hadoopfile`
+* `hadoop fs -appendToFile localfile hdfs://nn.example.com/hadoop/hadoopfile`
+* `hadoop fs -appendToFile - hdfs://nn.example.com/hadoop/hadoopfile` Reads the input from stdin.
+
+Exit Code:
+
+Returns 0 on success and 1 on error.
+
+cat
+---
+
+Usage: `hadoop fs -cat URI [URI ...]`
+
+Copies source paths to stdout.
+
+Example:
+
+* `hadoop fs -cat hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
+* `hadoop fs -cat file:///file3 /user/hadoop/file4`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+checksum
+--------
+
+Usage: `hadoop fs -checksum URI`
+
+Returns the checksum information of a file.
+
+Example:
+
+* `hadoop fs -checksum hdfs://nn1.example.com/file1`
+* `hadoop fs -checksum file:///etc/hosts`
+
+chgrp
+-----
+
+Usage: `hadoop fs -chgrp [-R] GROUP URI [URI ...]`
+
+Change group association of files. The user must be the owner of files, or else a super-user. Additional information is in the [Permissions Guide](../hadoop-hdfs/HdfsPermissionsGuide.html).
+
+Options
+
+* The -R option will make the change recursively through the directory structure.
+
+chmod
+-----
+
+Usage: `hadoop fs -chmod [-R] <MODE[,MODE]... | OCTALMODE> URI [URI ...]`
+
+Change the permissions of files. With -R, make the change recursively through the directory structure. The user must be the owner of the file, or else a super-user. Additional information is in the [Permissions Guide](../hadoop-hdfs/HdfsPermissionsGuide.html).
+
+Options
+
+* The -R option will make the change recursively through the directory structure.
+
+chown
+-----
+
+Usage: `hadoop fs -chown [-R] [OWNER][:[GROUP]] URI [URI ]`
+
+Change the owner of files. The user must be a super-user. Additional information is in the [Permissions Guide](../hadoop-hdfs/HdfsPermissionsGuide.html).
+
+Options
+
+* The -R option will make the change recursively through the directory structure.
+
+copyFromLocal
+-------------
+
+Usage: `hadoop fs -copyFromLocal <localsrc> URI`
+
+Similar to put command, except that the source is restricted to a local file reference.
+
+Options:
+
+* The -f option will overwrite the destination if it already exists.
+
+copyToLocal
+-----------
+
+Usage: `hadoop fs -copyToLocal [-ignorecrc] [-crc] URI <localdst> `
+
+Similar to get command, except that the destination is restricted to a local file reference.
+
+count
+-----
+
+Usage: `hadoop fs -count [-q] [-h] [-v] <paths> `
+
+Count the number of directories, files and bytes under the paths that match the specified file pattern. The output columns with -count are: DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+
+The output columns with -count -q are: QUOTA, REMAINING\_QUATA, SPACE\_QUOTA, REMAINING\_SPACE\_QUOTA, DIR\_COUNT, FILE\_COUNT, CONTENT\_SIZE, PATHNAME
+
+The -h option shows sizes in human readable format.
+
+The -v option displays a header line.
+
+Example:
+
+* `hadoop fs -count hdfs://nn1.example.com/file1 hdfs://nn2.example.com/file2`
+* `hadoop fs -count -q hdfs://nn1.example.com/file1`
+* `hadoop fs -count -q -h hdfs://nn1.example.com/file1`
+* `hdfs dfs -count -q -h -v hdfs://nn1.example.com/file1`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+cp
+----
+
+Usage: `hadoop fs -cp [-f] [-p | -p[topax]] URI [URI ...] <dest> `
+
+Copy files from source to destination. This command allows multiple sources as well in which case the destination must be a directory.
+
+'raw.\*' namespace extended attributes are preserved if (1) the source and destination filesystems support them (HDFS only), and (2) all source and destination pathnames are in the /.reserved/raw hierarchy. Determination of whether raw.\* namespace xattrs are preserved is independent of the -p (preserve) flag.
+
+Options:
+
+* The -f option will overwrite the destination if it already exists.
+* The -p option will preserve file attributes [topx] (timestamps, ownership, permission, ACL, XAttr). If -p is specified with no *arg*, then preserves timestamps, ownership, permission. If -pa is specified, then preserves permission also because ACL is a super-set of permission. Determination of whether raw namespace extended attributes are preserved is independent of the -p flag.
+
+Example:
+
+* `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2`
+* `hadoop fs -cp /user/hadoop/file1 /user/hadoop/file2 /user/hadoop/dir`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+createSnapshot
+--------------
+
+See [HDFS Snapshots Guide](../hadoop-hdfs/HdfsSnapshots.html).
+
+deleteSnapshot
+--------------
+
+See [HDFS Snapshots Guide](../hadoop-hdfs/HdfsSnapshots.html).
+
+df
+----
+
+Usage: `hadoop fs -df [-h] URI [URI ...]`
+
+Displays free space.
+
+Options:
+
+* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+
+Example:
+
+* `hadoop dfs -df /user/hadoop/dir1`
+
+du
+----
+
+Usage: `hadoop fs -du [-s] [-h] URI [URI ...]`
+
+Displays sizes of files and directories contained in the given directory or the length of a file in case its just a file.
+
+Options:
+
+* The -s option will result in an aggregate summary of file lengths being displayed, rather than the individual files.
+* The -h option will format file sizes in a "human-readable" fashion (e.g 64.0m instead of 67108864)
+
+Example:
+
+* `hadoop fs -du /user/hadoop/dir1 /user/hadoop/file1 hdfs://nn.example.com/user/hadoop/dir1`
+
+Exit Code: Returns 0 on success and -1 on error.
+
+dus
+---
+
+Usage: `hadoop fs -dus <args> `
+
+Displays a summary of file lengths.
+
+**Note:** This command is deprecated. Instead use `hadoop fs -du -s`.
+
+expunge
+-------
+
+Usage: `hadoop fs -expunge`
+
+Empty the Trash. Refer to the [HDFS Architecture Guide](../hadoop-hdfs/HdfsDesign.html) for more information on the Trash feature.
+
+find
+----
+
+Usage: `hadoop fs -find <path> ... <expression> ... `
+
+Finds all files that match the specified expression and applies selected actions to them. If no *path* is specified then defaults to the current working directory. If no expression is specified then defaults to -print.
+
+The following primary expressions are recognised:
+
+* -name pattern<br />-iname pattern
+
+ Evaluates as true if the basename of the file matches the pattern using standard file system globbing. If -iname is used then the match is case insensitive.
+
+* -print<br />-print0Always
+
+ evaluates to true. Causes the current pathname to be written to standard output. If the -print0 expression is used then an ASCII NULL character is appended.
+
+The following operators are recognised:
+
+* expression -a expression<br />expression -and expression<br />expression expression
+
+ Logical AND operator for joining two expressions. Returns true if both child expressions return true. Implied by the juxtaposition of two expressions and so does not need to be explicitly specified. The second expression will not be applied if the first fails.
+
+Example:
+
+`hadoop fs -find / -name test -print`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+get
+---
+
+Usage: `hadoop fs -get [-ignorecrc] [-crc] <src> <localdst> `
+
+Copy files to the local file system. Files that fail the CRC check may be copied with the -ignorecrc option. Files and CRCs may be copied using the -crc option.
+
+Example:
+
+* `hadoop fs -get /user/hadoop/file localfile`
+* `hadoop fs -get hdfs://nn.example.com/user/hadoop/file localfile`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+getfacl
+-------
+
+Usage: `hadoop fs -getfacl [-R] <path> `
+
+Displays the Access Control Lists (ACLs) of files and directories. If a directory has a default ACL, then getfacl also displays the default ACL.
+
+Options:
+
+* -R: List the ACLs of all files and directories recursively.
+* *path*: File or directory to list.
+
+Examples:
+
+* `hadoop fs -getfacl /file`
+* `hadoop fs -getfacl -R /dir`
+
+Exit Code:
+
+Returns 0 on success and non-zero on error.
+
+getfattr
+--------
+
+Usage: `hadoop fs -getfattr [-R] -n name | -d [-e en] <path> `
+
+Displays the extended attribute names and values (if any) for a file or directory.
+
+Options:
+
+* -R: Recursively list the attributes for all files and directories.
+* -n name: Dump the named extended attribute value.
+* -d: Dump all extended attribute values associated with pathname.
+* -e *encoding*: Encode values after retrieving them. Valid encodings are "text", "hex", and "base64". Values encoded as text strings are enclosed in double quotes ("), and values encoded as hexadecimal and base64 are prefixed with 0x and 0s, respectively.
+* *path*: The file or directory.
+
+Examples:
+
+* `hadoop fs -getfattr -d /file`
+* `hadoop fs -getfattr -R -n user.myAttr /dir`
+
+Exit Code:
+
+Returns 0 on success and non-zero on error.
+
+getmerge
+--------
+
+Usage: `hadoop fs -getmerge <src> <localdst> [addnl]`
+
+Takes a source directory and a destination file as input and concatenates files in src into the destination local file. Optionally addnl can be set to enable adding a newline character at the end of each file.
+
+help
+----
+
+Usage: `hadoop fs -help`
+
+Return usage output.
+
+ls
+----
+
+Usage: `hadoop fs -ls [-d] [-h] [-R] [-t] [-S] [-r] [-u] <args> `
+
+Options:
+
+* -d: Directories are listed as plain files.
+* -h: Format file sizes in a human-readable fashion (eg 64.0m instead of 67108864).
+* -R: Recursively list subdirectories encountered.
+* -t: Sort output by modification time (most recent first).
+* -S: Sort output by file size.
+* -r: Reverse the sort order.
+* -u: Use access time rather than modification time for display and sorting.
+
+For a file ls returns stat on the file with the following format:
+
+ permissions number_of_replicas userid groupid filesize modification_date modification_time filename
+
+For a directory it returns list of its direct children as in Unix. A directory is listed as:
+
+ permissions userid groupid modification_date modification_time dirname
+
+Files within a directory are order by filename by default.
+
+Example:
+
+* `hadoop fs -ls /user/hadoop/file1`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+lsr
+---
+
+Usage: `hadoop fs -lsr <args> `
+
+Recursive version of ls.
+
+**Note:** This command is deprecated. Instead use `hadoop fs -ls -R`
+
+mkdir
+-----
+
+Usage: `hadoop fs -mkdir [-p] <paths> `
+
+Takes path uri's as argument and creates directories.
+
+Options:
+
+* The -p option behavior is much like Unix mkdir -p, creating parent directories along the path.
+
+Example:
+
+* `hadoop fs -mkdir /user/hadoop/dir1 /user/hadoop/dir2`
+* `hadoop fs -mkdir hdfs://nn1.example.com/user/hadoop/dir hdfs://nn2.example.com/user/hadoop/dir`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+moveFromLocal
+-------------
+
+Usage: `hadoop fs -moveFromLocal <localsrc> <dst> `
+
+Similar to put command, except that the source localsrc is deleted after it's copied.
+
+moveToLocal
+-----------
+
+Usage: `hadoop fs -moveToLocal [-crc] <src> <dst> `
+
+Displays a "Not implemented yet" message.
+
+mv
+----
+
+Usage: `hadoop fs -mv URI [URI ...] <dest> `
+
+Moves files from source to destination. This command allows multiple sources as well in which case the destination needs to be a directory. Moving files across file systems is not permitted.
+
+Example:
+
+* `hadoop fs -mv /user/hadoop/file1 /user/hadoop/file2`
+* `hadoop fs -mv hdfs://nn.example.com/file1 hdfs://nn.example.com/file2 hdfs://nn.example.com/file3 hdfs://nn.example.com/dir1`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+put
+---
+
+Usage: `hadoop fs -put <localsrc> ... <dst> `
+
+Copy single src, or multiple srcs from local file system to the destination file system. Also reads input from stdin and writes to destination file system.
+
+* `hadoop fs -put localfile /user/hadoop/hadoopfile`
+* `hadoop fs -put localfile1 localfile2 /user/hadoop/hadoopdir`
+* `hadoop fs -put localfile hdfs://nn.example.com/hadoop/hadoopfile`
+* `hadoop fs -put - hdfs://nn.example.com/hadoop/hadoopfile` Reads the input from stdin.
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+renameSnapshot
+--------------
+
+See [HDFS Snapshots Guide](../hadoop-hdfs/HdfsSnapshots.html).
+
+rm
+----
+
+Usage: `hadoop fs -rm [-f] [-r |-R] [-skipTrash] URI [URI ...]`
+
+Delete files specified as args.
+
+Options:
+
+* The -f option will not display a diagnostic message or modify the exit status to reflect an error if the file does not exist.
+* The -R option deletes the directory and any content under it recursively.
+* The -r option is equivalent to -R.
+* The -skipTrash option will bypass trash, if enabled, and delete the specified file(s) immediately. This can be useful when it is necessary to delete files from an over-quota directory.
+
+Example:
+
+* `hadoop fs -rm hdfs://nn.example.com/file /user/hadoop/emptydir`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+rmdir
+-----
+
+Usage: `hadoop fs -rmdir [--ignore-fail-on-non-empty] URI [URI ...]`
+
+Delete a directory.
+
+Options:
+
+* `--ignore-fail-on-non-empty`: When using wildcards, do not fail if a directory still contains files.
+
+Example:
+
+* `hadoop fs -rmdir /user/hadoop/emptydir`
+
+rmr
+---
+
+Usage: `hadoop fs -rmr [-skipTrash] URI [URI ...]`
+
+Recursive version of delete.
+
+**Note:** This command is deprecated. Instead use `hadoop fs -rm -r`
+
+setfacl
+-------
+
+Usage: `hadoop fs -setfacl [-R] [-b |-k -m |-x <acl_spec> <path>] |[--set <acl_spec> <path>] `
+
+Sets Access Control Lists (ACLs) of files and directories.
+
+Options:
+
+* -b: Remove all but the base ACL entries. The entries for user, group and others are retained for compatibility with permission bits.
+* -k: Remove the default ACL.
+* -R: Apply operations to all files and directories recursively.
+* -m: Modify ACL. New entries are added to the ACL, and existing entries are retained.
+* -x: Remove specified ACL entries. Other ACL entries are retained.
+* ``--set``: Fully replace the ACL, discarding all existing entries. The *acl\_spec* must include entries for user, group, and others for compatibility with permission bits.
+* *acl\_spec*: Comma separated list of ACL entries.
+* *path*: File or directory to modify.
+
+Examples:
+
+* `hadoop fs -setfacl -m user:hadoop:rw- /file`
+* `hadoop fs -setfacl -x user:hadoop /file`
+* `hadoop fs -setfacl -b /file`
+* `hadoop fs -setfacl -k /dir`
+* `hadoop fs -setfacl --set user::rw-,user:hadoop:rw-,group::r--,other::r-- /file`
+* `hadoop fs -setfacl -R -m user:hadoop:r-x /dir`
+* `hadoop fs -setfacl -m default:user:hadoop:r-x /dir`
+
+Exit Code:
+
+Returns 0 on success and non-zero on error.
+
+setfattr
+--------
+
+Usage: `hadoop fs -setfattr -n name [-v value] | -x name <path> `
+
+Sets an extended attribute name and value for a file or directory.
+
+Options:
+
+* -b: Remove all but the base ACL entries. The entries for user, group and others are retained for compatibility with permission bits.
+* -n name: The extended attribute name.
+* -v value: The extended attribute value. There are three different encoding methods for the value. If the argument is enclosed in double quotes, then the value is the string inside the quotes. If the argument is prefixed with 0x or 0X, then it is taken as a hexadecimal number. If the argument begins with 0s or 0S, then it is taken as a base64 encoding.
+* -x name: Remove the extended attribute.
+* *path*: The file or directory.
+
+Examples:
+
+* `hadoop fs -setfattr -n user.myAttr -v myValue /file`
+* `hadoop fs -setfattr -n user.noValue /file`
+* `hadoop fs -setfattr -x user.myAttr /file`
+
+Exit Code:
+
+Returns 0 on success and non-zero on error.
+
+setrep
+------
+
+Usage: `hadoop fs -setrep [-R] [-w] <numReplicas> <path> `
+
+Changes the replication factor of a file. If *path* is a directory then the command recursively changes the replication factor of all files under the directory tree rooted at *path*.
+
+Options:
+
+* The -w flag requests that the command wait for the replication to complete. This can potentially take a very long time.
+* The -R flag is accepted for backwards compatibility. It has no effect.
+
+Example:
+
+* `hadoop fs -setrep -w 3 /user/hadoop/dir1`
+
+Exit Code:
+
+Returns 0 on success and -1 on error.
+
+stat
+----
+
+Usage: `hadoop fs -stat [format] <path> ...`
+
+Print statistics about the file/directory at \<path\> in the specified format. Format accepts filesize in blocks (%b), type (%F), group name of owner (%g), name (%n), block size (%o), replication (%r), user name of owner(%u), and modification date (%y, %Y). %y shows UTC date as "yyyy-MM-dd HH:mm:ss" and %Y shows milliseconds since January 1, 1970 UTC. If the format is not specified, %y is used by default.
+
+Example:
+
+* `hadoop fs -stat "%F %u:%g %b %y %n" /file`
+
+Exit Code: Returns 0 on success and -1 on error.
+
+tail
+----
+
+Usage: `hadoop fs -tail [-f] URI`
+
+Displays last kilobyte of the file to stdout.
+
+Options:
+
+* The -f option will output appended data as the file grows, as in Unix.
+
+Example:
+
+* `hadoop fs -tail pathname`
+
+Exit Code: Returns 0 on success and -1 on error.
+
+test
+----
+
+Usage: `hadoop fs -test -[defsz] URI`
+
+Options:
+
+* -d: f the path is a directory, return 0.
+* -e: if the path exists, return 0.
+* -f: if the path is a file, return 0.
+* -s: if the path is not empty, return 0.
+* -z: if the file is zero length, return 0.
+
+Example:
+
+* `hadoop fs -test -e filename`
+
+text
+----
+
+Usage: `hadoop fs -text <src> `
+
+Takes a source file and outputs the file in text format. The allowed formats are zip and TextRecordInputStream.
+
+touchz
+------
+
+Usage: `hadoop fs -touchz URI [URI ...]`
+
+Create a file of zero length.
+
+Example:
+
+* `hadoop fs -touchz pathname`
+
+Exit Code: Returns 0 on success and -1 on error.
+
+usage
+-----
+
+Usage: `hadoop fs -usage command`
+
+Return the help for an individual command.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
new file mode 100644
index 0000000..e0a2693
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/HttpAuthentication.md
@@ -0,0 +1,58 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Authentication for Hadoop HTTP web-consoles
+===========================================
+
+* [Authentication for Hadoop HTTP web-consoles](#Authentication_for_Hadoop_HTTP_web-consoles)
+ * [Introduction](#Introduction)
+ * [Configuration](#Configuration)
+
+Introduction
+------------
+
+This document describes how to configure Hadoop HTTP web-consoles to require user authentication.
+
+By default Hadoop HTTP web-consoles (JobTracker, NameNode, TaskTrackers and DataNodes) allow access without any form of authentication.
+
+Similarly to Hadoop RPC, Hadoop HTTP web-consoles can be configured to require Kerberos authentication using HTTP SPNEGO protocol (supported by browsers like Firefox and Internet Explorer).
+
+In addition, Hadoop HTTP web-consoles support the equivalent of Hadoop's Pseudo/Simple authentication. If this option is enabled, user must specify their user name in the first browser interaction using the user.name query string parameter. For example: `http://localhost:50030/jobtracker.jsp?user.name=babu`.
+
+If a custom authentication mechanism is required for the HTTP web-consoles, it is possible to implement a plugin to support the alternate authentication mechanism (refer to Hadoop hadoop-auth for details on writing an `AuthenticatorHandler`).
+
+The next section describes how to configure Hadoop HTTP web-consoles to require user authentication.
+
+Configuration
+-------------
+
+The following properties should be in the `core-site.xml` of all the nodes in the cluster.
+
+`hadoop.http.filter.initializers`: add to this property the `org.apache.hadoop.security.AuthenticationFilterInitializer` initializer class.
+
+`hadoop.http.authentication.type`: Defines authentication used for the HTTP web-consoles. The supported values are: `simple` | `kerberos` | `#AUTHENTICATION_HANDLER_CLASSNAME#`. The dfeault value is `simple`.
+
+`hadoop.http.authentication.token.validity`: Indicates how long (in seconds) an authentication token is valid before it has to be renewed. The default value is `36000`.
+
+`hadoop.http.authentication.signature.secret.file`: The signature secret file for signing the authentication tokens. The same secret should be used for all nodes in the cluster, JobTracker, NameNode, DataNode and TastTracker. The default value is `$user.home/hadoop-http-auth-signature-secret`. IMPORTANT: This file should be readable only by the Unix user running the daemons.
+
+`hadoop.http.authentication.cookie.domain`: The domain to use for the HTTP cookie that stores the authentication token. In order to authentiation to work correctly across all nodes in the cluster the domain must be correctly set. There is no default value, the HTTP cookie will not have a domain working only with the hostname issuing the HTTP cookie.
+
+IMPORTANT: when using IP addresses, browsers ignore cookies with domain settings. For this setting to work properly all nodes in the cluster must be configured to generate URLs with `hostname.domain` names on it.
+
+`hadoop.http.authentication.simple.anonymous.allowed`: Indicates if anonymous requests are allowed when using 'simple' authentication. The default value is `true`
+
+`hadoop.http.authentication.kerberos.principal`: Indicates the Kerberos principal to be used for HTTP endpoint when using 'kerberos' authentication. The principal short name must be `HTTP` per Kerberos HTTP SPNEGO specification. The default value is `HTTP/_HOST@$LOCALHOST`, where `_HOST` -if present- is replaced with bind address of the HTTP server.
+
+`hadoop.http.authentication.kerberos.keytab`: Location of the keytab file with the credentials for the Kerberos principal used for the HTTP endpoint. The default value is `$user.home/hadoop.keytab`.i
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md b/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
new file mode 100644
index 0000000..0392610
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/InterfaceClassification.md
@@ -0,0 +1,105 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Hadoop Interface Taxonomy: Audience and Stability Classification
+================================================================
+
+* [Hadoop Interface Taxonomy: Audience and Stability Classification](#Hadoop_Interface_Taxonomy:_Audience_and_Stability_Classification)
+ * [Motivation](#Motivation)
+ * [Interface Classification](#Interface_Classification)
+ * [Audience](#Audience)
+ * [Stability](#Stability)
+ * [How are the Classifications Recorded?](#How_are_the_Classifications_Recorded)
+ * [FAQ](#FAQ)
+
+Motivation
+----------
+
+The interface taxonomy classification provided here is for guidance to developers and users of interfaces. The classification guides a developer to declare the targeted audience or users of an interface and also its stability.
+
+* Benefits to the user of an interface: Knows which interfaces to use or not use and their stability.
+* Benefits to the developer: to prevent accidental changes of interfaces and hence accidental impact on users or other components or system. This is particularly useful in large systems with many developers who may not all have a shared state/history of the project.
+
+Interface Classification
+------------------------
+
+Hadoop adopts the following interface classification, this classification was derived from the [OpenSolaris taxonomy](http://www.opensolaris.org/os/community/arc/policies/interface-taxonomy/#Advice) and, to some extent, from taxonomy used inside Yahoo. Interfaces have two main attributes: Audience and Stability
+
+### Audience
+
+Audience denotes the potential consumers of the interface. While many interfaces are internal/private to the implementation, other are public/external interfaces are meant for wider consumption by applications and/or clients. For example, in posix, libc is an external or public interface, while large parts of the kernel are internal or private interfaces. Also, some interfaces are targeted towards other specific subsystems.
+
+Identifying the audience of an interface helps define the impact of breaking it. For instance, it might be okay to break the compatibility of an interface whose audience is a small number of specific subsystems. On the other hand, it is probably not okay to break a protocol interfaces that millions of Internet users depend on.
+
+Hadoop uses the following kinds of audience in order of increasing/wider visibility:
+
+* Private:
+ * The interface is for internal use within the project (such as HDFS or MapReduce) and should not be used by applications or by other projects. It is subject to change at anytime without notice. Most interfaces of a project are Private (also referred to as project-private).
+* Limited-Private:
+ * The interface is used by a specified set of projects or systems (typically closely related projects). Other projects or systems should not use the interface. Changes to the interface will be communicated/ negotiated with the specified projects. For example, in the Hadoop project, some interfaces are LimitedPrivate{HDFS, MapReduce} in that they are private to the HDFS and MapReduce projects.
+* Public
+ * The interface is for general use by any application.
+
+Hadoop doesn't have a Company-Private classification, which is meant for APIs which are intended to be used by other projects within the company, since it doesn't apply to opensource projects. Also, certain APIs are annotated as @VisibleForTesting (from com.google.common .annotations.VisibleForTesting) - these are meant to be used strictly for unit tests and should be treated as "Private" APIs.
+
+### Stability
+
+Stability denotes how stable an interface is, as in when incompatible changes to the interface are allowed. Hadoop APIs have the following levels of stability.
+
+* Stable
+ * Can evolve while retaining compatibility for minor release boundaries; in other words, incompatible changes to APIs marked Stable are allowed only at major releases (i.e. at m.0).
+* Evolving
+ * Evolving, but incompatible changes are allowed at minor release (i.e. m .x)
+* Unstable
+ * Incompatible changes to Unstable APIs are allowed any time. This usually makes sense for only private interfaces.
+ * However one may call this out for a supposedly public interface to highlight that it should not be used as an interface; for public interfaces, labeling it as Not-an-interface is probably more appropriate than "Unstable".
+ * Examples of publicly visible interfaces that are unstable (i.e. not-an-interface): GUI, CLIs whose output format will change
+* Deprecated
+ * APIs that could potentially removed in the future and should not be used.
+
+How are the Classifications Recorded?
+-------------------------------------
+
+How will the classification be recorded for Hadoop APIs?
+
+* Each interface or class will have the audience and stability recorded using annotations in org.apache.hadoop.classification package.
+* The javadoc generated by the maven target javadoc:javadoc lists only the public API.
+* One can derive the audience of java classes and java interfaces by the audience of the package in which they are contained. Hence it is useful to declare the audience of each java package as public or private (along with the private audience variations).
+
+FAQ
+---
+
+* Why aren’t the java scopes (private, package private and public) good enough?
+ * Java’s scoping is not very complete. One is often forced to make a class public in order for other internal components to use it. It does not have friends or sub-package-private like C++.
+* But I can easily access a private implementation interface if it is Java public. Where is the protection and control?
+ * The purpose of this is not providing absolute access control. Its purpose is to communicate to users and developers. One can access private implementation functions in libc; however if they change the internal implementation details, your application will break and you will have little sympathy from the folks who are supplying libc. If you use a non-public interface you understand the risks.
+* Why bother declaring the stability of a private interface? Aren’t private interfaces always unstable?
+ * Private interfaces are not always unstable. In the cases where they are stable they capture internal properties of the system and can communicate these properties to its internal users and to developers of the interface.
+ * e.g. In HDFS, NN-DN protocol is private but stable and can help implement rolling upgrades. It communicates that this interface should not be changed in incompatible ways even though it is private.
+ * e.g. In HDFS, FSImage stability can help provide more flexible roll backs.
+* What is the harm in applications using a private interface that is stable? How is it different than a public stable interface?
+ * While a private interface marked as stable is targeted to change only at major releases, it may break at other times if the providers of that interface are willing to changes the internal users of that interface. Further, a public stable interface is less likely to break even at major releases (even though it is allowed to break compatibility) because the impact of the change is larger. If you use a private interface (regardless of its stability) you run the risk of incompatibility.
+* Why bother with Limited-private? Isn’t it giving special treatment to some projects? That is not fair.
+ * First, most interfaces should be public or private; actually let us state it even stronger: make it private unless you really want to expose it to public for general use.
+ * Limited-private is for interfaces that are not intended for general use. They are exposed to related projects that need special hooks. Such a classification has a cost to both the supplier and consumer of the limited interface. Both will have to work together if ever there is a need to break the interface in the future; for example the supplier and the consumers will have to work together to get coordinated releases of their respective projects. This should not be taken lightly – if you can get away with private then do so; if the interface is really for general use for all applications then do so. But remember that making an interface public has huge responsibility. Sometimes Limited-private is just right.
+ * A good example of a limited-private interface is BlockLocations, This is fairly low-level interface that we are willing to expose to MR and perhaps HBase. We are likely to change it down the road and at that time we will have get a coordinated effort with the MR team to release matching releases. While MR and HDFS are always released in sync today, they may change down the road.
+ * If you have a limited-private interface with many projects listed then you are fooling yourself. It is practically public.
+ * It might be worth declaring a special audience classification called Hadoop-Private for the Hadoop family.
+* Lets treat all private interfaces as Hadoop-private. What is the harm in projects in the Hadoop family have access to private classes?
+ * Do we want MR accessing class files that are implementation details inside HDFS. There used to be many such layer violations in the code that we have been cleaning up over the last few years. We don’t want such layer violations to creep back in by no separating between the major components like HDFS and MR.
+* Aren't all public interfaces stable?
+ * One may mark a public interface as evolving in its early days. Here one is promising to make an effort to make compatible changes but may need to break it at minor releases.
+ * One example of a public interface that is unstable is where one is providing an implementation of a standards-body based interface that is still under development. For example, many companies, in an attampt to be first to market, have provided implementations of a new NFS protocol even when the protocol was not fully completed by IETF. The implementor cannot evolve the interface in a fashion that causes least distruption because the stability is controlled by the standards body. Hence it is appropriate to label the interface as unstable.
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
new file mode 100644
index 0000000..dbcf0d8
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/Metrics.md
@@ -0,0 +1,456 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Overview](#Overview)
+* [jvm context](#jvm_context)
+ * [JvmMetrics](#JvmMetrics)
+* [rpc context](#rpc_context)
+ * [rpc](#rpc)
+ * [RetryCache/NameNodeRetryCache](#RetryCacheNameNodeRetryCache)
+* [rpcdetailed context](#rpcdetailed_context)
+ * [rpcdetailed](#rpcdetailed)
+* [dfs context](#dfs_context)
+ * [namenode](#namenode)
+ * [FSNamesystem](#FSNamesystem)
+ * [JournalNode](#JournalNode)
+ * [datanode](#datanode)
+* [yarn context](#yarn_context)
+ * [ClusterMetrics](#ClusterMetrics)
+ * [QueueMetrics](#QueueMetrics)
+ * [NodeManagerMetrics](#NodeManagerMetrics)
+* [ugi context](#ugi_context)
+ * [UgiMetrics](#UgiMetrics)
+* [metricssystem context](#metricssystem_context)
+ * [MetricsSystem](#MetricsSystem)
+* [default context](#default_context)
+ * [StartupProgress](#StartupProgress)
+
+Overview
+========
+
+Metrics are statistical information exposed by Hadoop daemons, used for monitoring, performance tuning and debug. There are many metrics available by default and they are very useful for troubleshooting. This page shows the details of the available metrics.
+
+Each section describes each context into which metrics are grouped.
+
+The documentation of Metrics 2.0 framework is [here](../../api/org/apache/hadoop/metrics2/package-summary.html).
+
+jvm context
+===========
+
+JvmMetrics
+----------
+
+Each metrics record contains tags such as ProcessName, SessionID and Hostname as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `MemNonHeapUsedM` | Current non-heap memory used in MB |
+| `MemNonHeapCommittedM` | Current non-heap memory committed in MB |
+| `MemNonHeapMaxM` | Max non-heap memory size in MB |
+| `MemHeapUsedM` | Current heap memory used in MB |
+| `MemHeapCommittedM` | Current heap memory committed in MB |
+| `MemHeapMaxM` | Max heap memory size in MB |
+| `MemMaxM` | Max memory size in MB |
+| `ThreadsNew` | Current number of NEW threads |
+| `ThreadsRunnable` | Current number of RUNNABLE threads |
+| `ThreadsBlocked` | Current number of BLOCKED threads |
+| `ThreadsWaiting` | Current number of WAITING threads |
+| `ThreadsTimedWaiting` | Current number of TIMED\_WAITING threads |
+| `ThreadsTerminated` | Current number of TERMINATED threads |
+| `GcInfo` | Total GC count and GC time in msec, grouped by the kind of GC. ex.) GcCountPS Scavenge=6, GCTimeMillisPS Scavenge=40, GCCountPS MarkSweep=0, GCTimeMillisPS MarkSweep=0 |
+| `GcCount` | Total GC count |
+| `GcTimeMillis` | Total GC time in msec |
+| `LogFatal` | Total number of FATAL logs |
+| `LogError` | Total number of ERROR logs |
+| `LogWarn` | Total number of WARN logs |
+| `LogInfo` | Total number of INFO logs |
+| `GcNumWarnThresholdExceeded` | Number of times that the GC warn threshold is exceeded |
+| `GcNumInfoThresholdExceeded` | Number of times that the GC info threshold is exceeded |
+| `GcTotalExtraSleepTime` | Total GC extra sleep time in msec |
+
+rpc context
+===========
+
+rpc
+---
+
+Each metrics record contains tags such as Hostname and port (number to which server is bound) as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `ReceivedBytes` | Total number of received bytes |
+| `SentBytes` | Total number of sent bytes |
+| `RpcQueueTimeNumOps` | Total number of RPC calls |
+| `RpcQueueTimeAvgTime` | Average queue time in milliseconds |
+| `RpcProcessingTimeNumOps` | Total number of RPC calls (same to RpcQueueTimeNumOps) |
+| `RpcProcessingAvgTime` | Average Processing time in milliseconds |
+| `RpcAuthenticationFailures` | Total number of authentication failures |
+| `RpcAuthenticationSuccesses` | Total number of authentication successes |
+| `RpcAuthorizationFailures` | Total number of authorization failures |
+| `RpcAuthorizationSuccesses` | Total number of authorization successes |
+| `NumOpenConnections` | Current number of open connections |
+| `CallQueueLength` | Current length of the call queue |
+| `rpcQueueTime`*num*`sNumOps` | Shows total number of RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcQueueTime`*num*`s50thPercentileLatency` | Shows the 50th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcQueueTime`*num*`s75thPercentileLatency` | Shows the 75th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcQueueTime`*num*`s90thPercentileLatency` | Shows the 90th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcQueueTime`*num*`s95thPercentileLatency` | Shows the 95th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcQueueTime`*num*`s99thPercentileLatency` | Shows the 99th percentile of RPC queue time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcProcessingTime`*num*`sNumOps` | Shows total number of RPC calls (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcProcessingTime`*num*`s50thPercentileLatency` | Shows the 50th percentile of RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcProcessingTime`*num*`s75thPercentileLatency` | Shows the 75th percentile of RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcProcessingTime`*num*`s90thPercentileLatency` | Shows the 90th percentile of RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcProcessingTime`*num*`s95thPercentileLatency` | Shows the 95th percentile of RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+| `rpcProcessingTime`*num*`s99thPercentileLatency` | Shows the 99th percentile of RPC processing time in milliseconds (*num* seconds granularity) if `rpc.metrics.quantile.enable` is set to true. *num* is specified by `rpc.metrics.percentiles.intervals`. |
+
+RetryCache/NameNodeRetryCache
+-----------------------------
+
+RetryCache metrics is useful to monitor NameNode fail-over. Each metrics record contains Hostname tag.
+
+| Name | Description |
+|:---- |:---- |
+| `CacheHit` | Total number of RetryCache hit |
+| `CacheCleared` | Total number of RetryCache cleared |
+| `CacheUpdated` | Total number of RetryCache updated |
+
+rpcdetailed context
+===================
+
+Metrics of rpcdetailed context are exposed in unified manner by RPC layer. Two metrics are exposed for each RPC based on its name. Metrics named "(RPC method name)NumOps" indicates total number of method calls, and metrics named "(RPC method name)AvgTime" shows average turn around time for method calls in milliseconds.
+
+rpcdetailed
+-----------
+
+Each metrics record contains tags such as Hostname and port (number to which server is bound) as additional information along with metrics.
+
+The Metrics about RPCs which is not called are not included in metrics record.
+
+| Name | Description |
+|:---- |:---- |
+| *methodname*`NumOps` | Total number of the times the method is called |
+| *methodname*`AvgTime` | Average turn around time of the method in milliseconds |
+
+dfs context
+===========
+
+namenode
+--------
+
+Each metrics record contains tags such as ProcessName, SessionId, and Hostname as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `CreateFileOps` | Total number of files created |
+| `FilesCreated` | Total number of files and directories created by create or mkdir operations |
+| `FilesAppended` | Total number of files appended |
+| `GetBlockLocations` | Total number of getBlockLocations operations |
+| `FilesRenamed` | Total number of rename **operations** (NOT number of files/dirs renamed) |
+| `GetListingOps` | Total number of directory listing operations |
+| `DeleteFileOps` | Total number of delete operations |
+| `FilesDeleted` | Total number of files and directories deleted by delete or rename operations |
+| `FileInfoOps` | Total number of getFileInfo and getLinkFileInfo operations |
+| `AddBlockOps` | Total number of addBlock operations succeeded |
+| `GetAdditionalDatanodeOps` | Total number of getAdditionalDatanode operations |
+| `CreateSymlinkOps` | Total number of createSymlink operations |
+| `GetLinkTargetOps` | Total number of getLinkTarget operations |
+| `FilesInGetListingOps` | Total number of files and directories listed by directory listing operations |
+| `AllowSnapshotOps` | Total number of allowSnapshot operations |
+| `DisallowSnapshotOps` | Total number of disallowSnapshot operations |
+| `CreateSnapshotOps` | Total number of createSnapshot operations |
+| `DeleteSnapshotOps` | Total number of deleteSnapshot operations |
+| `RenameSnapshotOps` | Total number of renameSnapshot operations |
+| `ListSnapshottableDirOps` | Total number of snapshottableDirectoryStatus operations |
+| `SnapshotDiffReportOps` | Total number of getSnapshotDiffReport operations |
+| `TransactionsNumOps` | Total number of Journal transactions |
+| `TransactionsAvgTime` | Average time of Journal transactions in milliseconds |
+| `SyncsNumOps` | Total number of Journal syncs |
+| `SyncsAvgTime` | Average time of Journal syncs in milliseconds |
+| `TransactionsBatchedInSync` | Total number of Journal transactions batched in sync |
+| `BlockReportNumOps` | Total number of processing block reports from DataNode |
+| `BlockReportAvgTime` | Average time of processing block reports in milliseconds |
+| `CacheReportNumOps` | Total number of processing cache reports from DataNode |
+| `CacheReportAvgTime` | Average time of processing cache reports in milliseconds |
+| `SafeModeTime` | The interval between FSNameSystem starts and the last time safemode leaves in milliseconds. (sometimes not equal to the time in SafeMode, see [HDFS-5156](https://issues.apache.org/jira/browse/HDFS-5156)) |
+| `FsImageLoadTime` | Time loading FS Image at startup in milliseconds |
+| `FsImageLoadTime` | Time loading FS Image at startup in milliseconds |
+| `GetEditNumOps` | Total number of edits downloads from SecondaryNameNode |
+| `GetEditAvgTime` | Average edits download time in milliseconds |
+| `GetImageNumOps` | Total number of fsimage downloads from SecondaryNameNode |
+| `GetImageAvgTime` | Average fsimage download time in milliseconds |
+| `PutImageNumOps` | Total number of fsimage uploads to SecondaryNameNode |
+| `PutImageAvgTime` | Average fsimage upload time in milliseconds |
+
+FSNamesystem
+------------
+
+Each metrics record contains tags such as HAState and Hostname as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `MissingBlocks` | Current number of missing blocks |
+| `ExpiredHeartbeats` | Total number of expired heartbeats |
+| `TransactionsSinceLastCheckpoint` | Total number of transactions since last checkpoint |
+| `TransactionsSinceLastLogRoll` | Total number of transactions since last edit log roll |
+| `LastWrittenTransactionId` | Last transaction ID written to the edit log |
+| `LastCheckpointTime` | Time in milliseconds since epoch of last checkpoint |
+| `CapacityTotal` | Current raw capacity of DataNodes in bytes |
+| `CapacityTotalGB` | Current raw capacity of DataNodes in GB |
+| `CapacityUsed` | Current used capacity across all DataNodes in bytes |
+| `CapacityUsedGB` | Current used capacity across all DataNodes in GB |
+| `CapacityRemaining` | Current remaining capacity in bytes |
+| `CapacityRemainingGB` | Current remaining capacity in GB |
+| `CapacityUsedNonDFS` | Current space used by DataNodes for non DFS purposes in bytes |
+| `TotalLoad` | Current number of connections |
+| `SnapshottableDirectories` | Current number of snapshottable directories |
+| `Snapshots` | Current number of snapshots |
+| `BlocksTotal` | Current number of allocated blocks in the system |
+| `FilesTotal` | Current number of files and directories |
+| `PendingReplicationBlocks` | Current number of blocks pending to be replicated |
+| `UnderReplicatedBlocks` | Current number of blocks under replicated |
+| `CorruptBlocks` | Current number of blocks with corrupt replicas. |
+| `ScheduledReplicationBlocks` | Current number of blocks scheduled for replications |
+| `PendingDeletionBlocks` | Current number of blocks pending deletion |
+| `ExcessBlocks` | Current number of excess blocks |
+| `PostponedMisreplicatedBlocks` | (HA-only) Current number of blocks postponed to replicate |
+| `PendingDataNodeMessageCourt` | (HA-only) Current number of pending block-related messages for later processing in the standby NameNode |
+| `MillisSinceLastLoadedEdits` | (HA-only) Time in milliseconds since the last time standby NameNode load edit log. In active NameNode, set to 0 |
+| `BlockCapacity` | Current number of block capacity |
+| `StaleDataNodes` | Current number of DataNodes marked stale due to delayed heartbeat |
+| `TotalFiles` | Current number of files and directories (same as FilesTotal) |
+
+JournalNode
+-----------
+
+The server-side metrics for a journal from the JournalNode's perspective. Each metrics record contains Hostname tag as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `Syncs60sNumOps` | Number of sync operations (1 minute granularity) |
+| `Syncs60s50thPercentileLatencyMicros` | The 50th percentile of sync latency in microseconds (1 minute granularity) |
+| `Syncs60s75thPercentileLatencyMicros` | The 75th percentile of sync latency in microseconds (1 minute granularity) |
+| `Syncs60s90thPercentileLatencyMicros` | The 90th percentile of sync latency in microseconds (1 minute granularity) |
+| `Syncs60s95thPercentileLatencyMicros` | The 95th percentile of sync latency in microseconds (1 minute granularity) |
+| `Syncs60s99thPercentileLatencyMicros` | The 99th percentile of sync latency in microseconds (1 minute granularity) |
+| `Syncs300sNumOps` | Number of sync operations (5 minutes granularity) |
+| `Syncs300s50thPercentileLatencyMicros` | The 50th percentile of sync latency in microseconds (5 minutes granularity) |
+| `Syncs300s75thPercentileLatencyMicros` | The 75th percentile of sync latency in microseconds (5 minutes granularity) |
+| `Syncs300s90thPercentileLatencyMicros` | The 90th percentile of sync latency in microseconds (5 minutes granularity) |
+| `Syncs300s95thPercentileLatencyMicros` | The 95th percentile of sync latency in microseconds (5 minutes granularity) |
+| `Syncs300s99thPercentileLatencyMicros` | The 99th percentile of sync latency in microseconds (5 minutes granularity) |
+| `Syncs3600sNumOps` | Number of sync operations (1 hour granularity) |
+| `Syncs3600s50thPercentileLatencyMicros` | The 50th percentile of sync latency in microseconds (1 hour granularity) |
+| `Syncs3600s75thPercentileLatencyMicros` | The 75th percentile of sync latency in microseconds (1 hour granularity) |
+| `Syncs3600s90thPercentileLatencyMicros` | The 90th percentile of sync latency in microseconds (1 hour granularity) |
+| `Syncs3600s95thPercentileLatencyMicros` | The 95th percentile of sync latency in microseconds (1 hour granularity) |
+| `Syncs3600s99thPercentileLatencyMicros` | The 99th percentile of sync latency in microseconds (1 hour granularity) |
+| `BatchesWritten` | Total number of batches written since startup |
+| `TxnsWritten` | Total number of transactions written since startup |
+| `BytesWritten` | Total number of bytes written since startup |
+| `BatchesWrittenWhileLagging` | Total number of batches written where this node was lagging |
+| `LastWriterEpoch` | Current writer's epoch number |
+| `CurrentLagTxns` | The number of transactions that this JournalNode is lagging |
+| `LastWrittenTxId` | The highest transaction id stored on this JournalNode |
+| `LastPromisedEpoch` | The last epoch number which this node has promised not to accept any lower epoch, or 0 if no promises have been made |
+
+datanode
+--------
+
+Each metrics record contains tags such as SessionId and Hostname as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `BytesWritten` | Total number of bytes written to DataNode |
+| `BytesRead` | Total number of bytes read from DataNode |
+| `BlocksWritten` | Total number of blocks written to DataNode |
+| `BlocksRead` | Total number of blocks read from DataNode |
+| `BlocksReplicated` | Total number of blocks replicated |
+| `BlocksRemoved` | Total number of blocks removed |
+| `BlocksVerified` | Total number of blocks verified |
+| `BlockVerificationFailures` | Total number of verifications failures |
+| `BlocksCached` | Total number of blocks cached |
+| `BlocksUncached` | Total number of blocks uncached |
+| `ReadsFromLocalClient` | Total number of read operations from local client |
+| `ReadsFromRemoteClient` | Total number of read operations from remote client |
+| `WritesFromLocalClient` | Total number of write operations from local client |
+| `WritesFromRemoteClient` | Total number of write operations from remote client |
+| `BlocksGetLocalPathInfo` | Total number of operations to get local path names of blocks |
+| `FsyncCount` | Total number of fsync |
+| `VolumeFailures` | Total number of volume failures occurred |
+| `ReadBlockOpNumOps` | Total number of read operations |
+| `ReadBlockOpAvgTime` | Average time of read operations in milliseconds |
+| `WriteBlockOpNumOps` | Total number of write operations |
+| `WriteBlockOpAvgTime` | Average time of write operations in milliseconds |
+| `BlockChecksumOpNumOps` | Total number of blockChecksum operations |
+| `BlockChecksumOpAvgTime` | Average time of blockChecksum operations in milliseconds |
+| `CopyBlockOpNumOps` | Total number of block copy operations |
+| `CopyBlockOpAvgTime` | Average time of block copy operations in milliseconds |
+| `ReplaceBlockOpNumOps` | Total number of block replace operations |
+| `ReplaceBlockOpAvgTime` | Average time of block replace operations in milliseconds |
+| `HeartbeatsNumOps` | Total number of heartbeats |
+| `HeartbeatsAvgTime` | Average heartbeat time in milliseconds |
+| `BlockReportsNumOps` | Total number of block report operations |
+| `BlockReportsAvgTime` | Average time of block report operations in milliseconds |
+| `CacheReportsNumOps` | Total number of cache report operations |
+| `CacheReportsAvgTime` | Average time of cache report operations in milliseconds |
+| `PacketAckRoundTripTimeNanosNumOps` | Total number of ack round trip |
+| `PacketAckRoundTripTimeNanosAvgTime` | Average time from ack send to receive minus the downstream ack time in nanoseconds |
+| `FlushNanosNumOps` | Total number of flushes |
+| `FlushNanosAvgTime` | Average flush time in nanoseconds |
+| `FsyncNanosNumOps` | Total number of fsync |
+| `FsyncNanosAvgTime` | Average fsync time in nanoseconds |
+| `SendDataPacketBlockedOnNetworkNanosNumOps` | Total number of sending packets |
+| `SendDataPacketBlockedOnNetworkNanosAvgTime` | Average waiting time of sending packets in nanoseconds |
+| `SendDataPacketTransferNanosNumOps` | Total number of sending packets |
+| `SendDataPacketTransferNanosAvgTime` | Average transfer time of sending packets in nanoseconds |
+
+yarn context
+============
+
+ClusterMetrics
+--------------
+
+ClusterMetrics shows the metrics of the YARN cluster from the ResourceManager's perspective. Each metrics record contains Hostname tag as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `NumActiveNMs` | Current number of active NodeManagers |
+| `NumDecommissionedNMs` | Current number of decommissioned NodeManagers |
+| `NumLostNMs` | Current number of lost NodeManagers for not sending heartbeats |
+| `NumUnhealthyNMs` | Current number of unhealthy NodeManagers |
+| `NumRebootedNMs` | Current number of rebooted NodeManagers |
+
+QueueMetrics
+------------
+
+QueueMetrics shows an application queue from the ResourceManager's perspective. Each metrics record shows the statistics of each queue, and contains tags such as queue name and Hostname as additional information along with metrics.
+
+In `running_`*num* metrics such as `running_0`, you can set the property `yarn.resourcemanager.metrics.runtime.buckets` in yarn-site.xml to change the buckets. The default values is `60,300,1440`.
+
+| Name | Description |
+|:---- |:---- |
+| `running_0` | Current number of running applications whose elapsed time are less than 60 minutes |
+| `running_60` | Current number of running applications whose elapsed time are between 60 and 300 minutes |
+| `running_300` | Current number of running applications whose elapsed time are between 300 and 1440 minutes |
+| `running_1440` | Current number of running applications elapsed time are more than 1440 minutes |
+| `AppsSubmitted` | Total number of submitted applications |
+| `AppsRunning` | Current number of running applications |
+| `AppsPending` | Current number of applications that have not yet been assigned by any containers |
+| `AppsCompleted` | Total number of completed applications |
+| `AppsKilled` | Total number of killed applications |
+| `AppsFailed` | Total number of failed applications |
+| `AllocatedMB` | Current allocated memory in MB |
+| `AllocatedVCores` | Current allocated CPU in virtual cores |
+| `AllocatedContainers` | Current number of allocated containers |
+| `AggregateContainersAllocated` | Total number of allocated containers |
+| `AggregateContainersReleased` | Total number of released containers |
+| `AvailableMB` | Current available memory in MB |
+| `AvailableVCores` | Current available CPU in virtual cores |
+| `PendingMB` | Current pending memory resource requests in MB that are not yet fulfilled by the scheduler |
+| `PendingVCores` | Current pending CPU allocation requests in virtual cores that are not yet fulfilled by the scheduler |
+| `PendingContainers` | Current pending resource requests that are not yet fulfilled by the scheduler |
+| `ReservedMB` | Current reserved memory in MB |
+| `ReservedVCores` | Current reserved CPU in virtual cores |
+| `ReservedContainers` | Current number of reserved containers |
+| `ActiveUsers` | Current number of active users |
+| `ActiveApplications` | Current number of active applications |
+| `FairShareMB` | (FairScheduler only) Current fair share of memory in MB |
+| `FairShareVCores` | (FairScheduler only) Current fair share of CPU in virtual cores |
+| `MinShareMB` | (FairScheduler only) Minimum share of memory in MB |
+| `MinShareVCores` | (FairScheduler only) Minimum share of CPU in virtual cores |
+| `MaxShareMB` | (FairScheduler only) Maximum share of memory in MB |
+| `MaxShareVCores` | (FairScheduler only) Maximum share of CPU in virtual cores |
+
+NodeManagerMetrics
+------------------
+
+NodeManagerMetrics shows the statistics of the containers in the node. Each metrics record contains Hostname tag as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `containersLaunched` | Total number of launched containers |
+| `containersCompleted` | Total number of successfully completed containers |
+| `containersFailed` | Total number of failed containers |
+| `containersKilled` | Total number of killed containers |
+| `containersIniting` | Current number of initializing containers |
+| `containersRunning` | Current number of running containers |
+| `allocatedContainers` | Current number of allocated containers |
+| `allocatedGB` | Current allocated memory in GB |
+| `availableGB` | Current available memory in GB |
+
+ugi context
+===========
+
+UgiMetrics
+----------
+
+UgiMetrics is related to user and group information. Each metrics record contains Hostname tag as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `LoginSuccessNumOps` | Total number of successful kerberos logins |
+| `LoginSuccessAvgTime` | Average time for successful kerberos logins in milliseconds |
+| `LoginFailureNumOps` | Total number of failed kerberos logins |
+| `LoginFailureAvgTime` | Average time for failed kerberos logins in milliseconds |
+| `getGroupsNumOps` | Total number of group resolutions |
+| `getGroupsAvgTime` | Average time for group resolution in milliseconds |
+| `getGroups`*num*`sNumOps` | Total number of group resolutions (*num* seconds granularity). *num* is specified by `hadoop.user.group.metrics.percentiles.intervals`. |
+| `getGroups`*num*`s50thPercentileLatency` | Shows the 50th percentile of group resolution time in milliseconds (*num* seconds granularity). *num* is specified by `hadoop.user.group.metrics.percentiles.intervals`. |
+| `getGroups`*num*`s75thPercentileLatency` | Shows the 75th percentile of group resolution time in milliseconds (*num* seconds granularity). *num* is specified by `hadoop.user.group.metrics.percentiles.intervals`. |
+| `getGroups`*num*`s90thPercentileLatency` | Shows the 90th percentile of group resolution time in milliseconds (*num* seconds granularity). *num* is specified by `hadoop.user.group.metrics.percentiles.intervals`. |
+| `getGroups`*num*`s95thPercentileLatency` | Shows the 95th percentile of group resolution time in milliseconds (*num* seconds granularity). *num* is specified by `hadoop.user.group.metrics.percentiles.intervals`. |
+| `getGroups`*num*`s99thPercentileLatency` | Shows the 99th percentile of group resolution time in milliseconds (*num* seconds granularity). *num* is specified by `hadoop.user.group.metrics.percentiles.intervals`. |
+
+metricssystem context
+=====================
+
+MetricsSystem
+-------------
+
+MetricsSystem shows the statistics for metrics snapshots and publishes. Each metrics record contains Hostname tag as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `NumActiveSources` | Current number of active metrics sources |
+| `NumAllSources` | Total number of metrics sources |
+| `NumActiveSinks` | Current number of active sinks |
+| `NumAllSinks` | Total number of sinks (BUT usually less than `NumActiveSinks`, see [HADOOP-9946](https://issues.apache.org/jira/browse/HADOOP-9946)) |
+| `SnapshotNumOps` | Total number of operations to snapshot statistics from a metrics source |
+| `SnapshotAvgTime` | Average time in milliseconds to snapshot statistics from a metrics source |
+| `PublishNumOps` | Total number of operations to publish statistics to a sink |
+| `PublishAvgTime` | Average time in milliseconds to publish statistics to a sink |
+| `DroppedPubAll` | Total number of dropped publishes |
+| `Sink_`*instance*`NumOps` | Total number of sink operations for the *instance* |
+| `Sink_`*instance*`AvgTime` | Average time in milliseconds of sink operations for the *instance* |
+| `Sink_`*instance*`Dropped` | Total number of dropped sink operations for the *instance* |
+| `Sink_`*instance*`Qsize` | Current queue length of sink operations (BUT always set to 0 because nothing to increment this metrics, see [HADOOP-9941](https://issues.apache.org/jira/browse/HADOOP-9941)) |
+
+default context
+===============
+
+StartupProgress
+---------------
+
+StartupProgress metrics shows the statistics of NameNode startup. Four metrics are exposed for each startup phase based on its name. The startup *phase*s are `LoadingFsImage`, `LoadingEdits`, `SavingCheckpoint`, and `SafeMode`. Each metrics record contains Hostname tag as additional information along with metrics.
+
+| Name | Description |
+|:---- |:---- |
+| `ElapsedTime` | Total elapsed time in milliseconds |
+| `PercentComplete` | Current rate completed in NameNode startup progress (The max value is not 100 but 1.0) |
+| *phase*`Count` | Total number of steps completed in the phase |
+| *phase*`ElapsedTime` | Total elapsed time in the phase in milliseconds |
+| *phase*`Total` | Total number of steps in the phase |
+| *phase*`PercentComplete` | Current rate completed in the phase (The max value is not 100 but 1.0) |
+
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm b/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
new file mode 100644
index 0000000..5a2c70c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/NativeLibraries.md.vm
@@ -0,0 +1,145 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+Native Libraries Guide
+======================
+
+* [Native Libraries Guide](#Native_Libraries_Guide)
+ * [Overview](#Overview)
+ * [Native Hadoop Library](#Native_Hadoop_Library)
+ * [Usage](#Usage)
+ * [Components](#Components)
+ * [Supported Platforms](#Supported_Platforms)
+ * [Download](#Download)
+ * [Build](#Build)
+ * [Runtime](#Runtime)
+ * [Check](#Check)
+ * [Native Shared Libraries](#Native_Shared_Libraries)
+
+Overview
+--------
+
+This guide describes the native hadoop library and includes a small discussion about native shared libraries.
+
+Note: Depending on your environment, the term "native libraries" could refer to all \*.so's you need to compile; and, the term "native compression" could refer to all \*.so's you need to compile that are specifically related to compression. Currently, however, this document only addresses the native hadoop library (`libhadoop.so`). The document for libhdfs library (`libhdfs.so`) is [here](../hadoop-hdfs/LibHdfs.html).
+
+Native Hadoop Library
+---------------------
+
+Hadoop has native implementations of certain components for performance reasons and for non-availability of Java implementations. These components are available in a single, dynamically-linked native library called the native hadoop library. On the \*nix platforms the library is named `libhadoop.so`.
+
+Usage
+-----
+
+It is fairly easy to use the native hadoop library:
+
+1. Review the components.
+2. Review the supported platforms.
+3. Either download a hadoop release, which will include a pre-built version of the native hadoop library, or build your own version of the native hadoop library. Whether you download or build, the name for the library is the same: libhadoop.so
+4. Install the compression codec development packages (\>zlib-1.2, \>gzip-1.2):
+ * If you download the library, install one or more development packages - whichever compression codecs you want to use with your deployment.
+ * If you build the library, it is mandatory to install both development packages.
+5. Check the runtime log files.
+
+Components
+----------
+
+The native hadoop library includes various components:
+
+* Compression Codecs (bzip2, lz4, snappy, zlib)
+* Native IO utilities for [HDFS Short-Circuit Local Reads](../hadoop-hdfs/ShortCircuitLocalReads.html) and [Centralized Cache Management in HDFS](../hadoop-hdfs/CentralizedCacheManagement.html)
+* CRC32 checksum implementation
+
+Supported Platforms
+-------------------
+
+The native hadoop library is supported on \*nix platforms only. The library does not to work with Cygwin or the Mac OS X platform.
+
+The native hadoop library is mainly used on the GNU/Linus platform and has been tested on these distributions:
+
+* RHEL4/Fedora
+* Ubuntu
+* Gentoo
+
+On all the above distributions a 32/64 bit native hadoop library will work with a respective 32/64 bit jvm.
+
+Download
+--------
+
+The pre-built 32-bit i386-Linux native hadoop library is available as part of the hadoop distribution and is located in the `lib/native` directory. You can download the hadoop distribution from Hadoop Common Releases.
+
+Be sure to install the zlib and/or gzip development packages - whichever compression codecs you want to use with your deployment.
+
+Build
+-----
+
+The native hadoop library is written in ANSI C and is built using the GNU autotools-chain (autoconf, autoheader, automake, autoscan, libtool). This means it should be straight-forward to build the library on any platform with a standards-compliant C compiler and the GNU autotools-chain (see the supported platforms).
+
+The packages you need to install on the target platform are:
+
+* C compiler (e.g. GNU C Compiler)
+* GNU Autools Chain: autoconf, automake, libtool
+* zlib-development package (stable version \>= 1.2.0)
+* openssl-development package(e.g. libssl-dev)
+
+Once you installed the prerequisite packages use the standard hadoop pom.xml file and pass along the native flag to build the native hadoop library:
+
+ $ mvn package -Pdist,native -DskipTests -Dtar
+
+You should see the newly-built library in:
+
+ $ hadoop-dist/target/hadoop-${project.version}/lib/native
+
+Please note the following:
+
+* It is mandatory to install both the zlib and gzip development packages on the target platform in order to build the native hadoop library; however, for deployment it is sufficient to install just one package if you wish to use only one codec.
+* It is necessary to have the correct 32/64 libraries for zlib, depending on the 32/64 bit jvm for the target platform, in order to build and deploy the native hadoop library.
+
+Runtime
+-------
+
+The bin/hadoop script ensures that the native hadoop library is on the library path via the system property: `-Djava.library.path=<path> `
+
+During runtime, check the hadoop log files for your MapReduce tasks.
+
+* If everything is all right, then: `DEBUG util.NativeCodeLoader - Trying to load the custom-built native-hadoop library...` `INFO util.NativeCodeLoader - Loaded the native-hadoop library`
+* If something goes wrong, then: `INFO util.NativeCodeLoader - Unable to load native-hadoop library for your platform... using builtin-java classes where applicable`
+
+Check
+-----
+
+NativeLibraryChecker is a tool to check whether native libraries are loaded correctly. You can launch NativeLibraryChecker as follows:
+
+ $ hadoop checknative -a
+ 14/12/06 01:30:45 WARN bzip2.Bzip2Factory: Failed to load/initialize native-bzip2 library system-native, will use pure-Java version
+ 14/12/06 01:30:45 INFO zlib.ZlibFactory: Successfully loaded & initialized native-zlib library
+ Native library checking:
+ hadoop: true /home/ozawa/hadoop/lib/native/libhadoop.so.1.0.0
+ zlib: true /lib/x86_64-linux-gnu/libz.so.1
+ snappy: true /usr/lib/libsnappy.so.1
+ lz4: true revision:99
+ bzip2: false
+
+Native Shared Libraries
+-----------------------
+
+You can load any native shared library using DistributedCache for distributing and symlinking the library files.
+
+This example shows you how to distribute a shared library, mylib.so, and load it from a MapReduce task.
+
+1. First copy the library to the HDFS: `bin/hadoop fs -copyFromLocal mylib.so.1 /libraries/mylib.so.1`
+2. The job launching program should contain the following: `DistributedCache.createSymlink(conf);` `DistributedCache.addCacheFile("hdfs://host:port/libraries/mylib.so. 1#mylib.so", conf);`
+3. The MapReduce task can contain: `System.loadLibrary("mylib.so");`
+
+Note: If you downloaded or built the native hadoop library, you don’t need to use DistibutedCache to make the library available to your MapReduce tasks.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
new file mode 100644
index 0000000..41fcb37
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/RackAwareness.md
@@ -0,0 +1,104 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+* [Rack Awareness](#Rack_Awareness)
+ * [python Example](#python_Example)
+ * [bash Example](#bash_Example)
+
+Rack Awareness
+==============
+
+Hadoop components are rack-aware. For example, HDFS block placement will use rack awareness for fault tolerance by placing one block replica on a different rack. This provides data availability in the event of a network switch failure or partition within the cluster.
+
+Hadoop master daemons obtain the rack id of the cluster slaves by invoking either an external script or java class as specified by configuration files. Using either the java class or external script for topology, output must adhere to the java **org.apache.hadoop.net.DNSToSwitchMapping** interface. The interface expects a one-to-one correspondence to be maintained and the topology information in the format of '/myrack/myhost', where '/' is the topology delimiter, 'myrack' is the rack identifier, and 'myhost' is the individual host. Assuming a single /24 subnet per rack, one could use the format of '/192.168.100.0/192.168.100.5' as a unique rack-host topology mapping.
+
+To use the java class for topology mapping, the class name is specified by the **topology.node.switch.mapping.impl** parameter in the configuration file. An example, NetworkTopology.java, is included with the hadoop distribution and can be customized by the Hadoop administrator. Using a Java class instead of an external script has a performance benefit in that Hadoop doesn't need to fork an external process when a new slave node registers itself.
+
+If implementing an external script, it will be specified with the **topology.script.file.name** parameter in the configuration files. Unlike the java class, the external topology script is not included with the Hadoop distribution and is provided by the administrator. Hadoop will send multiple IP addresses to ARGV when forking the topology script. The number of IP addresses sent to the topology script is controlled with **net.topology.script.number.args** and defaults to 100. If **net.topology.script.number.args** was changed to 1, a topology script would get forked for each IP submitted by DataNodes and/or NodeManagers.
+
+If **topology.script.file.name** or **topology.node.switch.mapping.impl** is not set, the rack id '/default-rack' is returned for any passed IP address. While this behavior appears desirable, it can cause issues with HDFS block replication as default behavior is to write one replicated block off rack and is unable to do so as there is only a single rack named '/default-rack'.
+
+An additional configuration setting is **mapreduce.jobtracker.taskcache.levels** which determines the number of levels (in the network topology) of caches MapReduce will use. So, for example, if it is the default value of 2, two levels of caches will be constructed - one for hosts (host -\> task mapping) and another for racks (rack -\> task mapping). Giving us our one-to-one mapping of '/myrack/myhost'.
+
+python Example
+--------------
+```python
+#!/usr/bin/python
+# this script makes assumptions about the physical environment.
+# 1) each rack is its own layer 3 network with a /24 subnet, which
+# could be typical where each rack has its own
+# switch with uplinks to a central core router.
+#
+# +-----------+
+# |core router|
+# +-----------+
+# / \
+# +-----------+ +-----------+
+# |rack switch| |rack switch|
+# +-----------+ +-----------+
+# | data node | | data node |
+# +-----------+ +-----------+
+# | data node | | data node |
+# +-----------+ +-----------+
+#
+# 2) topology script gets list of IP's as input, calculates network address, and prints '/network_address/ip'.
+
+import netaddr
+import sys
+sys.argv.pop(0) # discard name of topology script from argv list as we just want IP addresses
+
+netmask = '255.255.255.0' # set netmask to what's being used in your environment. The example uses a /24
+
+for ip in sys.argv: # loop over list of datanode IP's
+address = '{0}/{1}'.format(ip, netmask) # format address string so it looks like 'ip/netmask' to make netaddr work
+try:
+ network_address = netaddr.IPNetwork(address).network # calculate and print network address
+ print "/{0}".format(network_address)
+except:
+ print "/rack-unknown" # print catch-all value if unable to calculate network address
+```
+
+bash Example
+------------
+
+```bash
+#!/bin/bash
+# Here's a bash example to show just how simple these scripts can be
+# Assuming we have flat network with everything on a single switch, we can fake a rack topology.
+# This could occur in a lab environment where we have limited nodes,like 2-8 physical machines on a unmanaged switch.
+# This may also apply to multiple virtual machines running on the same physical hardware.
+# The number of machines isn't important, but that we are trying to fake a network topology when there isn't one.
+#
+# +----------+ +--------+
+# |jobtracker| |datanode|
+# +----------+ +--------+
+# \ /
+# +--------+ +--------+ +--------+
+# |datanode|--| switch |--|datanode|
+# +--------+ +--------+ +--------+
+# / \
+# +--------+ +--------+
+# |datanode| |namenode|
+# +--------+ +--------+
+#
+# With this network topology, we are treating each host as a rack. This is being done by taking the last octet
+# in the datanode's IP and prepending it with the word '/rack-'. The advantage for doing this is so HDFS
+# can create its 'off-rack' block copy.
+# 1) 'echo $@' will echo all ARGV values to xargs.
+# 2) 'xargs' will enforce that we print a single argv value per line
+# 3) 'awk' will split fields on dots and append the last field to the string '/rack-'. If awk
+# fails to split on four dots, it will still print '/rack-' last field value
+
+echo $@ | xargs -n 1 | awk -F '.' '{print "/rack-"$NF}'
+```
\ No newline at end of file
[29/50] [abbrv] hadoop git commit: HADOOP-11495. Convert site
documentation from apt to markdown (Masatake Iwasaki via aw)
Posted by zj...@apache.org.
HADOOP-11495. Convert site documentation from apt to markdown (Masatake Iwasaki via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e9d26fe9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e9d26fe9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e9d26fe9
Branch: refs/heads/YARN-2928
Commit: e9d26fe9eb16a0482d3581504ecad22b4cd65077
Parents: 6338ce3
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Feb 10 13:39:57 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Feb 10 13:39:57 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../src/site/apt/CLIMiniCluster.apt.vm | 83 --
.../src/site/apt/ClusterSetup.apt.vm | 651 --------------
.../src/site/apt/CommandsManual.apt.vm | 327 -------
.../src/site/apt/Compatibility.apt.vm | 541 ------------
.../src/site/apt/DeprecatedProperties.apt.vm | 552 ------------
.../src/site/apt/FileSystemShell.apt.vm | 764 ----------------
.../src/site/apt/HttpAuthentication.apt.vm | 98 ---
.../src/site/apt/InterfaceClassification.apt.vm | 239 -----
.../hadoop-common/src/site/apt/Metrics.apt.vm | 879 -------------------
.../src/site/apt/NativeLibraries.apt.vm | 205 -----
.../src/site/apt/RackAwareness.apt.vm | 140 ---
.../src/site/apt/SecureMode.apt.vm | 689 ---------------
.../src/site/apt/ServiceLevelAuth.apt.vm | 216 -----
.../src/site/apt/SingleCluster.apt.vm | 286 ------
.../src/site/apt/SingleNodeSetup.apt.vm | 24 -
.../src/site/apt/Superusers.apt.vm | 144 ---
.../hadoop-common/src/site/apt/Tracing.apt.vm | 233 -----
.../src/site/markdown/CLIMiniCluster.md.vm | 68 ++
.../src/site/markdown/ClusterSetup.md | 339 +++++++
.../src/site/markdown/CommandsManual.md | 227 +++++
.../src/site/markdown/Compatibility.md | 313 +++++++
.../src/site/markdown/DeprecatedProperties.md | 288 ++++++
.../src/site/markdown/FileSystemShell.md | 689 +++++++++++++++
.../src/site/markdown/HttpAuthentication.md | 58 ++
.../site/markdown/InterfaceClassification.md | 105 +++
.../hadoop-common/src/site/markdown/Metrics.md | 456 ++++++++++
.../src/site/markdown/NativeLibraries.md.vm | 145 +++
.../src/site/markdown/RackAwareness.md | 104 +++
.../src/site/markdown/SecureMode.md | 375 ++++++++
.../src/site/markdown/ServiceLevelAuth.md | 144 +++
.../src/site/markdown/SingleCluster.md.vm | 232 +++++
.../src/site/markdown/SingleNodeSetup.md | 20 +
.../src/site/markdown/Superusers.md | 106 +++
.../hadoop-common/src/site/markdown/Tracing.md | 182 ++++
35 files changed, 3854 insertions(+), 6071 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index fadc744..1ba93e8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -168,6 +168,9 @@ Trunk (Unreleased)
HADOOP-6964. Allow compact property description in xml (Kengo Seki
via aw)
+ HADOOP-11495. Convert site documentation from apt to markdown
+ (Masatake Iwasaki via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm
deleted file mode 100644
index 2d12c39..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/CLIMiniCluster.apt.vm
+++ /dev/null
@@ -1,83 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop MapReduce Next Generation ${project.version} - CLI MiniCluster.
- ---
- ---
- ${maven.build.timestamp}
-
-Hadoop MapReduce Next Generation - CLI MiniCluster.
-
-%{toc|section=1|fromDepth=0}
-
-* {Purpose}
-
- Using the CLI MiniCluster, users can simply start and stop a single-node
- Hadoop cluster with a single command, and without the need to set any
- environment variables or manage configuration files. The CLI MiniCluster
- starts both a <<<YARN>>>/<<<MapReduce>>> & <<<HDFS>>> clusters.
-
- This is useful for cases where users want to quickly experiment with a real
- Hadoop cluster or test non-Java programs that rely on significant Hadoop
- functionality.
-
-* {Hadoop Tarball}
-
- You should be able to obtain the Hadoop tarball from the release. Also, you
- can directly create a tarball from the source:
-
-+---+
-$ mvn clean install -DskipTests
-$ mvn package -Pdist -Dtar -DskipTests -Dmaven.javadoc.skip
-+---+
- <<NOTE:>> You will need {{{http://code.google.com/p/protobuf/}protoc 2.5.0}}
- installed.
-
- The tarball should be available in <<<hadoop-dist/target/>>> directory.
-
-* {Running the MiniCluster}
-
- From inside the root directory of the extracted tarball, you can start the CLI
- MiniCluster using the following command:
-
-+---+
-$ bin/hadoop jar ./share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${project.version}-tests.jar minicluster -rmport RM_PORT -jhsport JHS_PORT
-+---+
-
- In the example command above, <<<RM_PORT>>> and <<<JHS_PORT>>> should be
- replaced by the user's choice of these port numbers. If not specified, random
- free ports will be used.
-
- There are a number of command line arguments that the users can use to control
- which services to start, and to pass other configuration properties.
- The available command line arguments:
-
-+---+
-$ -D <property=value> Options to pass into configuration object
-$ -datanodes <arg> How many datanodes to start (default 1)
-$ -format Format the DFS (default false)
-$ -help Prints option help.
-$ -jhsport <arg> JobHistoryServer port (default 0--we choose)
-$ -namenode <arg> URL of the namenode (default is either the DFS
-$ cluster or a temporary dir)
-$ -nnport <arg> NameNode port (default 0--we choose)
-$ -nodemanagers <arg> How many nodemanagers to start (default 1)
-$ -nodfs Don't start a mini DFS cluster
-$ -nomr Don't start a mini MR cluster
-$ -rmport <arg> ResourceManager port (default 0--we choose)
-$ -writeConfig <path> Save configuration to this XML file.
-$ -writeDetails <path> Write basic information to this JSON file.
-+---+
-
- To display this full list of available arguments, the user can pass the
- <<<-help>>> argument to the above command.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
deleted file mode 100644
index 52b0552..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/ClusterSetup.apt.vm
+++ /dev/null
@@ -1,651 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
- Hadoop ${project.version} - Cluster Setup
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc|section=1|fromDepth=0}
-
-Hadoop Cluster Setup
-
-* {Purpose}
-
- This document describes how to install and configure
- Hadoop clusters ranging from a few nodes to extremely large clusters
- with thousands of nodes. To play with Hadoop, you may first want to
- install it on a single machine (see {{{./SingleCluster.html}Single Node Setup}}).
-
- This document does not cover advanced topics such as {{{./SecureMode.html}Security}} or
- High Availability.
-
-* {Prerequisites}
-
- * Install Java. See the {{{http://wiki.apache.org/hadoop/HadoopJavaVersions}Hadoop Wiki}} for known good versions.
- * Download a stable version of Hadoop from Apache mirrors.
-
-* {Installation}
-
- Installing a Hadoop cluster typically involves unpacking the software on all
- the machines in the cluster or installing it via a packaging system as
- appropriate for your operating system. It is important to divide up the hardware
- into functions.
-
- Typically one machine in the cluster is designated as the NameNode and
- another machine the as ResourceManager, exclusively. These are the masters. Other
- services (such as Web App Proxy Server and MapReduce Job History server) are usually
- run either on dedicated hardware or on shared infrastrucutre, depending upon the load.
-
- The rest of the machines in the cluster act as both DataNode and NodeManager.
- These are the slaves.
-
-* {Configuring Hadoop in Non-Secure Mode}
-
- Hadoop's Java configuration is driven by two types of important configuration files:
-
- * Read-only default configuration - <<<core-default.xml>>>,
- <<<hdfs-default.xml>>>, <<<yarn-default.xml>>> and
- <<<mapred-default.xml>>>.
-
- * Site-specific configuration - <<<etc/hadoop/core-site.xml>>>,
- <<<etc/hadoop/hdfs-site.xml>>>, <<<etc/hadoop/yarn-site.xml>>> and
- <<<etc/hadoop/mapred-site.xml>>>.
-
-
- Additionally, you can control the Hadoop scripts found in the bin/
- directory of the distribution, by setting site-specific values via the
- <<<etc/hadoop/hadoop-env.sh>>> and <<<etc/hadoop/yarn-env.sh>>>.
-
- To configure the Hadoop cluster you will need to configure the
- <<<environment>>> in which the Hadoop daemons execute as well as the
- <<<configuration parameters>>> for the Hadoop daemons.
-
- HDFS daemons are NameNode, SecondaryNameNode, and DataNode. YARN damones
- are ResourceManager, NodeManager, and WebAppProxy. If MapReduce is to be
- used, then the MapReduce Job History Server will also be running. For
- large installations, these are generally running on separate hosts.
-
-
-** {Configuring Environment of Hadoop Daemons}
-
- Administrators should use the <<<etc/hadoop/hadoop-env.sh>>> and optionally the
- <<<etc/hadoop/mapred-env.sh>>> and <<<etc/hadoop/yarn-env.sh>>> scripts to do
- site-specific customization of the Hadoop daemons' process environment.
-
- At the very least, you must specify the <<<JAVA_HOME>>> so that it is
- correctly defined on each remote node.
-
- Administrators can configure individual daemons using the configuration
- options shown below in the table:
-
-*--------------------------------------+--------------------------------------+
-|| Daemon || Environment Variable |
-*--------------------------------------+--------------------------------------+
-| NameNode | HADOOP_NAMENODE_OPTS |
-*--------------------------------------+--------------------------------------+
-| DataNode | HADOOP_DATANODE_OPTS |
-*--------------------------------------+--------------------------------------+
-| Secondary NameNode | HADOOP_SECONDARYNAMENODE_OPTS |
-*--------------------------------------+--------------------------------------+
-| ResourceManager | YARN_RESOURCEMANAGER_OPTS |
-*--------------------------------------+--------------------------------------+
-| NodeManager | YARN_NODEMANAGER_OPTS |
-*--------------------------------------+--------------------------------------+
-| WebAppProxy | YARN_PROXYSERVER_OPTS |
-*--------------------------------------+--------------------------------------+
-| Map Reduce Job History Server | HADOOP_JOB_HISTORYSERVER_OPTS |
-*--------------------------------------+--------------------------------------+
-
-
- For example, To configure Namenode to use parallelGC, the following
- statement should be added in hadoop-env.sh :
-
-----
- export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC"
-----
-
- See <<<etc/hadoop/hadoop-env.sh>>> for other examples.
-
- Other useful configuration parameters that you can customize include:
-
- * <<<HADOOP_PID_DIR>>> - The directory where the
- daemons' process id files are stored.
-
- * <<<HADOOP_LOG_DIR>>> - The directory where the
- daemons' log files are stored. Log files are automatically created
- if they don't exist.
-
- * <<<HADOOP_HEAPSIZE_MAX>>> - The maximum amount of
- memory to use for the Java heapsize. Units supported by the JVM
- are also supported here. If no unit is present, it will be assumed
- the number is in megabytes. By default, Hadoop will let the JVM
- determine how much to use. This value can be overriden on
- a per-daemon basis using the appropriate <<<_OPTS>>> variable listed above.
- For example, setting <<<HADOOP_HEAPSIZE_MAX=1g>>> and
- <<<HADOOP_NAMENODE_OPTS="-Xmx5g">>> will configure the NameNode with 5GB heap.
-
- In most cases, you should specify the <<<HADOOP_PID_DIR>>> and
- <<<HADOOP_LOG_DIR>>> directories such that they can only be
- written to by the users that are going to run the hadoop daemons.
- Otherwise there is the potential for a symlink attack.
-
- It is also traditional to configure <<<HADOOP_PREFIX>>> in the system-wide
- shell environment configuration. For example, a simple script inside
- <<</etc/profile.d>>>:
-
----
- HADOOP_PREFIX=/path/to/hadoop
- export HADOOP_PREFIX
----
-
-*--------------------------------------+--------------------------------------+
-|| Daemon || Environment Variable |
-*--------------------------------------+--------------------------------------+
-| ResourceManager | YARN_RESOURCEMANAGER_HEAPSIZE |
-*--------------------------------------+--------------------------------------+
-| NodeManager | YARN_NODEMANAGER_HEAPSIZE |
-*--------------------------------------+--------------------------------------+
-| WebAppProxy | YARN_PROXYSERVER_HEAPSIZE |
-*--------------------------------------+--------------------------------------+
-| Map Reduce Job History Server | HADOOP_JOB_HISTORYSERVER_HEAPSIZE |
-*--------------------------------------+--------------------------------------+
-
-** {Configuring the Hadoop Daemons}
-
- This section deals with important parameters to be specified in
- the given configuration files:
-
- * <<<etc/hadoop/core-site.xml>>>
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<fs.defaultFS>>> | NameNode URI | <hdfs://host:port/> |
-*-------------------------+-------------------------+------------------------+
-| <<<io.file.buffer.size>>> | 131072 | |
-| | | Size of read/write buffer used in SequenceFiles. |
-*-------------------------+-------------------------+------------------------+
-
- * <<<etc/hadoop/hdfs-site.xml>>>
-
- * Configurations for NameNode:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.name.dir>>> | | |
-| | Path on the local filesystem where the NameNode stores the namespace | |
-| | and transactions logs persistently. | |
-| | | If this is a comma-delimited list of directories then the name table is |
-| | | replicated in all of the directories, for redundancy. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.hosts>>> / <<<dfs.namenode.hosts.exclude>>> | | |
-| | List of permitted/excluded DataNodes. | |
-| | | If necessary, use these files to control the list of allowable |
-| | | datanodes. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.blocksize>>> | 268435456 | |
-| | | HDFS blocksize of 256MB for large file-systems. |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.namenode.handler.count>>> | 100 | |
-| | | More NameNode server threads to handle RPCs from large number of |
-| | | DataNodes. |
-*-------------------------+-------------------------+------------------------+
-
- * Configurations for DataNode:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<dfs.datanode.data.dir>>> | | |
-| | Comma separated list of paths on the local filesystem of a | |
-| | <<<DataNode>>> where it should store its blocks. | |
-| | | If this is a comma-delimited list of directories, then data will be |
-| | | stored in all named directories, typically on different devices. |
-*-------------------------+-------------------------+------------------------+
-
- * <<<etc/hadoop/yarn-site.xml>>>
-
- * Configurations for ResourceManager and NodeManager:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.acl.enable>>> | | |
-| | <<<true>>> / <<<false>>> | |
-| | | Enable ACLs? Defaults to <false>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.admin.acl>>> | | |
-| | Admin ACL | |
-| | | ACL to set admins on the cluster. |
-| | | ACLs are of for <comma-separated-users><space><comma-separated-groups>. |
-| | | Defaults to special value of <<*>> which means <anyone>. |
-| | | Special value of just <space> means no one has access. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.log-aggregation-enable>>> | | |
-| | <false> | |
-| | | Configuration to enable or disable log aggregation |
-*-------------------------+-------------------------+------------------------+
-
-
- * Configurations for ResourceManager:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.address>>> | | |
-| | <<<ResourceManager>>> host:port for clients to submit jobs. | |
-| | | <host:port>\ |
-| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.scheduler.address>>> | | |
-| | <<<ResourceManager>>> host:port for ApplicationMasters to talk to | |
-| | Scheduler to obtain resources. | |
-| | | <host:port>\ |
-| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.resource-tracker.address>>> | | |
-| | <<<ResourceManager>>> host:port for NodeManagers. | |
-| | | <host:port>\ |
-| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.admin.address>>> | | |
-| | <<<ResourceManager>>> host:port for administrative commands. | |
-| | | <host:port>\ |
-| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.webapp.address>>> | | |
-| | <<<ResourceManager>>> web-ui host:port. | |
-| | | <host:port>\ |
-| | | If set, overrides the hostname set in <<<yarn.resourcemanager.hostname>>>. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.hostname>>> | | |
-| | <<<ResourceManager>>> host. | |
-| | | <host>\ |
-| | | Single hostname that can be set in place of setting all <<<yarn.resourcemanager*address>>> resources. Results in default ports for ResourceManager components. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.scheduler.class>>> | | |
-| | <<<ResourceManager>>> Scheduler class. | |
-| | | <<<CapacityScheduler>>> (recommended), <<<FairScheduler>>> (also recommended), or <<<FifoScheduler>>> |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.scheduler.minimum-allocation-mb>>> | | |
-| | Minimum limit of memory to allocate to each container request at the <<<Resource Manager>>>. | |
-| | | In MBs |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.scheduler.maximum-allocation-mb>>> | | |
-| | Maximum limit of memory to allocate to each container request at the <<<Resource Manager>>>. | |
-| | | In MBs |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.resourcemanager.nodes.include-path>>> / | | |
-| <<<yarn.resourcemanager.nodes.exclude-path>>> | | |
-| | List of permitted/excluded NodeManagers. | |
-| | | If necessary, use these files to control the list of allowable |
-| | | NodeManagers. |
-*-------------------------+-------------------------+------------------------+
-
- * Configurations for NodeManager:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.resource.memory-mb>>> | | |
-| | Resource i.e. available physical memory, in MB, for given <<<NodeManager>>> | |
-| | | Defines total available resources on the <<<NodeManager>>> to be made |
-| | | available to running containers |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.vmem-pmem-ratio>>> | | |
-| | Maximum ratio by which virtual memory usage of tasks may exceed |
-| | physical memory | |
-| | | The virtual memory usage of each task may exceed its physical memory |
-| | | limit by this ratio. The total amount of virtual memory used by tasks |
-| | | on the NodeManager may exceed its physical memory usage by this ratio. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.local-dirs>>> | | |
-| | Comma-separated list of paths on the local filesystem where | |
-| | intermediate data is written. ||
-| | | Multiple paths help spread disk i/o. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.log-dirs>>> | | |
-| | Comma-separated list of paths on the local filesystem where logs | |
-| | are written. | |
-| | | Multiple paths help spread disk i/o. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.log.retain-seconds>>> | | |
-| | <10800> | |
-| | | Default time (in seconds) to retain log files on the NodeManager |
-| | | Only applicable if log-aggregation is disabled. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.remote-app-log-dir>>> | | |
-| | </logs> | |
-| | | HDFS directory where the application logs are moved on application |
-| | | completion. Need to set appropriate permissions. |
-| | | Only applicable if log-aggregation is enabled. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.remote-app-log-dir-suffix>>> | | |
-| | <logs> | |
-| | | Suffix appended to the remote log dir. Logs will be aggregated to |
-| | | $\{yarn.nodemanager.remote-app-log-dir\}/$\{user\}/$\{thisParam\} |
-| | | Only applicable if log-aggregation is enabled. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.aux-services>>> | | |
-| | mapreduce_shuffle | |
-| | | Shuffle service that needs to be set for Map Reduce applications. |
-*-------------------------+-------------------------+------------------------+
-
- * Configurations for History Server (Needs to be moved elsewhere):
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.log-aggregation.retain-seconds>>> | | |
-| | <-1> | |
-| | | How long to keep aggregation logs before deleting them. -1 disables. |
-| | | Be careful, set this too small and you will spam the name node. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.log-aggregation.retain-check-interval-seconds>>> | | |
-| | <-1> | |
-| | | Time between checks for aggregated log retention. If set to 0 or a |
-| | | negative value then the value is computed as one-tenth of the |
-| | | aggregated log retention time. |
-| | | Be careful, set this too small and you will spam the name node. |
-*-------------------------+-------------------------+------------------------+
-
- * <<<etc/hadoop/mapred-site.xml>>>
-
- * Configurations for MapReduce Applications:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.framework.name>>> | | |
-| | yarn | |
-| | | Execution framework set to Hadoop YARN. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.map.memory.mb>>> | 1536 | |
-| | | Larger resource limit for maps. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.map.java.opts>>> | -Xmx1024M | |
-| | | Larger heap-size for child jvms of maps. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.reduce.memory.mb>>> | 3072 | |
-| | | Larger resource limit for reduces. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.reduce.java.opts>>> | -Xmx2560M | |
-| | | Larger heap-size for child jvms of reduces. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.task.io.sort.mb>>> | 512 | |
-| | | Higher memory-limit while sorting data for efficiency. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.task.io.sort.factor>>> | 100 | |
-| | | More streams merged at once while sorting files. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.reduce.shuffle.parallelcopies>>> | 50 | |
-| | | Higher number of parallel copies run by reduces to fetch outputs |
-| | | from very large number of maps. |
-*-------------------------+-------------------------+------------------------+
-
- * Configurations for MapReduce JobHistory Server:
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.address>>> | | |
-| | MapReduce JobHistory Server <host:port> | Default port is 10020. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.webapp.address>>> | | |
-| | MapReduce JobHistory Server Web UI <host:port> | Default port is 19888. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.intermediate-done-dir>>> | /mr-history/tmp | |
-| | | Directory where history files are written by MapReduce jobs. |
-*-------------------------+-------------------------+------------------------+
-| <<<mapreduce.jobhistory.done-dir>>> | /mr-history/done| |
-| | | Directory where history files are managed by the MR JobHistory Server. |
-*-------------------------+-------------------------+------------------------+
-
-* {Monitoring Health of NodeManagers}
-
- Hadoop provides a mechanism by which administrators can configure the
- NodeManager to run an administrator supplied script periodically to
- determine if a node is healthy or not.
-
- Administrators can determine if the node is in a healthy state by
- performing any checks of their choice in the script. If the script
- detects the node to be in an unhealthy state, it must print a line to
- standard output beginning with the string ERROR. The NodeManager spawns
- the script periodically and checks its output. If the script's output
- contains the string ERROR, as described above, the node's status is
- reported as <<<unhealthy>>> and the node is black-listed by the
- ResourceManager. No further tasks will be assigned to this node.
- However, the NodeManager continues to run the script, so that if the
- node becomes healthy again, it will be removed from the blacklisted nodes
- on the ResourceManager automatically. The node's health along with the
- output of the script, if it is unhealthy, is available to the
- administrator in the ResourceManager web interface. The time since the
- node was healthy is also displayed on the web interface.
-
- The following parameters can be used to control the node health
- monitoring script in <<<etc/hadoop/yarn-site.xml>>>.
-
-*-------------------------+-------------------------+------------------------+
-|| Parameter || Value || Notes |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.health-checker.script.path>>> | | |
-| | Node health script | |
-| | | Script to check for node's health status. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.health-checker.script.opts>>> | | |
-| | Node health script options | |
-| | | Options for script to check for node's health status. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.health-checker.script.interval-ms>>> | | |
-| | Node health script interval | |
-| | | Time interval for running health script. |
-*-------------------------+-------------------------+------------------------+
-| <<<yarn.nodemanager.health-checker.script.timeout-ms>>> | | |
-| | Node health script timeout interval | |
-| | | Timeout for health script execution. |
-*-------------------------+-------------------------+------------------------+
-
- The health checker script is not supposed to give ERROR if only some of the
- local disks become bad. NodeManager has the ability to periodically check
- the health of the local disks (specifically checks nodemanager-local-dirs
- and nodemanager-log-dirs) and after reaching the threshold of number of
- bad directories based on the value set for the config property
- yarn.nodemanager.disk-health-checker.min-healthy-disks, the whole node is
- marked unhealthy and this info is sent to resource manager also. The boot
- disk is either raided or a failure in the boot disk is identified by the
- health checker script.
-
-* {Slaves File}
-
- List all slave hostnames or IP addresses in your <<<etc/hadoop/slaves>>>
- file, one per line. Helper scripts (described below) will use the
- <<<etc/hadoop/slaves>>> file to run commands on many hosts at once. It is not
- used for any of the Java-based Hadoop configuration. In order
- to use this functionality, ssh trusts (via either passphraseless ssh or
- some other means, such as Kerberos) must be established for the accounts
- used to run Hadoop.
-
-* {Hadoop Rack Awareness}
-
- Many Hadoop components are rack-aware and take advantage of the
- network topology for performance and safety. Hadoop daemons obtain the
- rack information of the slaves in the cluster by invoking an administrator
- configured module. See the {{{./RackAwareness.html}Rack Awareness}}
- documentation for more specific information.
-
- It is highly recommended configuring rack awareness prior to starting HDFS.
-
-* {Logging}
-
- Hadoop uses the {{{http://logging.apache.org/log4j/2.x/}Apache log4j}} via the Apache Commons Logging framework for
- logging. Edit the <<<etc/hadoop/log4j.properties>>> file to customize the
- Hadoop daemons' logging configuration (log-formats and so on).
-
-* {Operating the Hadoop Cluster}
-
- Once all the necessary configuration is complete, distribute the files to the
- <<<HADOOP_CONF_DIR>>> directory on all the machines. This should be the
- same directory on all machines.
-
- In general, it is recommended that HDFS and YARN run as separate users.
- In the majority of installations, HDFS processes execute as 'hdfs'. YARN
- is typically using the 'yarn' account.
-
-** Hadoop Startup
-
- To start a Hadoop cluster you will need to start both the HDFS and YARN
- cluster.
-
- The first time you bring up HDFS, it must be formatted. Format a new
- distributed filesystem as <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/bin/hdfs namenode -format <cluster_name>
-----
-
- Start the HDFS NameNode with the following command on the
- designated node as <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start namenode
-----
-
- Start a HDFS DataNode with the following command on each
- designated node as <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon start datanode
-----
-
- If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
- (see {{{./SingleCluster.html}Single Node Setup}}), all of the
- HDFS processes can be started with a utility script. As <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/sbin/start-dfs.sh
-----
-
- Start the YARN with the following command, run on the designated
- ResourceManager as <yarn>:
-
-----
-[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start resourcemanager
-----
-
- Run a script to start a NodeManager on each designated host as <yarn>:
-
-----
-[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start nodemanager
-----
-
- Start a standalone WebAppProxy server. Run on the WebAppProxy
- server as <yarn>. If multiple servers are used with load balancing
- it should be run on each of them:
-
-----
-[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon start proxyserver
-----
-
- If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
- (see {{{./SingleCluster.html}Single Node Setup}}), all of the
- YARN processes can be started with a utility script. As <yarn>:
-
-----
-[yarn]$ $HADOOP_PREFIX/sbin/start-yarn.sh
-----
-
- Start the MapReduce JobHistory Server with the following command, run
- on the designated server as <mapred>:
-
-----
-[mapred]$ $HADOOP_PREFIX/bin/mapred --daemon start historyserver
-----
-
-** Hadoop Shutdown
-
- Stop the NameNode with the following command, run on the designated NameNode
- as <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop namenode
-----
-
- Run a script to stop a DataNode as <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/bin/hdfs --daemon stop datanode
-----
-
- If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
- (see {{{./SingleCluster.html}Single Node Setup}}), all of the
- HDFS processes may be stopped with a utility script. As <hdfs>:
-
-----
-[hdfs]$ $HADOOP_PREFIX/sbin/stop-dfs.sh
-----
-
- Stop the ResourceManager with the following command, run on the designated
- ResourceManager as <yarn>:
-
-----
-[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop resourcemanager
-----
-
- Run a script to stop a NodeManager on a slave as <yarn>:
-
-----
-[yarn]$ $HADOOP_PREFIX/bin/yarn --daemon stop nodemanager
-----
-
- If <<<etc/hadoop/slaves>>> and ssh trusted access is configured
- (see {{{./SingleCluster.html}Single Node Setup}}), all of the
- YARN processes can be stopped with a utility script. As <yarn>:
-
-----
-[yarn]$ $HADOOP_PREFIX/sbin/stop-yarn.sh
-----
-
- Stop the WebAppProxy server. Run on the WebAppProxy server as
- <yarn>. If multiple servers are used with load balancing it
- should be run on each of them:
-
-----
-[yarn]$ $HADOOP_PREFIX/bin/yarn stop proxyserver
-----
-
- Stop the MapReduce JobHistory Server with the following command, run on the
- designated server as <mapred>:
-
-----
-[mapred]$ $HADOOP_PREFIX/bin/mapred --daemon stop historyserver
-----
-
-* {Web Interfaces}
-
- Once the Hadoop cluster is up and running check the web-ui of the
- components as described below:
-
-*-------------------------+-------------------------+------------------------+
-|| Daemon || Web Interface || Notes |
-*-------------------------+-------------------------+------------------------+
-| NameNode | http://<nn_host:port>/ | Default HTTP port is 50070. |
-*-------------------------+-------------------------+------------------------+
-| ResourceManager | http://<rm_host:port>/ | Default HTTP port is 8088. |
-*-------------------------+-------------------------+------------------------+
-| MapReduce JobHistory Server | http://<jhs_host:port>/ | |
-| | | Default HTTP port is 19888. |
-*-------------------------+-------------------------+------------------------+
-
-
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
deleted file mode 100644
index 67c8bc3..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/CommandsManual.apt.vm
+++ /dev/null
@@ -1,327 +0,0 @@
-~~ Licensed to the Apache Software Foundation (ASF) under one or more
-~~ contributor license agreements. See the NOTICE file distributed with
-~~ this work for additional information regarding copyright ownership.
-~~ The ASF licenses this file to You under the Apache License, Version 2.0
-~~ (the "License"); you may not use this file except in compliance with
-~~ the License. You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License.
-
- ---
- Hadoop Commands Guide
- ---
- ---
- ${maven.build.timestamp}
-
-%{toc}
-
-Hadoop Commands Guide
-
-* Overview
-
- All of the Hadoop commands and subprojects follow the same basic structure:
-
- Usage: <<<shellcommand [SHELL_OPTIONS] [COMMAND] [GENERIC_OPTIONS] [COMMAND_OPTIONS]>>>
-
-*--------+---------+
-|| FIELD || Description
-*-----------------------+---------------+
-| shellcommand | The command of the project being invoked. For example,
- | Hadoop common uses <<<hadoop>>>, HDFS uses <<<hdfs>>>,
- | and YARN uses <<<yarn>>>.
-*---------------+-------------------+
-| SHELL_OPTIONS | Options that the shell processes prior to executing Java.
-*-----------------------+---------------+
-| COMMAND | Action to perform.
-*-----------------------+---------------+
-| GENERIC_OPTIONS | The common set of options supported by
- | multiple commands.
-*-----------------------+---------------+
-| COMMAND_OPTIONS | Various commands with their options are
- | described in this documention for the
- | Hadoop common sub-project. HDFS and YARN are
- | covered in other documents.
-*-----------------------+---------------+
-
-** {Shell Options}
-
- All of the shell commands will accept a common set of options. For some commands,
- these options are ignored. For example, passing <<<---hostnames>>> on a
- command that only executes on a single host will be ignored.
-
-*-----------------------+---------------+
-|| SHELL_OPTION || Description
-*-----------------------+---------------+
-| <<<--buildpaths>>> | Enables developer versions of jars.
-*-----------------------+---------------+
-| <<<--config confdir>>> | Overwrites the default Configuration
- | directory. Default is <<<${HADOOP_PREFIX}/conf>>>.
-*-----------------------+----------------+
-| <<<--daemon mode>>> | If the command supports daemonization (e.g.,
- | <<<hdfs namenode>>>), execute in the appropriate
- | mode. Supported modes are <<<start>>> to start the
- | process in daemon mode, <<<stop>>> to stop the
- | process, and <<<status>>> to determine the active
- | status of the process. <<<status>>> will return
- | an {{{http://refspecs.linuxbase.org/LSB_3.0.0/LSB-generic/LSB-generic/iniscrptact.html}LSB-compliant}} result code.
- | If no option is provided, commands that support
- | daemonization will run in the foreground.
-*-----------------------+---------------+
-| <<<--debug>>> | Enables shell level configuration debugging information
-*-----------------------+---------------+
-| <<<--help>>> | Shell script usage information.
-*-----------------------+---------------+
-| <<<--hostnames>>> | A space delimited list of hostnames where to execute
- | a multi-host subcommand. By default, the content of
- | the <<<slaves>>> file is used.
-*-----------------------+----------------+
-| <<<--hosts>>> | A file that contains a list of hostnames where to execute
- | a multi-host subcommand. By default, the content of the
- | <<<slaves>>> file is used.
-*-----------------------+----------------+
-| <<<--loglevel loglevel>>> | Overrides the log level. Valid log levels are
-| | FATAL, ERROR, WARN, INFO, DEBUG, and TRACE.
-| | Default is INFO.
-*-----------------------+---------------+
-
-** {Generic Options}
-
- Many subcommands honor a common set of configuration options to alter their behavior:
-
-*------------------------------------------------+-----------------------------+
-|| GENERIC_OPTION || Description
-*------------------------------------------------+-----------------------------+
-|<<<-archives \<comma separated list of archives\> >>> | Specify comma separated
- | archives to be unarchived on
- | the compute machines. Applies
- | only to job.
-*------------------------------------------------+-----------------------------+
-|<<<-conf \<configuration file\> >>> | Specify an application
- | configuration file.
-*------------------------------------------------+-----------------------------+
-|<<<-D \<property\>=\<value\> >>> | Use value for given property.
-*------------------------------------------------+-----------------------------+
-|<<<-files \<comma separated list of files\> >>> | Specify comma separated files
- | to be copied to the map
- | reduce cluster. Applies only
- | to job.
-*------------------------------------------------+-----------------------------+
-|<<<-jt \<local\> or \<resourcemanager:port\>>>> | Specify a ResourceManager.
- | Applies only to job.
-*------------------------------------------------+-----------------------------+
-|<<<-libjars \<comma seperated list of jars\> >>>| Specify comma separated jar
- | files to include in the
- | classpath. Applies only to
- | job.
-*------------------------------------------------+-----------------------------+
-
-Hadoop Common Commands
-
- All of these commands are executed from the <<<hadoop>>> shell command. They
- have been broken up into {{User Commands}} and
- {{Admininistration Commands}}.
-
-* User Commands
-
- Commands useful for users of a hadoop cluster.
-
-** <<<archive>>>
-
- Creates a hadoop archive. More information can be found at
- {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/HadoopArchives.html}
- Hadoop Archives Guide}}.
-
-** <<<checknative>>>
-
- Usage: <<<hadoop checknative [-a] [-h] >>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| -a | Check all libraries are available.
-*-----------------+-----------------------------------------------------------+
-| -h | print help
-*-----------------+-----------------------------------------------------------+
-
- This command checks the availability of the Hadoop native code. See
- {{{NativeLibraries.html}}} for more information. By default, this command
- only checks the availability of libhadoop.
-
-** <<<classpath>>>
-
- Usage: <<<hadoop classpath [--glob|--jar <path>|-h|--help]>>>
-
-*-----------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*-----------------+-----------------------------------------------------------+
-| --glob | expand wildcards
-*-----------------+-----------------------------------------------------------+
-| --jar <path> | write classpath as manifest in jar named <path>
-*-----------------+-----------------------------------------------------------+
-| -h, --help | print help
-*-----------------+-----------------------------------------------------------+
-
- Prints the class path needed to get the Hadoop jar and the required
- libraries. If called without arguments, then prints the classpath set up by
- the command scripts, which is likely to contain wildcards in the classpath
- entries. Additional options print the classpath after wildcard expansion or
- write the classpath into the manifest of a jar file. The latter is useful in
- environments where wildcards cannot be used and the expanded classpath exceeds
- the maximum supported command line length.
-
-** <<<credential>>>
-
- Usage: <<<hadoop credential <subcommand> [options]>>>
-
-*-------------------+-------------------------------------------------------+
-||COMMAND_OPTION || Description
-*-------------------+-------------------------------------------------------+
-| create <alias> [-v <value>][-provider <provider-path>]| Prompts the user for
- | a credential to be stored as the given alias when a value
- | is not provided via <<<-v>>>. The
- | <hadoop.security.credential.provider.path> within the
- | core-site.xml file will be used unless a <<<-provider>>> is
- | indicated.
-*-------------------+-------------------------------------------------------+
-| delete <alias> [-i][-provider <provider-path>] | Deletes the credential with
- | the provided alias and optionally warns the user when
- | <<<--interactive>>> is used.
- | The <hadoop.security.credential.provider.path> within the
- | core-site.xml file will be used unless a <<<-provider>>> is
- | indicated.
-*-------------------+-------------------------------------------------------+
-| list [-provider <provider-path>] | Lists all of the credential aliases
- | The <hadoop.security.credential.provider.path> within the
- | core-site.xml file will be used unless a <<<-provider>>> is
- | indicated.
-*-------------------+-------------------------------------------------------+
-
- Command to manage credentials, passwords and secrets within credential providers.
-
- The CredentialProvider API in Hadoop allows for the separation of applications
- and how they store their required passwords/secrets. In order to indicate
- a particular provider type and location, the user must provide the
- <hadoop.security.credential.provider.path> configuration element in core-site.xml
- or use the command line option <<<-provider>>> on each of the following commands.
- This provider path is a comma-separated list of URLs that indicates the type and
- location of a list of providers that should be consulted. For example, the following path:
- <<<user:///,jceks://file/tmp/test.jceks,jceks://hdfs@nn1.example.com/my/path/test.jceks>>>
-
- indicates that the current user's credentials file should be consulted through
- the User Provider, that the local file located at <<</tmp/test.jceks>>> is a Java Keystore
- Provider and that the file located within HDFS at <<<nn1.example.com/my/path/test.jceks>>>
- is also a store for a Java Keystore Provider.
-
- When utilizing the credential command it will often be for provisioning a password
- or secret to a particular credential store provider. In order to explicitly
- indicate which provider store to use the <<<-provider>>> option should be used. Otherwise,
- given a path of multiple providers, the first non-transient provider will be used.
- This may or may not be the one that you intended.
-
- Example: <<<-provider jceks://file/tmp/test.jceks>>>
-
-** <<<distch>>>
-
- Usage: <<<hadoop distch [-f urilist_url] [-i] [-log logdir] path:owner:group:permissions>>>
-
-*-------------------+-------------------------------------------------------+
-||COMMAND_OPTION || Description
-*-------------------+-------------------------------------------------------+
-| -f | List of objects to change
-*----+------------+
-| -i | Ignore failures
-*----+------------+
-| -log | Directory to log output
-*-----+---------+
-
- Change the ownership and permissions on many files at once.
-
-** <<<distcp>>>
-
- Copy file or directories recursively. More information can be found at
- {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/DistCp.html}
- Hadoop DistCp Guide}}.
-
-** <<<fs>>>
-
- This command is documented in the {{{./FileSystemShell.html}File System Shell Guide}}. It is a synonym for <<<hdfs dfs>>> when HDFS is in use.
-
-** <<<jar>>>
-
- Usage: <<<hadoop jar <jar> [mainClass] args...>>>
-
- Runs a jar file.
-
- Use {{{../../hadoop-yarn/hadoop-yarn-site/YarnCommands.html#jar}<<<yarn jar>>>}}
- to launch YARN applications instead.
-
-** <<<jnipath>>>
-
- Usage: <<<hadoop jnipath>>>
-
- Print the computed java.library.path.
-
-** <<<key>>>
-
- Manage keys via the KeyProvider.
-
-** <<<trace>>>
-
- View and modify Hadoop tracing settings. See the {{{./Tracing.html}Tracing Guide}}.
-
-** <<<version>>>
-
- Usage: <<<hadoop version>>>
-
- Prints the version.
-
-** <<<CLASSNAME>>>
-
- Usage: <<<hadoop CLASSNAME>>>
-
- Runs the class named <<<CLASSNAME>>>. The class must be part of a package.
-
-* {Administration Commands}
-
- Commands useful for administrators of a hadoop cluster.
-
-** <<<daemonlog>>>
-
- Usage: <<<hadoop daemonlog -getlevel <host:port> <name> >>>
- Usage: <<<hadoop daemonlog -setlevel <host:port> <name> <level> >>>
-
-*------------------------------+-----------------------------------------------------------+
-|| COMMAND_OPTION || Description
-*------------------------------+-----------------------------------------------------------+
-| -getlevel <host:port> <name> | Prints the log level of the daemon running at
- | <host:port>. This command internally connects
- | to http://<host:port>/logLevel?log=<name>
-*------------------------------+-----------------------------------------------------------+
-| -setlevel <host:port> <name> <level> | Sets the log level of the daemon
- | running at <host:port>. This command internally
- | connects to http://<host:port>/logLevel?log=<name>
-*------------------------------+-----------------------------------------------------------+
-
- Get/Set the log level for each daemon.
-
-* Files
-
-** <<etc/hadoop/hadoop-env.sh>>
-
- This file stores the global settings used by all Hadoop shell commands.
-
-** <<etc/hadoop/hadoop-user-functions.sh>>
-
- This file allows for advanced users to override some shell functionality.
-
-** <<~/.hadooprc>>
-
- This stores the personal environment for an individual user. It is
- processed after the hadoop-env.sh and hadoop-user-functions.sh files
- and can contain the same settings.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e9d26fe9/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm b/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm
deleted file mode 100644
index 98d1f57..0000000
--- a/hadoop-common-project/hadoop-common/src/site/apt/Compatibility.apt.vm
+++ /dev/null
@@ -1,541 +0,0 @@
-~~ Licensed under the Apache License, Version 2.0 (the "License");
-~~ you may not use this file except in compliance with the License.
-~~ You may obtain a copy of the License at
-~~
-~~ http://www.apache.org/licenses/LICENSE-2.0
-~~
-~~ Unless required by applicable law or agreed to in writing, software
-~~ distributed under the License is distributed on an "AS IS" BASIS,
-~~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-~~ See the License for the specific language governing permissions and
-~~ limitations under the License. See accompanying LICENSE file.
-
- ---
-Apache Hadoop Compatibility
- ---
- ---
- ${maven.build.timestamp}
-
-Apache Hadoop Compatibility
-
-%{toc|section=1|fromDepth=0}
-
-* Purpose
-
- This document captures the compatibility goals of the Apache Hadoop
- project. The different types of compatibility between Hadoop
- releases that affects Hadoop developers, downstream projects, and
- end-users are enumerated. For each type of compatibility we:
-
- * describe the impact on downstream projects or end-users
-
- * where applicable, call out the policy adopted by the Hadoop
- developers when incompatible changes are permitted.
-
-* Compatibility types
-
-** Java API
-
- Hadoop interfaces and classes are annotated to describe the intended
- audience and stability in order to maintain compatibility with previous
- releases. See {{{./InterfaceClassification.html}Hadoop Interface
- Classification}}
- for details.
-
- * InterfaceAudience: captures the intended audience, possible
- values are Public (for end users and external projects),
- LimitedPrivate (for other Hadoop components, and closely related
- projects like YARN, MapReduce, HBase etc.), and Private (for intra component
- use).
-
- * InterfaceStability: describes what types of interface changes are
- permitted. Possible values are Stable, Evolving, Unstable, and Deprecated.
-
-*** Use Cases
-
- * Public-Stable API compatibility is required to ensure end-user programs
- and downstream projects continue to work without modification.
-
- * LimitedPrivate-Stable API compatibility is required to allow upgrade of
- individual components across minor releases.
-
- * Private-Stable API compatibility is required for rolling upgrades.
-
-*** Policy
-
- * Public-Stable APIs must be deprecated for at least one major release
- prior to their removal in a major release.
-
- * LimitedPrivate-Stable APIs can change across major releases,
- but not within a major release.
-
- * Private-Stable APIs can change across major releases,
- but not within a major release.
-
- * Classes not annotated are implicitly "Private". Class members not
- annotated inherit the annotations of the enclosing class.
-
- * Note: APIs generated from the proto files need to be compatible for
- rolling-upgrades. See the section on wire-compatibility for more details.
- The compatibility policies for APIs and wire-communication need to go
- hand-in-hand to address this.
-
-** Semantic compatibility
-
- Apache Hadoop strives to ensure that the behavior of APIs remains
- consistent over versions, though changes for correctness may result in
- changes in behavior. Tests and javadocs specify the API's behavior.
- The community is in the process of specifying some APIs more rigorously,
- and enhancing test suites to verify compliance with the specification,
- effectively creating a formal specification for the subset of behaviors
- that can be easily tested.
-
-*** Policy
-
- The behavior of API may be changed to fix incorrect behavior,
- such a change to be accompanied by updating existing buggy tests or adding
- tests in cases there were none prior to the change.
-
-** Wire compatibility
-
- Wire compatibility concerns data being transmitted over the wire
- between Hadoop processes. Hadoop uses Protocol Buffers for most RPC
- communication. Preserving compatibility requires prohibiting
- modification as described below.
- Non-RPC communication should be considered as well,
- for example using HTTP to transfer an HDFS image as part of
- snapshotting or transferring MapTask output. The potential
- communications can be categorized as follows:
-
- * Client-Server: communication between Hadoop clients and servers (e.g.,
- the HDFS client to NameNode protocol, or the YARN client to
- ResourceManager protocol).
-
- * Client-Server (Admin): It is worth distinguishing a subset of the
- Client-Server protocols used solely by administrative commands (e.g.,
- the HAAdmin protocol) as these protocols only impact administrators
- who can tolerate changes that end users (which use general
- Client-Server protocols) can not.
-
- * Server-Server: communication between servers (e.g., the protocol between
- the DataNode and NameNode, or NodeManager and ResourceManager)
-
-*** Use Cases
-
- * Client-Server compatibility is required to allow users to
- continue using the old clients even after upgrading the server
- (cluster) to a later version (or vice versa). For example, a
- Hadoop 2.1.0 client talking to a Hadoop 2.3.0 cluster.
-
- * Client-Server compatibility is also required to allow users to upgrade the
- client before upgrading the server (cluster). For example, a Hadoop 2.4.0
- client talking to a Hadoop 2.3.0 cluster. This allows deployment of
- client-side bug fixes ahead of full cluster upgrades. Note that new cluster
- features invoked by new client APIs or shell commands will not be usable.
- YARN applications that attempt to use new APIs (including new fields in data
- structures) that have not yet deployed to the cluster can expect link
- exceptions.
-
- * Client-Server compatibility is also required to allow upgrading
- individual components without upgrading others. For example,
- upgrade HDFS from version 2.1.0 to 2.2.0 without upgrading MapReduce.
-
- * Server-Server compatibility is required to allow mixed versions
- within an active cluster so the cluster may be upgraded without
- downtime in a rolling fashion.
-
-*** Policy
-
- * Both Client-Server and Server-Server compatibility is preserved within a
- major release. (Different policies for different categories are yet to be
- considered.)
-
- * Compatibility can be broken only at a major release, though breaking compatibility
- even at major releases has grave consequences and should be discussed in the Hadoop community.
-
- * Hadoop protocols are defined in .proto (ProtocolBuffers) files.
- Client-Server protocols and Server-protocol .proto files are marked as stable.
- When a .proto file is marked as stable it means that changes should be made
- in a compatible fashion as described below:
-
- * The following changes are compatible and are allowed at any time:
-
- * Add an optional field, with the expectation that the code deals with the field missing due to communication with an older version of the code.
-
- * Add a new rpc/method to the service
-
- * Add a new optional request to a Message
-
- * Rename a field
-
- * Rename a .proto file
-
- * Change .proto annotations that effect code generation (e.g. name of java package)
-
- * The following changes are incompatible but can be considered only at a major release
-
- * Change the rpc/method name
-
- * Change the rpc/method parameter type or return type
-
- * Remove an rpc/method
-
- * Change the service name
-
- * Change the name of a Message
-
- * Modify a field type in an incompatible way (as defined recursively)
-
- * Change an optional field to required
-
- * Add or delete a required field
-
- * Delete an optional field as long as the optional field has reasonable defaults to allow deletions
-
- * The following changes are incompatible and hence never allowed
-
- * Change a field id
-
- * Reuse an old field that was previously deleted.
-
- * Field numbers are cheap and changing and reusing is not a good idea.
-
-
-** Java Binary compatibility for end-user applications i.e. Apache Hadoop ABI
-
- As Apache Hadoop revisions are upgraded end-users reasonably expect that
- their applications should continue to work without any modifications.
- This is fulfilled as a result of support API compatibility, Semantic
- compatibility and Wire compatibility.
-
- However, Apache Hadoop is a very complex, distributed system and services a
- very wide variety of use-cases. In particular, Apache Hadoop MapReduce is a
- very, very wide API; in the sense that end-users may make wide-ranging
- assumptions such as layout of the local disk when their map/reduce tasks are
- executing, environment variables for their tasks etc. In such cases, it
- becomes very hard to fully specify, and support, absolute compatibility.
-
-*** Use cases
-
- * Existing MapReduce applications, including jars of existing packaged
- end-user applications and projects such as Apache Pig, Apache Hive,
- Cascading etc. should work unmodified when pointed to an upgraded Apache
- Hadoop cluster within a major release.
-
- * Existing YARN applications, including jars of existing packaged
- end-user applications and projects such as Apache Tez etc. should work
- unmodified when pointed to an upgraded Apache Hadoop cluster within a
- major release.
-
- * Existing applications which transfer data in/out of HDFS, including jars
- of existing packaged end-user applications and frameworks such as Apache
- Flume, should work unmodified when pointed to an upgraded Apache Hadoop
- cluster within a major release.
-
-*** Policy
-
- * Existing MapReduce, YARN & HDFS applications and frameworks should work
- unmodified within a major release i.e. Apache Hadoop ABI is supported.
-
- * A very minor fraction of applications maybe affected by changes to disk
- layouts etc., the developer community will strive to minimize these
- changes and will not make them within a minor version. In more egregious
- cases, we will consider strongly reverting these breaking changes and
- invalidating offending releases if necessary.
-
- * In particular for MapReduce applications, the developer community will
- try our best to support provide binary compatibility across major
- releases e.g. applications using org.apache.hadoop.mapred.
-
- * APIs are supported compatibly across hadoop-1.x and hadoop-2.x. See
- {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html}
- Compatibility for MapReduce applications between hadoop-1.x and hadoop-2.x}}
- for more details.
-
-** REST APIs
-
- REST API compatibility corresponds to both the request (URLs) and responses
- to each request (content, which may contain other URLs). Hadoop REST APIs
- are specifically meant for stable use by clients across releases,
- even major releases. The following are the exposed REST APIs:
-
- * {{{../hadoop-hdfs/WebHDFS.html}WebHDFS}} - Stable
-
- * {{{../../hadoop-yarn/hadoop-yarn-site/ResourceManagerRest.html}ResourceManager}}
-
- * {{{../../hadoop-yarn/hadoop-yarn-site/NodeManagerRest.html}NodeManager}}
-
- * {{{../../hadoop-yarn/hadoop-yarn-site/MapredAppMasterRest.html}MR Application Master}}
-
- * {{{../../hadoop-yarn/hadoop-yarn-site/HistoryServerRest.html}History Server}}
-
-*** Policy
-
- The APIs annotated stable in the text above preserve compatibility
- across at least one major release, and maybe deprecated by a newer
- version of the REST API in a major release.
-
-** Metrics/JMX
-
- While the Metrics API compatibility is governed by Java API compatibility,
- the actual metrics exposed by Hadoop need to be compatible for users to
- be able to automate using them (scripts etc.). Adding additional metrics
- is compatible. Modifying (eg changing the unit or measurement) or removing
- existing metrics breaks compatibility. Similarly, changes to JMX MBean
- object names also break compatibility.
-
-*** Policy
-
- Metrics should preserve compatibility within the major release.
-
-** File formats & Metadata
-
- User and system level data (including metadata) is stored in files of
- different formats. Changes to the metadata or the file formats used to
- store data/metadata can lead to incompatibilities between versions.
-
-*** User-level file formats
-
- Changes to formats that end-users use to store their data can prevent
- them for accessing the data in later releases, and hence it is highly
- important to keep those file-formats compatible. One can always add a
- "new" format improving upon an existing format. Examples of these formats
- include har, war, SequenceFileFormat etc.
-
-**** Policy
-
- * Non-forward-compatible user-file format changes are
- restricted to major releases. When user-file formats change, new
- releases are expected to read existing formats, but may write data
- in formats incompatible with prior releases. Also, the community
- shall prefer to create a new format that programs must opt in to
- instead of making incompatible changes to existing formats.
-
-*** System-internal file formats
-
- Hadoop internal data is also stored in files and again changing these
- formats can lead to incompatibilities. While such changes are not as
- devastating as the user-level file formats, a policy on when the
- compatibility can be broken is important.
-
-**** MapReduce
-
- MapReduce uses formats like I-File to store MapReduce-specific data.
-
-
-***** Policy
-
- MapReduce-internal formats like IFile maintain compatibility within a
- major release. Changes to these formats can cause in-flight jobs to fail
- and hence we should ensure newer clients can fetch shuffle-data from old
- servers in a compatible manner.
-
-**** HDFS Metadata
-
- HDFS persists metadata (the image and edit logs) in a particular format.
- Incompatible changes to either the format or the metadata prevent
- subsequent releases from reading older metadata. Such incompatible
- changes might require an HDFS "upgrade" to convert the metadata to make
- it accessible. Some changes can require more than one such "upgrades".
-
- Depending on the degree of incompatibility in the changes, the following
- potential scenarios can arise:
-
- * Automatic: The image upgrades automatically, no need for an explicit
- "upgrade".
-
- * Direct: The image is upgradable, but might require one explicit release
- "upgrade".
-
- * Indirect: The image is upgradable, but might require upgrading to
- intermediate release(s) first.
-
- * Not upgradeable: The image is not upgradeable.
-
-***** Policy
-
- * A release upgrade must allow a cluster to roll-back to the older
- version and its older disk format. The rollback needs to restore the
- original data, but not required to restore the updated data.
-
- * HDFS metadata changes must be upgradeable via any of the upgrade
- paths - automatic, direct or indirect.
-
- * More detailed policies based on the kind of upgrade are yet to be
- considered.
-
-** Command Line Interface (CLI)
-
- The Hadoop command line programs may be use either directly via the
- system shell or via shell scripts. Changing the path of a command,
- removing or renaming command line options, the order of arguments,
- or the command return code and output break compatibility and
- may adversely affect users.
-
-*** Policy
-
- CLI commands are to be deprecated (warning when used) for one
- major release before they are removed or incompatibly modified in
- a subsequent major release.
-
-** Web UI
-
- Web UI, particularly the content and layout of web pages, changes
- could potentially interfere with attempts to screen scrape the web
- pages for information.
-
-*** Policy
-
- Web pages are not meant to be scraped and hence incompatible
- changes to them are allowed at any time. Users are expected to use
- REST APIs to get any information.
-
-** Hadoop Configuration Files
-
- Users use (1) Hadoop-defined properties to configure and provide hints to
- Hadoop and (2) custom properties to pass information to jobs. Hence,
- compatibility of config properties is two-fold:
-
- * Modifying key-names, units of values, and default values of Hadoop-defined
- properties.
-
- * Custom configuration property keys should not conflict with the
- namespace of Hadoop-defined properties. Typically, users should
- avoid using prefixes used by Hadoop: hadoop, io, ipc, fs, net,
- file, ftp, s3, kfs, ha, file, dfs, mapred, mapreduce, yarn.
-
-*** Policy
-
- * Hadoop-defined properties are to be deprecated at least for one
- major release before being removed. Modifying units for existing
- properties is not allowed.
-
- * The default values of Hadoop-defined properties can
- be changed across minor/major releases, but will remain the same
- across point releases within a minor release.
-
- * Currently, there is NO explicit policy regarding when new
- prefixes can be added/removed, and the list of prefixes to be
- avoided for custom configuration properties. However, as noted above,
- users should avoid using prefixes used by Hadoop: hadoop, io, ipc, fs,
- net, file, ftp, s3, kfs, ha, file, dfs, mapred, mapreduce, yarn.
-
-** Directory Structure
-
- Source code, artifacts (source and tests), user logs, configuration files,
- output and job history are all stored on disk either local file system or
- HDFS. Changing the directory structure of these user-accessible
- files break compatibility, even in cases where the original path is
- preserved via symbolic links (if, for example, the path is accessed
- by a servlet that is configured to not follow symbolic links).
-
-*** Policy
-
- * The layout of source code and build artifacts can change
- anytime, particularly so across major versions. Within a major
- version, the developers will attempt (no guarantees) to preserve
- the directory structure; however, individual files can be
- added/moved/deleted. The best way to ensure patches stay in sync
- with the code is to get them committed to the Apache source tree.
-
- * The directory structure of configuration files, user logs, and
- job history will be preserved across minor and point releases
- within a major release.
-
-** Java Classpath
-
- User applications built against Hadoop might add all Hadoop jars
- (including Hadoop's library dependencies) to the application's
- classpath. Adding new dependencies or updating the version of
- existing dependencies may interfere with those in applications'
- classpaths.
-
-*** Policy
-
- Currently, there is NO policy on when Hadoop's dependencies can
- change.
-
-** Environment variables
-
- Users and related projects often utilize the exported environment
- variables (eg HADOOP_CONF_DIR), therefore removing or renaming
- environment variables is an incompatible change.
-
-*** Policy
-
- Currently, there is NO policy on when the environment variables
- can change. Developers try to limit changes to major releases.
-
-** Build artifacts
-
- Hadoop uses maven for project management and changing the artifacts
- can affect existing user workflows.
-
-*** Policy
-
- * Test artifacts: The test jars generated are strictly for internal
- use and are not expected to be used outside of Hadoop, similar to
- APIs annotated @Private, @Unstable.
-
- * Built artifacts: The hadoop-client artifact (maven
- groupId:artifactId) stays compatible within a major release,
- while the other artifacts can change in incompatible ways.
-
-** Hardware/Software Requirements
-
- To keep up with the latest advances in hardware, operating systems,
- JVMs, and other software, new Hadoop releases or some of their
- features might require higher versions of the same. For a specific
- environment, upgrading Hadoop might require upgrading other
- dependent software components.
-
-*** Policies
-
- * Hardware
-
- * Architecture: The community has no plans to restrict Hadoop to
- specific architectures, but can have family-specific
- optimizations.
-
- * Minimum resources: While there are no guarantees on the
- minimum resources required by Hadoop daemons, the community
- attempts to not increase requirements within a minor release.
-
- * Operating Systems: The community will attempt to maintain the
- same OS requirements (OS kernel versions) within a minor
- release. Currently GNU/Linux and Microsoft Windows are the OSes officially
- supported by the community while Apache Hadoop is known to work reasonably
- well on other OSes such as Apple MacOSX, Solaris etc.
-
- * The JVM requirements will not change across point releases
- within the same minor release except if the JVM version under
- question becomes unsupported. Minor/major releases might require
- later versions of JVM for some/all of the supported operating
- systems.
-
- * Other software: The community tries to maintain the minimum
- versions of additional software required by Hadoop. For example,
- ssh, kerberos etc.
-
-* References
-
- Here are some relevant JIRAs and pages related to the topic:
-
- * The evolution of this document -
- {{{https://issues.apache.org/jira/browse/HADOOP-9517}HADOOP-9517}}
-
- * Binary compatibility for MapReduce end-user applications between hadoop-1.x and hadoop-2.x -
- {{{../../hadoop-mapreduce-client/hadoop-mapreduce-client-core/MapReduce_Compatibility_Hadoop1_Hadoop2.html}
- MapReduce Compatibility between hadoop-1.x and hadoop-2.x}}
-
- * Annotations for interfaces as per interface classification
- schedule -
- {{{https://issues.apache.org/jira/browse/HADOOP-7391}HADOOP-7391}}
- {{{./InterfaceClassification.html}Hadoop Interface Classification}}
-
- * Compatibility for Hadoop 1.x releases -
- {{{https://issues.apache.org/jira/browse/HADOOP-5071}HADOOP-5071}}
-
- * The {{{http://wiki.apache.org/hadoop/Roadmap}Hadoop Roadmap}} page
- that captures other release policies
-
[06/50] [abbrv] hadoop git commit: HADOOP-8934. Shell command ls
should include sort options (Jonathan Allen via aw) (missed file)
Posted by zj...@apache.org.
HADOOP-8934. Shell command ls should include sort options (Jonathan Allen via aw) (missed file)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/57645980
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/57645980
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/57645980
Branch: refs/heads/YARN-2928
Commit: 576459801c4e21effc4e3bca796527896b6e4f4b
Parents: 84cc071
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 12:54:25 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 12:54:25 2015 -0800
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/shell/TestLs.java | 1308 ++++++++++++++++++
1 file changed, 1308 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/57645980/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
new file mode 100644
index 0000000..66403db
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestLs.java
@@ -0,0 +1,1308 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.shell;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.*;
+
+import java.io.IOException;
+import java.io.PrintStream;
+import java.net.URI;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.LinkedList;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.AclEntry;
+import org.apache.hadoop.fs.permission.AclStatus;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+/**
+ * JUnit test class for {@link org.apache.hadoop.fs.shell.Ls}
+ *
+ */
+public class TestLs {
+ private static Configuration conf;
+ private static FileSystem mockFs;
+
+ private static final Date NOW = new Date();
+
+ @BeforeClass
+ public static void setup() throws IOException {
+ conf = new Configuration();
+ conf.set("fs.defaultFS", "mockfs:///");
+ conf.setClass("fs.mockfs.impl", MockFileSystem.class, FileSystem.class);
+ mockFs = mock(FileSystem.class);
+ }
+
+ @Before
+ public void resetMock() throws IOException {
+ reset(mockFs);
+ AclStatus mockAclStatus = mock(AclStatus.class);
+ when(mockAclStatus.getEntries()).thenReturn(new ArrayList<AclEntry>());
+ when(mockFs.getAclStatus(any(Path.class))).thenReturn(mockAclStatus);
+ }
+
+ // check that default options are correct
+ @Test
+ public void processOptionsNone() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -d option is recognised
+ @Test
+ public void processOptionsDirectory() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-d");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertFalse(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -h option is recognised
+ @Test
+ public void processOptionsHuman() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-h");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertTrue(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -R option is recognised
+ @Test
+ public void processOptionsRecursive() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-R");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertTrue(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -r option is recognised
+ @Test
+ public void processOptionsReverse() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-r");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertTrue(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -S option is recognised
+ @Test
+ public void processOptionsSize() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-S");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertTrue(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the -t option is recognised
+ @Test
+ public void processOptionsMtime() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertTrue(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the precedence of the -t and -S options
+ @Test
+ public void processOptionsMtimeSize() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ options.add("-S");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertTrue(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // check the precedence of the -t, -S and -r options
+ @Test
+ public void processOptionsMtimeSizeReverse() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ options.add("-S");
+ options.add("-r");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertTrue(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertTrue(ls.isOrderTime());
+ assertFalse(ls.isUseAtime());
+ }
+
+ // chheck the -u option is recognised
+ @Test
+ public void processOptionsAtime() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-u");
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertTrue(ls.isDirRecurse());
+ assertFalse(ls.isHumanReadable());
+ assertFalse(ls.isRecursive());
+ assertFalse(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize());
+ assertFalse(ls.isOrderTime());
+ assertTrue(ls.isUseAtime());
+ }
+
+ // check all options is handled correctly
+ @Test
+ public void processOptionsAll() throws IOException {
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-d"); // directory
+ options.add("-h"); // human readable
+ options.add("-R"); // recursive
+ options.add("-r"); // reverse order
+ options.add("-t"); // time order
+ options.add("-S"); // size order
+ options.add("-u"); // show atime
+ Ls ls = new Ls();
+ ls.processOptions(options);
+ assertFalse(ls.isDirRecurse());
+ assertTrue(ls.isHumanReadable());
+ assertFalse(ls.isRecursive()); // -d overrules -R
+ assertTrue(ls.isOrderReverse());
+ assertFalse(ls.isOrderSize()); // -t overrules -S
+ assertTrue(ls.isOrderTime());
+ assertTrue(ls.isUseAtime());
+ }
+
+ // check listing of a single file
+ @Test
+ public void processPathFile() throws IOException {
+ TestFile testfile = new TestFile("testDir", "testFile");
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testfile.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println(testfile.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check listing of multiple files
+ @Test
+ public void processPathFiles() throws IOException {
+ TestFile testfile01 = new TestFile("testDir01", "testFile01");
+ TestFile testfile02 = new TestFile("testDir02", "testFile02");
+ TestFile testfile03 = new TestFile("testDir03", "testFile03");
+ TestFile testfile04 = new TestFile("testDir04", "testFile04");
+ TestFile testfile05 = new TestFile("testDir05", "testFile05");
+ TestFile testfile06 = new TestFile("testDir06", "testFile06");
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testfile01.getPathData());
+ pathData.add(testfile02.getPathData());
+ pathData.add(testfile03.getPathData());
+ pathData.add(testfile04.getPathData());
+ pathData.add(testfile05.getPathData());
+ pathData.add(testfile06.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check listing of a single directory
+ @Test
+ public void processPathDirectory() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check listing of multiple directories
+ @Test
+ public void processPathDirectories() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory01", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory01", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory01", "testFile03");
+
+ TestFile testDir01 = new TestFile("", "testDirectory01");
+ testDir01.setIsDir(true);
+ testDir01.addContents(testfile01, testfile02, testfile03);
+
+ TestFile testfile04 = new TestFile("testDirectory02", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory02", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory02", "testFile06");
+ TestFile testDir02 = new TestFile("", "testDirectory02");
+ testDir02.setIsDir(true);
+ testDir02.addContents(testfile04, testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir01.getPathData());
+ pathData.add(testDir02.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 3 items");
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println("Found 3 items");
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check the default ordering
+ @Test
+ public void processPathDirOrderDefault() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ // add contents in non-lexigraphic order to show they get sorted
+ testDir.addContents(testfile01, testfile03, testfile05, testfile02,
+ testfile04, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check reverse default ordering
+ @Test
+ public void processPathDirOrderDefaultReverse() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ // add contents in non-lexigraphic order to show they get sorted
+ testDir.addContents(testfile01, testfile03, testfile05, testfile02,
+ testfile04, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-r");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check mtime ordering (-t option); most recent first in line with unix
+ // convention
+ @Test
+ public void processPathDirOrderMtime() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file mtime in different order to file names
+ testfile01.setMtime(NOW.getTime() + 10);
+ testfile02.setMtime(NOW.getTime() + 30);
+ testfile03.setMtime(NOW.getTime() + 20);
+ testfile04.setMtime(NOW.getTime() + 60);
+ testfile05.setMtime(NOW.getTime() + 50);
+ testfile06.setMtime(NOW.getTime() + 40);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check reverse mtime ordering (-t -r options)
+ @Test
+ public void processPathDirOrderMtimeReverse() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file mtime in different order to file names
+ testfile01.setMtime(NOW.getTime() + 10);
+ testfile02.setMtime(NOW.getTime() + 30);
+ testfile03.setMtime(NOW.getTime() + 20);
+ testfile04.setMtime(NOW.getTime() + 60);
+ testfile05.setMtime(NOW.getTime() + 50);
+ testfile06.setMtime(NOW.getTime() + 40);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ options.add("-r");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check multiple directories are order independently
+ @Test
+ public void processPathDirsOrderMtime() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory01", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory01", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory01", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory02", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory02", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory02", "testFile06");
+
+ // set file mtime in different order to file names
+ testfile01.setMtime(NOW.getTime() + 10);
+ testfile02.setMtime(NOW.getTime() + 30);
+ testfile03.setMtime(NOW.getTime() + 20);
+ testfile04.setMtime(NOW.getTime() + 60);
+ testfile05.setMtime(NOW.getTime() + 40);
+ testfile06.setMtime(NOW.getTime() + 50);
+
+ TestFile testDir01 = new TestFile("", "testDirectory01");
+ testDir01.setIsDir(true);
+ testDir01.addContents(testfile01, testfile02, testfile03);
+
+ TestFile testDir02 = new TestFile("", "testDirectory02");
+ testDir02.setIsDir(true);
+ testDir02.addContents(testfile04, testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir01.getPathData());
+ pathData.add(testDir02.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 3 items");
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println("Found 3 items");
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check mtime ordering with large time gaps between files (checks integer
+ // overflow issues)
+ @Test
+ public void processPathDirOrderMtimeYears() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file mtime in different order to file names
+ testfile01.setMtime(NOW.getTime() + Integer.MAX_VALUE);
+ testfile02.setMtime(NOW.getTime() + Integer.MIN_VALUE);
+ testfile03.setMtime(NOW.getTime() + 0);
+ testfile04.setMtime(NOW.getTime() + Integer.MAX_VALUE + Integer.MAX_VALUE);
+ testfile05.setMtime(NOW.getTime() + 0);
+ testfile06.setMtime(NOW.getTime() + Integer.MIN_VALUE + Integer.MIN_VALUE);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check length order (-S option)
+ @Test
+ public void processPathDirOrderLength() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file length in different order to file names
+ long length = 1234567890;
+ testfile01.setLength(length + 10);
+ testfile02.setLength(length + 30);
+ testfile03.setLength(length + 20);
+ testfile04.setLength(length + 60);
+ testfile05.setLength(length + 50);
+ testfile06.setLength(length + 40);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-S");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check reverse length order (-S -r options)
+ @Test
+ public void processPathDirOrderLengthReverse() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file length in different order to file names
+ long length = 1234567890;
+ testfile01.setLength(length + 10);
+ testfile02.setLength(length + 30);
+ testfile03.setLength(length + 20);
+ testfile04.setLength(length + 60);
+ testfile05.setLength(length + 50);
+ testfile06.setLength(length + 40);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-S");
+ options.add("-r");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check length ordering with large size gaps between files (checks integer
+ // overflow issues)
+ @Test
+ public void processPathDirOrderLengthLarge() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file length in different order to file names
+ long length = 1234567890;
+ testfile01.setLength(length + 3l * Integer.MAX_VALUE);
+ testfile02.setLength(length + Integer.MAX_VALUE);
+ testfile03.setLength(length + 2l * Integer.MAX_VALUE);
+ testfile04.setLength(length + 4l * Integer.MAX_VALUE);
+ testfile05.setLength(length + 2l * Integer.MAX_VALUE);
+ testfile06.setLength(length + 0);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-S");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile04.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineMtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineMtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check access time display (-u option)
+ @Test
+ public void processPathDirectoryAtime() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-u");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile01.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineAtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check access time order (-u -t options)
+ @Test
+ public void processPathDirOrderAtime() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file atime in different order to file names
+ testfile01.setAtime(NOW.getTime() + 10);
+ testfile02.setAtime(NOW.getTime() + 30);
+ testfile03.setAtime(NOW.getTime() + 20);
+ testfile04.setAtime(NOW.getTime() + 60);
+ testfile05.setAtime(NOW.getTime() + 50);
+ testfile06.setAtime(NOW.getTime() + 40);
+
+ // set file mtime in different order to atime
+ testfile01.setMtime(NOW.getTime() + 60);
+ testfile02.setMtime(NOW.getTime() + 50);
+ testfile03.setMtime(NOW.getTime() + 20);
+ testfile04.setMtime(NOW.getTime() + 30);
+ testfile05.setMtime(NOW.getTime() + 10);
+ testfile06.setMtime(NOW.getTime() + 40);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ options.add("-u");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile04.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile01.formatLineAtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check reverse access time order (-u -t -r options)
+ @Test
+ public void processPathDirOrderAtimeReverse() throws IOException {
+ TestFile testfile01 = new TestFile("testDirectory", "testFile01");
+ TestFile testfile02 = new TestFile("testDirectory", "testFile02");
+ TestFile testfile03 = new TestFile("testDirectory", "testFile03");
+ TestFile testfile04 = new TestFile("testDirectory", "testFile04");
+ TestFile testfile05 = new TestFile("testDirectory", "testFile05");
+ TestFile testfile06 = new TestFile("testDirectory", "testFile06");
+
+ // set file atime in different order to file names
+ testfile01.setAtime(NOW.getTime() + 10);
+ testfile02.setAtime(NOW.getTime() + 30);
+ testfile03.setAtime(NOW.getTime() + 20);
+ testfile04.setAtime(NOW.getTime() + 60);
+ testfile05.setAtime(NOW.getTime() + 50);
+ testfile06.setAtime(NOW.getTime() + 40);
+
+ // set file mtime in different order to atime
+ testfile01.setMtime(NOW.getTime() + 60);
+ testfile02.setMtime(NOW.getTime() + 50);
+ testfile03.setMtime(NOW.getTime() + 20);
+ testfile04.setMtime(NOW.getTime() + 30);
+ testfile05.setMtime(NOW.getTime() + 10);
+ testfile06.setMtime(NOW.getTime() + 40);
+
+ TestFile testDir = new TestFile("", "testDirectory");
+ testDir.setIsDir(true);
+ testDir.addContents(testfile01, testfile02, testfile03, testfile04,
+ testfile05, testfile06);
+
+ LinkedList<PathData> pathData = new LinkedList<PathData>();
+ pathData.add(testDir.getPathData());
+
+ PrintStream out = mock(PrintStream.class);
+
+ Ls ls = new Ls();
+ ls.out = out;
+
+ LinkedList<String> options = new LinkedList<String>();
+ options.add("-t");
+ options.add("-u");
+ options.add("-r");
+ ls.processOptions(options);
+ String lineFormat = TestFile.computeLineFormat(pathData);
+
+ ls.processArguments(pathData);
+ InOrder inOrder = inOrder(out);
+ inOrder.verify(out).println("Found 6 items");
+ inOrder.verify(out).println(testfile01.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile03.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile02.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile06.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile05.formatLineAtime(lineFormat));
+ inOrder.verify(out).println(testfile04.formatLineAtime(lineFormat));
+ verifyNoMoreInteractions(out);
+ }
+
+ // check the deprecated flag isn't set
+ @Test
+ public void isDeprecated() {
+ Ls ls = new Ls();
+ boolean actual = ls.isDeprecated();
+ boolean expected = false;
+ assertEquals("Ls.isDeprecated", expected, actual);
+ }
+
+ // check there's no replacement command
+ @Test
+ public void getReplacementCommand() {
+ Ls ls = new Ls();
+ String actual = ls.getReplacementCommand();
+ String expected = null;
+ assertEquals("Ls.getReplacementCommand", expected, actual);
+ }
+
+ // check the correct name is returned
+ @Test
+ public void getName() {
+ Ls ls = new Ls();
+ String actual = ls.getName();
+ String expected = "ls";
+ assertEquals("Ls.getName", expected, actual);
+ }
+
+ // test class representing a file to be listed
+ static class TestFile {
+ private static final SimpleDateFormat DATE_FORMAT = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm");
+
+ private static final boolean DEFAULT_ISDIR = false;
+ private static final String DEFAULT_MODE = "750";
+ private static final int DEFAULT_REPLICATION = 3;
+ private static final String DEFAULT_OWNER = "test_owner";
+ private static final String DEFAULT_GROUP = "test_group";
+ private static final long DEFAULT_LENGTH = 1234567890L;
+ private static final long DEFAULT_MTIME = NOW.getTime() - 86400000;
+ private static final long DEFAULT_ATIME = NOW.getTime() + 86400000;
+ private static final long DEFAULT_BLOCKSIZE = 64L * 1024 * 1024;
+
+ private String dirname;
+ private String filename;
+ private boolean isDir;
+ private FsPermission permission;
+ private int replication;
+ private String owner;
+ private String group;
+ private long length;
+ private long mtime;
+ private long atime;
+ private long blocksize;
+ private ArrayList<FileStatus> contents = new ArrayList<FileStatus>();
+
+ private Path path = null;
+ private FileStatus fileStatus = null;
+ private PathData pathData = null;
+
+ public TestFile(String dirname, String filename) {
+ setDirname(dirname);
+ setFilename(filename);
+ setIsDir(DEFAULT_ISDIR);
+ setPermission(DEFAULT_MODE);
+ setReplication(DEFAULT_REPLICATION);
+ setOwner(DEFAULT_OWNER);
+ setGroup(DEFAULT_GROUP);
+ setLength(DEFAULT_LENGTH);
+ setMtime(DEFAULT_MTIME);
+ setAtime(DEFAULT_ATIME);
+ setBlocksize(DEFAULT_BLOCKSIZE);
+ }
+
+ public void setDirname(String dirname) {
+ this.dirname = dirname;
+ }
+
+ public void setFilename(String filename) {
+ this.filename = filename;
+ }
+
+ public void setIsDir(boolean isDir) {
+ this.isDir = isDir;
+ }
+
+ public void setPermission(String mode) {
+ setPermission(new FsPermission(mode));
+ }
+
+ public void setPermission(FsPermission permission) {
+ this.permission = permission;
+ }
+
+ public void setReplication(int replication) {
+ this.replication = replication;
+ }
+
+ public void setOwner(String owner) {
+ this.owner = owner;
+ }
+
+ public void setGroup(String group) {
+ this.group = group;
+ }
+
+ public void setLength(long length) {
+ this.length = length;
+ }
+
+ public void setMtime(long mtime) {
+ this.mtime = mtime;
+ }
+
+ public void setAtime(long atime) {
+ this.atime = atime;
+ }
+
+ public void setBlocksize(long blocksize) {
+ this.blocksize = blocksize;
+ }
+
+ public void addContents(TestFile... contents) {
+ for (TestFile testFile : contents) {
+ this.contents.add(testFile.getFileStatus());
+ }
+ }
+
+ private String getDirname() {
+ return this.dirname;
+ }
+
+ private String getFilename() {
+ return this.filename;
+ }
+
+ private String getPathname() {
+ return getDirname() + "/" + getFilename();
+ }
+
+ private boolean isDir() {
+ return this.isDir;
+ }
+
+ private boolean isFile() {
+ return !this.isDir();
+ }
+
+ private FsPermission getPermission() {
+ return this.permission;
+ }
+
+ private int getReplication() {
+ return this.replication;
+ }
+
+ private String getOwner() {
+ return this.owner;
+ }
+
+ private String getGroup() {
+ return this.group;
+ }
+
+ private long getLength() {
+ return this.length;
+ }
+
+ private long getMtime() {
+ return this.mtime;
+ }
+
+ private long getAtime() {
+ return this.atime;
+ }
+
+ private long getBlocksize() {
+ return this.blocksize;
+ }
+
+ private FileStatus[] getContents() {
+ return this.contents.toArray(new FileStatus[0]);
+ }
+
+ /**
+ * Returns a formated output line based on the given format mask, file
+ * status and file name.
+ *
+ * @param lineFormat
+ * format mask
+ * @param fileStatus
+ * file status
+ * @param fileName
+ * file name
+ * @return formated line
+ */
+ private String formatLineMtime(String lineFormat) {
+ return String.format(lineFormat, (isDir() ? "d" : "-"), getPermission(),
+ (isFile() ? getReplication() : "-"), getOwner(), getGroup(),
+ String.valueOf(getLength()),
+ DATE_FORMAT.format(new Date(getMtime())), getPathname());
+ }
+
+ /**
+ * Returns a formated output line based on the given format mask, file
+ * status and file name.
+ *
+ * @param lineFormat
+ * format mask
+ * @param fileStatus
+ * file status
+ * @param fileName
+ * file name
+ * @return formated line
+ */
+ private String formatLineAtime(String lineFormat) {
+ return String.format(lineFormat, (isDir() ? "d" : "-"), getPermission(),
+ (isFile() ? getReplication() : "-"), getOwner(), getGroup(),
+ String.valueOf(getLength()),
+ DATE_FORMAT.format(new Date(getAtime())), getPathname());
+ }
+
+ public FileStatus getFileStatus() {
+ if (fileStatus == null) {
+ Path path = getPath();
+ fileStatus = new FileStatus(getLength(), isDir(), getReplication(),
+ getBlocksize(), getMtime(), getAtime(), getPermission(),
+ getOwner(), getGroup(), path);
+ }
+ return fileStatus;
+ }
+
+ public Path getPath() {
+ if (path == null) {
+ if ((getDirname() != null) && (!getDirname().equals(""))) {
+ path = new Path(getDirname(), getFilename());
+ } else {
+ path = new Path(getFilename());
+ }
+ }
+ return path;
+ }
+
+ public PathData getPathData() throws IOException {
+ if (pathData == null) {
+ FileStatus fileStatus = getFileStatus();
+ Path path = getPath();
+ when(mockFs.getFileStatus(eq(path))).thenReturn(fileStatus);
+ pathData = new PathData(path.toString(), conf);
+
+ if (getContents().length != 0) {
+ when(mockFs.listStatus(eq(path))).thenReturn(getContents());
+ }
+
+ }
+ return pathData;
+ }
+
+ /**
+ * Compute format string based on maximum column widths. Copied from
+ * Ls.adjustColumnWidths as these tests are more interested in proving
+ * regression rather than absolute format.
+ *
+ * @param items
+ * to find the max field width for each column
+ */
+ public static String computeLineFormat(LinkedList<PathData> items) {
+ int maxRepl = 3, maxLen = 10, maxOwner = 0, maxGroup = 0;
+ for (PathData item : items) {
+ FileStatus stat = item.stat;
+ maxRepl = maxLength(maxRepl, stat.getReplication());
+ maxLen = maxLength(maxLen, stat.getLen());
+ maxOwner = maxLength(maxOwner, stat.getOwner());
+ maxGroup = maxLength(maxGroup, stat.getGroup());
+ }
+
+ StringBuilder fmt = new StringBuilder();
+ fmt.append("%s%s "); // permission string
+ fmt.append("%" + maxRepl + "s ");
+ // Do not use '%-0s' as a formatting conversion, since it will throw a
+ // a MissingFormatWidthException if it is used in String.format().
+ // http://docs.oracle.com/javase/1.5.0/docs/api/java/util/Formatter.html#intFlags
+ fmt.append((maxOwner > 0) ? "%-" + maxOwner + "s " : "%s");
+ fmt.append((maxGroup > 0) ? "%-" + maxGroup + "s " : "%s");
+ fmt.append("%" + maxLen + "s ");
+ fmt.append("%s %s"); // mod time & path
+ return fmt.toString();
+ }
+
+ /**
+ * Return the maximum of two values, treating null as 0
+ *
+ * @param n
+ * integer to be compared
+ * @param value
+ * value to be compared
+ * @return maximum of the two inputs
+ */
+ private static int maxLength(int n, Object value) {
+ return Math.max(n, (value != null) ? String.valueOf(value).length() : 0);
+ }
+ }
+
+ static class MockFileSystem extends FilterFileSystem {
+ Configuration conf;
+
+ MockFileSystem() {
+ super(mockFs);
+ }
+
+ @Override
+ public void initialize(URI uri, Configuration conf) {
+ this.conf = conf;
+ }
+
+ @Override
+ public Path makeQualified(Path path) {
+ return path;
+ }
+
+ @Override
+ public Configuration getConf() {
+ return conf;
+ }
+ }
+}
[10/50] [abbrv] hadoop git commit: HDFS-7546. Document,
and set an accepting default for
dfs.namenode.kerberos.principal.pattern (Harsh J via aw)
Posted by zj...@apache.org.
HDFS-7546. Document, and set an accepting default for dfs.namenode.kerberos.principal.pattern (Harsh J via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63613c79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63613c79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63613c79
Branch: refs/heads/YARN-2928
Commit: 63613c79c1042ea3d7706ed6f7eccc8cf48ff6ea
Parents: 0af6a99
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 13:58:50 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 13:58:58 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/main/resources/hdfs-default.xml | 9 +++++++++
2 files changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63613c79/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index fe19a28..5a77829 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -136,6 +136,9 @@ Trunk (Unreleased)
HDFS-7430. Rewrite the BlockScanner to use O(1) memory and use multiple
threads (cmccabe)
+ HDFS-7546. Document, and set an accepting default for
+ dfs.namenode.kerberos.principal.pattern (Harsh J via aw)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63613c79/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 966f5f0..9697543 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1288,6 +1288,15 @@
</property>
<property>
+ <name>dfs.namenode.kerberos.principal.pattern</name>
+ <value>*</value>
+ <description>
+ A client-side RegEx that can be configured to control
+ allowed realms to authenticate with (useful in cross-realm env.)
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.avoid.read.stale.datanode</name>
<value>false</value>
<description>
[05/50] [abbrv] hadoop git commit: HDFS-7670. HDFS Quota guide has
typos, incomplete command lines (Brahma Reddy Battula via aw)
Posted by zj...@apache.org.
HDFS-7670. HDFS Quota guide has typos, incomplete command lines (Brahma Reddy Battula via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/84cc071a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/84cc071a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/84cc071a
Branch: refs/heads/YARN-2928
Commit: 84cc071a7bfc9ba7ec1674fd41a38a33fd9cdf12
Parents: 30b797e
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 12:54:03 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 12:54:03 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/site/apt/HdfsQuotaAdminGuide.apt.vm | 10 +++++-----
2 files changed, 8 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cc071a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4396e3d..0d9000a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -287,6 +287,9 @@ Trunk (Unreleased)
HDFS-7751. Fix TestHDFSCLI for quota with storage type. (Xiaoyu Yao
via szetszwo)
+ HDFS-7670. HDFS Quota guide has typos, incomplete command lines
+ (Brahma Reddy Battula via aw)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/84cc071a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsQuotaAdminGuide.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsQuotaAdminGuide.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsQuotaAdminGuide.apt.vm
index 0821946..222cfbf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsQuotaAdminGuide.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/HdfsQuotaAdminGuide.apt.vm
@@ -71,20 +71,20 @@ HDFS Quotas Guide
Quotas are managed by a set of commands available only to the
administrator.
- * <<<dfsadmin -setQuota <N> <directory>...<directory> >>>
+ * <<<hdfs dfsadmin -setQuota <N> <directory>...<directory> >>>
Set the name quota to be N for each directory. Best effort for each
directory, with faults reported if N is not a positive long
integer, the directory does not exist or it is a file, or the
directory would immediately exceed the new quota.
- * <<<dfsadmin -clrQuota <directory>...<directory> >>>
+ * <<<hdfs dfsadmin -clrQuota <directory>...<directory> >>>
Remove any name quota for each directory. Best effort for each
directory, with faults reported if the directory does not exist or
it is a file. It is not a fault if the directory has no quota.
- * <<<dfsadmin -setSpaceQuota <N> <directory>...<directory> >>>
+ * <<<hdfs dfsadmin -setSpaceQuota <N> <directory>...<directory> >>>
Set the space quota to be N bytes for each directory. This is a
hard limit on total size of all the files under the directory tree.
@@ -96,7 +96,7 @@ HDFS Quotas Guide
integer, the directory does not exist or it is a file, or the
directory would immediately exceed the new quota.
- * <<<dfsadmin -clrSpaceQuota <directory>...<director> >>>
+ * <<<hdfs dfsadmin -clrSpaceQuota <directory>...<directory> >>>
Remove any space quota for each directory. Best effort for each
directory, with faults reported if the directory does not exist or
@@ -107,7 +107,7 @@ HDFS Quotas Guide
An an extension to the count command of the HDFS shell reports quota
values and the current count of names and bytes in use.
- * <<<fs -count -q <directory>...<directory> >>>
+ * <<<hadoop fs -count -q <directory>...<directory> >>>
With the -q option, also report the name quota value set for each
directory, the available name quota remaining, the space quota
[19/50] [abbrv] hadoop git commit: HADOOP-11512. Use
getTrimmedStrings when reading serialization keys. Contributed by Ryan P.
Posted by zj...@apache.org.
HADOOP-11512. Use getTrimmedStrings when reading serialization keys. Contributed by Ryan P.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e0ec0718
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e0ec0718
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e0ec0718
Branch: refs/heads/YARN-2928
Commit: e0ec0718d033e84bda2ebeab7beb00b7dbd990c0
Parents: b73956f
Author: Harsh J <ha...@cloudera.com>
Authored: Mon Feb 9 10:41:25 2015 +0530
Committer: Harsh J <ha...@cloudera.com>
Committed: Tue Feb 10 12:51:56 2015 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../apache/hadoop/io/serializer/SerializationFactory.java | 2 +-
.../hadoop/io/serializer/TestSerializationFactory.java | 10 ++++++++++
3 files changed, 14 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ec0718/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index aa86360..8b80998 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -588,6 +588,9 @@ Release 2.7.0 - UNRELEASED
BUG FIXES
+ HADOOP-11512. Use getTrimmedStrings when reading serialization keys
+ (Ryan P via harsh)
+
HADOOP-11488. Difference in default connection timeout for S3A FS
(Daisuke Kobayashi via harsh)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ec0718/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
index d6c6588..3f177f8 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/serializer/SerializationFactory.java
@@ -60,7 +60,7 @@ public class SerializationFactory extends Configured {
+ CommonConfigurationKeys.IO_SERIALIZATIONS_KEY
+ " properly to have serialization support (it is currently not set).");
} else {
- for (String serializerName : conf.getStrings(
+ for (String serializerName : conf.getTrimmedStrings(
CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, new String[] {
WritableSerialization.class.getName(),
AvroSpecificSerialization.class.getName(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e0ec0718/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
index c5805be..6774155 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/serializer/TestSerializationFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.io.serializer;
+import org.apache.hadoop.io.LongWritable;
import org.junit.BeforeClass;
import org.junit.Test;
import static org.junit.Assert.assertNull;
@@ -76,4 +77,13 @@ public class TestSerializationFactory {
assertNull("A null should be returned if there are no deserializers found",
factory.getDeserializer(TestSerializationFactory.class));
}
+
+ @Test
+ public void testSerializationKeyIsTrimmed() {
+ Configuration conf = new Configuration();
+ conf.set(CommonConfigurationKeys.IO_SERIALIZATIONS_KEY, " org.apache.hadoop.io.serializer.WritableSerialization ");
+ SerializationFactory factory = new SerializationFactory(conf);
+ assertNotNull("Valid class must be returned",
+ factory.getSerializer(LongWritable.class));
+ }
}
[02/50] [abbrv] hadoop git commit: MAPREDUCE-6237. Multiple mappers
with DBInputFormat don't work because of reusing conections. Contributed by
Kannan Rajah.
Posted by zj...@apache.org.
MAPREDUCE-6237. Multiple mappers with DBInputFormat don't work because of reusing conections. Contributed by Kannan Rajah.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/241336ca
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/241336ca
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/241336ca
Branch: refs/heads/YARN-2928
Commit: 241336ca2b7cf97d7e0bd84dbe0542b72f304dc9
Parents: 7e42088
Author: Tsuyoshi Ozawa <oz...@apache.org>
Authored: Tue Feb 10 03:52:42 2015 +0900
Committer: Tsuyoshi Ozawa <oz...@apache.org>
Committed: Tue Feb 10 03:52:42 2015 +0900
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 15 ++++++++++
.../hadoop/mapreduce/lib/db/DBInputFormat.java | 31 ++++++++++++--------
.../lib/db/DataDrivenDBInputFormat.java | 5 ++--
.../lib/db/OracleDataDrivenDBInputFormat.java | 2 +-
4 files changed, 37 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/241336ca/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 583c6c1..c71fee8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -346,6 +346,21 @@ Release 2.7.0 - UNRELEASED
MAPREDUCE-6233. org.apache.hadoop.mapreduce.TestLargeSort.testLargeSort
failed in trunk (zxu via rkanter)
+Release 2.6.1 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
+ MAPREDUCE-6237. Multiple mappers with DBInputFormat don't work because of
+ reusing conections. (Kannan Rajah via ozawa)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/241336ca/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java
index c0530c2..00fbeda 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DBInputFormat.java
@@ -159,7 +159,7 @@ public class DBInputFormat<T extends DBWritable>
dbConf = new DBConfiguration(conf);
try {
- getConnection();
+ this.connection = createConnection();
DatabaseMetaData dbMeta = connection.getMetaData();
this.dbProductName = dbMeta.getDatabaseProductName().toUpperCase();
@@ -182,18 +182,25 @@ public class DBInputFormat<T extends DBWritable>
}
public Connection getConnection() {
+ // TODO Remove this code that handles backward compatibility.
+ if (this.connection == null) {
+ this.connection = createConnection();
+ }
+
+ return this.connection;
+ }
+
+ public Connection createConnection() {
try {
- if (null == this.connection) {
- // The connection was closed; reinstantiate it.
- this.connection = dbConf.getConnection();
- this.connection.setAutoCommit(false);
- this.connection.setTransactionIsolation(
- Connection.TRANSACTION_SERIALIZABLE);
- }
+ Connection newConnection = dbConf.getConnection();
+ newConnection.setAutoCommit(false);
+ newConnection.setTransactionIsolation(
+ Connection.TRANSACTION_SERIALIZABLE);
+
+ return newConnection;
} catch (Exception e) {
throw new RuntimeException(e);
}
- return connection;
}
public String getDBProductName() {
@@ -210,17 +217,17 @@ public class DBInputFormat<T extends DBWritable>
if (dbProductName.startsWith("ORACLE")) {
// use Oracle-specific db reader.
return new OracleDBRecordReader<T>(split, inputClass,
- conf, getConnection(), getDBConf(), conditions, fieldNames,
+ conf, createConnection(), getDBConf(), conditions, fieldNames,
tableName);
} else if (dbProductName.startsWith("MYSQL")) {
// use MySQL-specific db reader.
return new MySQLDBRecordReader<T>(split, inputClass,
- conf, getConnection(), getDBConf(), conditions, fieldNames,
+ conf, createConnection(), getDBConf(), conditions, fieldNames,
tableName);
} else {
// Generic reader.
return new DBRecordReader<T>(split, inputClass,
- conf, getConnection(), getDBConf(), conditions, fieldNames,
+ conf, createConnection(), getDBConf(), conditions, fieldNames,
tableName);
}
} catch (SQLException ex) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/241336ca/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
index 131b7bb..753c880 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/DataDrivenDBInputFormat.java
@@ -178,7 +178,6 @@ public class DataDrivenDBInputFormat<T extends DBWritable>
ResultSet results = null;
Statement statement = null;
- Connection connection = getConnection();
try {
statement = connection.createStatement();
@@ -289,12 +288,12 @@ public class DataDrivenDBInputFormat<T extends DBWritable>
if (dbProductName.startsWith("MYSQL")) {
// use MySQL-specific db reader.
return new MySQLDataDrivenDBRecordReader<T>(split, inputClass,
- conf, getConnection(), dbConf, dbConf.getInputConditions(),
+ conf, createConnection(), dbConf, dbConf.getInputConditions(),
dbConf.getInputFieldNames(), dbConf.getInputTableName());
} else {
// Generic reader.
return new DataDrivenDBRecordReader<T>(split, inputClass,
- conf, getConnection(), dbConf, dbConf.getInputConditions(),
+ conf, createConnection(), dbConf, dbConf.getInputConditions(),
dbConf.getInputFieldNames(), dbConf.getInputTableName(),
dbProductName);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/241336ca/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
index 8fbd473..a02471e 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/lib/db/OracleDataDrivenDBInputFormat.java
@@ -84,7 +84,7 @@ public class OracleDataDrivenDBInputFormat<T extends DBWritable>
try {
// Use Oracle-specific db reader
return new OracleDataDrivenDBRecordReader<T>(split, inputClass,
- conf, getConnection(), dbConf, dbConf.getInputConditions(),
+ conf, createConnection(), dbConf, dbConf.getInputConditions(),
dbConf.getInputFieldNames(), dbConf.getInputTableName());
} catch (SQLException ex) {
throw new IOException(ex.getMessage());
[33/50] [abbrv] hadoop git commit: HDFS-7753. Fix Multithreaded
correctness Warnings in BackupImage. Contributed by Rakesh R and Konstantin
Shvachko.
Posted by zj...@apache.org.
HDFS-7753. Fix Multithreaded correctness Warnings in BackupImage. Contributed by Rakesh R and Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4ceea60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4ceea60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4ceea60
Branch: refs/heads/YARN-2928
Commit: a4ceea60f57a32d531549e492aa5894dd34e0d0f
Parents: 7c6b654
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Wed Feb 11 00:45:18 2015 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Wed Feb 11 00:45:18 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/server/namenode/BackupImage.java | 21 +++++++++++++-------
2 files changed, 17 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4ceea60/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 2e90478..1d2a76a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -904,6 +904,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7769. TestHDFSCLI should not create files in hdfs project root dir.
(szetszwo)
+ HDFS-7753. Fix Multithreaded correctness Warnings in BackupImage.
+ (Rakesh R and shv)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4ceea60/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
index 1900b40..dc44959 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/BackupImage.java
@@ -92,9 +92,16 @@ public class BackupImage extends FSImage {
storage.setDisablePreUpgradableLayoutCheck(true);
bnState = BNState.DROP_UNTIL_NEXT_ROLL;
}
-
- void setNamesystem(FSNamesystem fsn) {
- this.namesystem = fsn;
+
+ synchronized FSNamesystem getNamesystem() {
+ return namesystem;
+ }
+
+ synchronized void setNamesystem(FSNamesystem fsn) {
+ // Avoids overriding this.namesystem object
+ if (namesystem == null) {
+ this.namesystem = fsn;
+ }
}
/**
@@ -195,7 +202,7 @@ public class BackupImage extends FSImage {
}
FSEditLogLoader logLoader =
- new FSEditLogLoader(namesystem, lastAppliedTxId);
+ new FSEditLogLoader(getNamesystem(), lastAppliedTxId);
int logVersion = storage.getLayoutVersion();
backupInputStream.setBytes(data, logVersion);
@@ -209,7 +216,7 @@ public class BackupImage extends FSImage {
}
lastAppliedTxId = logLoader.getLastAppliedTxId();
- FSImage.updateCountForQuota(namesystem.dir.rootDir); // inefficient!
+ FSImage.updateCountForQuota(getNamesystem().dir.getRoot()); // inefficient!
} finally {
backupInputStream.clear();
}
@@ -258,7 +265,7 @@ public class BackupImage extends FSImage {
editStreams.add(s);
}
}
- loadEdits(editStreams, namesystem);
+ loadEdits(editStreams, getNamesystem());
}
// now, need to load the in-progress file
@@ -293,7 +300,7 @@ public class BackupImage extends FSImage {
+ " txns from in-progress stream " + stream);
FSEditLogLoader loader =
- new FSEditLogLoader(namesystem, lastAppliedTxId);
+ new FSEditLogLoader(getNamesystem(), lastAppliedTxId);
loader.loadFSEdits(stream, lastAppliedTxId + 1);
lastAppliedTxId = loader.getLastAppliedTxId();
assert lastAppliedTxId == getEditLog().getLastWrittenTxId();
[11/50] [abbrv] hadoop git commit: MAPREDUCE-4413. MR lib dir
contains jdiff (which is gpl) (Nemon Lou via aw)
Posted by zj...@apache.org.
MAPREDUCE-4413. MR lib dir contains jdiff (which is gpl) (Nemon Lou via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aab459c9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aab459c9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aab459c9
Branch: refs/heads/YARN-2928
Commit: aab459c904bf2007c5b230af8c058793935faf89
Parents: 63613c7
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 14:02:47 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 14:02:47 2015 -0800
----------------------------------------------------------------------
.../src/main/resources/assemblies/hadoop-mapreduce-dist.xml | 1 +
hadoop-mapreduce-project/CHANGES.txt | 2 ++
2 files changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab459c9/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
----------------------------------------------------------------------
diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
index 247b09c..df08c6c 100644
--- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
+++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-mapreduce-dist.xml
@@ -195,6 +195,7 @@
<exclude>org.slf4j:slf4j-api</exclude>
<exclude>org.slf4j:slf4j-log4j12</exclude>
<exclude>org.hsqldb:hsqldb</exclude>
+ <exclude>jdiff:jdiff:jar</exclude>
</excludes>
</dependencySet>
<dependencySet>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aab459c9/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index c71fee8..246f18d 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -168,6 +168,8 @@ Trunk (Unreleased)
MAPREDUCE-6161. mapred hsadmin command missing from trunk (Allen Wittenauer
via jlowe)
+ MAPREDUCE-4413. MR lib dir contains jdiff (which is gpl) (Nemon Lou via aw)
+
BREAKDOWN OF MAPREDUCE-2841 (NATIVE TASK) SUBTASKS
MAPREDUCE-5985. native-task: Fix build on macosx. Contributed by
[34/50] [abbrv] hadoop git commit: YARN-3160. Fix non-atomic
operation on nodeUpdateQueue in RMNodeImpl. (Contributed by Chengbing Liu)
Posted by zj...@apache.org.
YARN-3160. Fix non-atomic operation on nodeUpdateQueue in RMNodeImpl. (Contributed by Chengbing Liu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c541a374
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c541a374
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c541a374
Branch: refs/heads/YARN-2928
Commit: c541a374d88ffed6ee71b0e5d556939ccd2c5159
Parents: a4ceea6
Author: Junping Du <ju...@apache.org>
Authored: Wed Feb 11 03:44:31 2015 -0800
Committer: Junping Du <ju...@apache.org>
Committed: Wed Feb 11 03:44:31 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java | 5 +++--
2 files changed, 6 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c541a374/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 92e1803..41927d7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -537,6 +537,9 @@ Release 2.7.0 - UNRELEASED
http(s)://proxy addr:port/proxy/<appId> to avoid duplicate sections. (Devaraj
K via zjshen)
+ YARN-3160. Fix non-atomic operation on nodeUpdateQueue in RMNodeImpl.
+ (Chengbing Liu via junping_du)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c541a374/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index b3020b7..1bc98b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -842,8 +842,9 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
public List<UpdatedContainerInfo> pullContainerUpdates() {
List<UpdatedContainerInfo> latestContainerInfoList =
new ArrayList<UpdatedContainerInfo>();
- while(nodeUpdateQueue.peek() != null){
- latestContainerInfoList.add(nodeUpdateQueue.poll());
+ UpdatedContainerInfo containerInfo;
+ while ((containerInfo = nodeUpdateQueue.poll()) != null) {
+ latestContainerInfoList.add(containerInfo);
}
this.nextHeartBeat = true;
return latestContainerInfoList;
[14/50] [abbrv] hadoop git commit: HDFS-7744. Fix potential NPE in
DFSInputStream after setDropBehind or setReadahead is called (cmccabe)
Posted by zj...@apache.org.
HDFS-7744. Fix potential NPE in DFSInputStream after setDropBehind or setReadahead is called (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a9dc5cd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a9dc5cd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a9dc5cd7
Branch: refs/heads/YARN-2928
Commit: a9dc5cd7069f721e8c55794b877026ba02537167
Parents: 260b5e3
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Feb 9 20:16:41 2015 -0800
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Feb 9 20:16:41 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSInputStream.java | 11 ++-----
.../server/datanode/TestCachingStrategy.java | 30 ++++++++++++++++++++
3 files changed, 36 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9dc5cd7/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index a841c7e..446c6a3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -890,6 +890,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7756. Restore method signature for LocatedBlock#getLocations(). (Ted
Yu via yliu)
+ HDFS-7744. Fix potential NPE in DFSInputStream after setDropBehind or
+ setReadahead is called (cmccabe)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9dc5cd7/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 9e75333..618f040 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -580,10 +580,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
}
// Will be getting a new BlockReader.
- if (blockReader != null) {
- blockReader.close();
- blockReader = null;
- }
+ closeCurrentBlockReader();
//
// Connect to best DataNode for desired Block, with potential offset
@@ -686,10 +683,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
"unreleased ByteBuffers allocated by read(). " +
"Please release " + builder.toString() + ".");
}
- if (blockReader != null) {
- blockReader.close();
- blockReader = null;
- }
+ closeCurrentBlockReader();
super.close();
}
@@ -1649,6 +1643,7 @@ implements ByteBufferReadable, CanSetDropBehind, CanSetReadahead,
DFSClient.LOG.error("error closing blockReader", e);
}
blockReader = null;
+ blockEnd = -1;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a9dc5cd7/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
index b1df8ad..709554a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/TestCachingStrategy.java
@@ -369,4 +369,34 @@ public class TestCachingStrategy {
}
}
}
+
+ @Test(timeout=120000)
+ public void testSeekAfterSetDropBehind() throws Exception {
+ // start a cluster
+ LOG.info("testSeekAfterSetDropBehind");
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ String TEST_PATH = "/test";
+ int TEST_PATH_LEN = MAX_TEST_FILE_LEN;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1)
+ .build();
+ cluster.waitActive();
+ FileSystem fs = cluster.getFileSystem();
+ createHdfsFile(fs, new Path(TEST_PATH), TEST_PATH_LEN, false);
+ // verify that we can seek after setDropBehind
+ FSDataInputStream fis = fs.open(new Path(TEST_PATH));
+ try {
+ Assert.assertTrue(fis.read() != -1); // create BlockReader
+ fis.setDropBehind(false); // clear BlockReader
+ fis.seek(2); // seek
+ } finally {
+ fis.close();
+ }
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
[40/50] [abbrv] hadoop git commit: YARN-3074. Nodemanager dies when
localizer runner tries to write to a full disk. Contributed by Varun Saxena
Posted by zj...@apache.org.
YARN-3074. Nodemanager dies when localizer runner tries to write to a full disk. Contributed by Varun Saxena
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b379972a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b379972a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b379972a
Branch: refs/heads/YARN-2928
Commit: b379972ab39551d4b57436a54c0098a63742c7e1
Parents: b94c111
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Feb 11 16:33:43 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Feb 11 16:33:43 2015 +0000
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../localizer/ResourceLocalizationService.java | 19 +++--
.../TestResourceLocalizationService.java | 82 ++++++++++++++++++++
3 files changed, 98 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b379972a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a1c3407..ba5490c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -546,6 +546,9 @@ Release 2.7.0 - UNRELEASED
YARN-3160. Fix non-atomic operation on nodeUpdateQueue in RMNodeImpl.
(Chengbing Liu via junping_du)
+ YARN-3074. Nodemanager dies when localizer runner tries to write to a full
+ disk (Varun Saxena via jlowe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b379972a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 8c84132..dd50ead 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -55,6 +55,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
@@ -1063,6 +1064,7 @@ public class ResourceLocalizationService extends CompositeService
@SuppressWarnings("unchecked") // dispatcher not typed
public void run() {
Path nmPrivateCTokensPath = null;
+ Throwable exception = null;
try {
// Get nmPrivateDir
nmPrivateCTokensPath =
@@ -1090,14 +1092,19 @@ public class ResourceLocalizationService extends CompositeService
+ dirsHandler.getDisksHealthReport(false));
}
// TODO handle ExitCodeException separately?
+ } catch (FSError fe) {
+ exception = fe;
} catch (Exception e) {
- LOG.info("Localizer failed", e);
- // 3) on error, report failure to Container and signal ABORT
- // 3.1) notify resource of failed localization
- ContainerId cId = context.getContainerId();
- dispatcher.getEventHandler().handle(
- new ContainerResourceFailedEvent(cId, null, e.getMessage()));
+ exception = e;
} finally {
+ if (exception != null) {
+ LOG.info("Localizer failed", exception);
+ // On error, report failure to Container and signal ABORT
+ // Notify resource of failed localization
+ ContainerId cId = context.getContainerId();
+ dispatcher.getEventHandler().handle(new ContainerResourceFailedEvent(
+ cId, null, exception.getMessage()));
+ }
for (LocalizerResourceRequestEvent event : scheduled.values()) {
event.getResource().unlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b379972a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 30af5a4..d3c3521 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -43,6 +43,7 @@ import static org.mockito.Mockito.when;
import java.io.File;
import java.io.IOException;
+import java.lang.reflect.Constructor;
import java.net.InetSocketAddress;
import java.net.URI;
import java.net.URISyntaxException;
@@ -69,6 +70,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FSError;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Options.ChecksumOpt;
@@ -715,6 +717,86 @@ public class TestResourceLocalizationService {
stateStore.close();
}
}
+
+
+ @Test( timeout = 10000)
+ @SuppressWarnings("unchecked") // mocked generics
+ public void testLocalizerRunnerException() throws Exception {
+ DrainDispatcher dispatcher = new DrainDispatcher();
+ dispatcher.init(conf);
+ dispatcher.start();
+ EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
+ dispatcher.register(ApplicationEventType.class, applicationBus);
+ EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
+ dispatcher.register(ContainerEventType.class, containerBus);
+
+ ContainerExecutor exec = mock(ContainerExecutor.class);
+ LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
+ LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
+ dirsHandlerSpy.init(conf);
+
+ DeletionService delServiceReal = new DeletionService(exec);
+ DeletionService delService = spy(delServiceReal);
+ delService.init(new Configuration());
+ delService.start();
+
+ ResourceLocalizationService rawService =
+ new ResourceLocalizationService(dispatcher, exec, delService,
+ dirsHandlerSpy, nmContext);
+ ResourceLocalizationService spyService = spy(rawService);
+ doReturn(mockServer).when(spyService).createServer();
+ try {
+ spyService.init(conf);
+ spyService.start();
+
+ // init application
+ final Application app = mock(Application.class);
+ final ApplicationId appId =
+ BuilderUtils.newApplicationId(314159265358979L, 3);
+ when(app.getUser()).thenReturn("user0");
+ when(app.getAppId()).thenReturn(appId);
+ spyService.handle(new ApplicationLocalizationEvent(
+ LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
+ dispatcher.await();
+
+ Random r = new Random();
+ long seed = r.nextLong();
+ System.out.println("SEED: " + seed);
+ r.setSeed(seed);
+ final Container c = getMockContainer(appId, 42, "user0");
+ final LocalResource resource1 = getPrivateMockedResource(r);
+ System.out.println("Here 4");
+
+ final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs =
+ new HashMap<LocalResourceVisibility,
+ Collection<LocalResourceRequest>>();
+ List<LocalResourceRequest> privateResourceList =
+ new ArrayList<LocalResourceRequest>();
+ privateResourceList.add(req1);
+ rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
+
+ final Constructor<?>[] constructors =
+ FSError.class.getDeclaredConstructors();
+ constructors[0].setAccessible(true);
+ FSError fsError =
+ (FSError) constructors[0].newInstance(new IOException("Disk Error"));
+
+ Mockito
+ .doThrow(fsError)
+ .when(dirsHandlerSpy)
+ .getLocalPathForWrite(isA(String.class));
+ spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
+ Thread.sleep(1000);
+ dispatcher.await();
+ // Verify if ContainerResourceFailedEvent is invoked on FSError
+ verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
+ } finally {
+ spyService.stop();
+ dispatcher.stop();
+ delService.stop();
+ }
+ }
@Test( timeout = 10000)
@SuppressWarnings("unchecked") // mocked generics
[03/50] [abbrv] hadoop git commit: HDFS-7647.
DatanodeManager.sortLocatedBlocks sorts DatanodeInfos but not StorageIDs.
(Contributed by Milan Desai)
Posted by zj...@apache.org.
HDFS-7647. DatanodeManager.sortLocatedBlocks sorts DatanodeInfos but not StorageIDs. (Contributed by Milan Desai)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab934e85
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab934e85
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab934e85
Branch: refs/heads/YARN-2928
Commit: ab934e85947dcf2092050023909dd81ae274ff45
Parents: 241336c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Feb 9 12:17:40 2015 -0800
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Feb 9 12:17:40 2015 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/net/NetworkTopology.java | 2 +-
.../net/NetworkTopologyWithNodeGroup.java | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop/hdfs/protocol/LocatedBlock.java | 77 ++++++++++++++----
.../server/blockmanagement/DatanodeManager.java | 2 +
.../protocol/DatanodeInfoWithStorage.java | 59 ++++++++++++++
.../apache/hadoop/hdfs/TestDecommission.java | 10 ++-
.../blockmanagement/TestDatanodeManager.java | 84 ++++++++++++++++++++
8 files changed, 218 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
index aaa5ae3..fc8bf52 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopology.java
@@ -859,7 +859,7 @@ public class NetworkTopology {
// Start off by initializing to off rack
int weight = 2;
if (reader != null) {
- if (reader == node) {
+ if (reader.equals(node)) {
weight = 0;
} else if (isOnSameRack(reader, node)) {
weight = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
index 13160eb..3de49dc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/net/NetworkTopologyWithNodeGroup.java
@@ -254,7 +254,7 @@ public class NetworkTopologyWithNodeGroup extends NetworkTopology {
// Start off by initializing to off rack
int weight = 3;
if (reader != null) {
- if (reader == node) {
+ if (reader.equals(node)) {
weight = 0;
} else if (isOnSameNodeGroup(reader, node)) {
weight = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eda3744..4396e3d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -872,6 +872,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7741. Remove unnecessary synchronized in FSDataInputStream and
HdfsDataInputStream. (yliu)
+ HDFS-7647. DatanodeManager.sortLocatedBlocks sorts DatanodeInfos
+ but not StorageIDs. (Milan Desai via Arpit Agarwal)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
index 30368f6..7fb2e30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/LocatedBlock.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.security.token.block.BlockTokenIdentifier;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeInfoWithStorage;
import org.apache.hadoop.security.token.Token;
import com.google.common.collect.Lists;
@@ -41,11 +42,13 @@ public class LocatedBlock {
private final ExtendedBlock b;
private long offset; // offset of the first byte of the block in the file
- private final DatanodeInfo[] locs;
- /** Storage ID for each replica */
- private final String[] storageIDs;
- // Storage type for each replica, if reported.
- private final StorageType[] storageTypes;
+ private final DatanodeInfoWithStorage[] locs;
+ private final boolean hasStorageIDs;
+ private final boolean hasStorageTypes;
+ /** Cached storage ID for each replica */
+ private String[] storageIDs;
+ /** Cached storage type for each replica, if reported. */
+ private StorageType[] storageTypes;
// corrupt flag is true if all of the replicas of a block are corrupt.
// else false. If block has few corrupt replicas, they are filtered and
// their locations are not part of this object
@@ -57,7 +60,8 @@ public class LocatedBlock {
private DatanodeInfo[] cachedLocs;
// Used when there are no locations
- private static final DatanodeInfo[] EMPTY_LOCS = new DatanodeInfo[0];
+ private static final DatanodeInfoWithStorage[] EMPTY_LOCS =
+ new DatanodeInfoWithStorage[0];
public LocatedBlock(ExtendedBlock b, DatanodeInfo[] locs) {
this(b, locs, -1, false); // startOffset is unknown
@@ -94,10 +98,22 @@ public class LocatedBlock {
if (locs==null) {
this.locs = EMPTY_LOCS;
} else {
- this.locs = locs;
+ this.locs = new DatanodeInfoWithStorage[locs.length];
+ for(int i = 0; i < locs.length; i++) {
+ DatanodeInfo di = locs[i];
+ DatanodeInfoWithStorage storage = new DatanodeInfoWithStorage(di,
+ storageIDs != null ? storageIDs[i] : null,
+ storageTypes != null ? storageTypes[i] : null);
+ storage.setDependentHostNames(di.getDependentHostNames());
+ storage.setLevel(di.getLevel());
+ storage.setParent(di.getParent());
+ this.locs[i] = storage;
+ }
}
this.storageIDs = storageIDs;
this.storageTypes = storageTypes;
+ this.hasStorageIDs = storageIDs != null;
+ this.hasStorageTypes = storageTypes != null;
if (cachedLocs == null || cachedLocs.length == 0) {
this.cachedLocs = EMPTY_LOCS;
@@ -118,18 +134,53 @@ public class LocatedBlock {
return b;
}
- public DatanodeInfo[] getLocations() {
+ /**
+ * Returns the locations associated with this block. The returned array is not
+ * expected to be modified. If it is, caller must immediately invoke
+ * {@link org.apache.hadoop.hdfs.protocol.LocatedBlock#invalidateCachedStorageInfo}
+ * to invalidate the cached Storage ID/Type arrays.
+ */
+ public DatanodeInfoWithStorage[] getLocations() {
return locs;
}
public StorageType[] getStorageTypes() {
+ if(!hasStorageTypes) {
+ return null;
+ }
+ if(storageTypes != null) {
+ return storageTypes;
+ }
+ storageTypes = new StorageType[locs.length];
+ for(int i = 0; i < locs.length; i++) {
+ storageTypes[i] = locs[i].getStorageType();
+ }
return storageTypes;
}
public String[] getStorageIDs() {
+ if(!hasStorageIDs) {
+ return null;
+ }
+ if(storageIDs != null) {
+ return storageIDs;
+ }
+ storageIDs = new String[locs.length];
+ for(int i = 0; i < locs.length; i++) {
+ storageIDs[i] = locs[i].getStorageID();
+ }
return storageIDs;
}
+ /**
+ * Invalidates the cached StorageID and StorageType information. Must be
+ * called when the locations array is modified.
+ */
+ public void invalidateCachedStorageInfo() {
+ storageIDs = null;
+ storageTypes = null;
+ }
+
public long getStartOffset() {
return offset;
}
@@ -161,9 +212,9 @@ public class LocatedBlock {
return;
}
// Try to re-use a DatanodeInfo already in loc
- for (int i=0; i<locs.length; i++) {
- if (locs[i].equals(loc)) {
- cachedList.add(locs[i]);
+ for (DatanodeInfoWithStorage di : locs) {
+ if (loc.equals(di)) {
+ cachedList.add(di);
cachedLocs = cachedList.toArray(cachedLocs);
return;
}
@@ -187,10 +238,6 @@ public class LocatedBlock {
+ "; corrupt=" + corrupt
+ "; offset=" + offset
+ "; locs=" + Arrays.asList(locs)
- + "; storageIDs=" +
- (storageIDs != null ? Arrays.asList(storageIDs) : null)
- + "; storageTypes=" +
- (storageTypes != null ? Arrays.asList(storageTypes) : null)
+ "}";
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index a33d990..c166e94 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -391,6 +391,8 @@ public class DatanodeManager {
}
int activeLen = lastActiveIndex + 1;
networktopology.sortByDistance(client, b.getLocations(), activeLen);
+ // must invalidate cache since we modified locations array
+ b.invalidateCachedStorageInfo();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java
new file mode 100644
index 0000000..ec8c346
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/protocol/DatanodeInfoWithStorage.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.protocol;
+
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+
+public class DatanodeInfoWithStorage extends DatanodeInfo {
+ private final String storageID;
+ private final StorageType storageType;
+
+ public DatanodeInfoWithStorage(DatanodeInfo from, String storageID,
+ StorageType storageType) {
+ super(from);
+ this.storageID = storageID;
+ this.storageType = storageType;
+ }
+
+ public String getStorageID() {
+ return storageID;
+ }
+
+ public StorageType getStorageType() {
+ return storageType;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ // allows this class to be used interchangeably with DatanodeInfo
+ return super.equals(o);
+ }
+
+ @Override
+ public int hashCode() {
+ // allows this class to be used interchangeably with DatanodeInfo
+ return super.hashCode();
+ }
+
+ @Override
+ public String toString() {
+ return "DatanodeInfoWithStorage[" + super.toString() + "," + storageID +
+ "," + storageType + "]";
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
index 24e5db6..35c0d8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDecommission.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.hdfs.server.namenode.ha.HATestUtil;
@@ -801,22 +802,23 @@ public class TestDecommission {
ArrayList<String> nodes = new ArrayList<String>();
ArrayList<DatanodeInfo> dnInfos = new ArrayList<DatanodeInfo>();
-
+
+ DatanodeManager dm = ns.getBlockManager().getDatanodeManager();
for (DatanodeInfo datanodeInfo : dnInfos4FirstBlock) {
DatanodeInfo found = datanodeInfo;
for (DatanodeInfo dif: dnInfos4LastBlock) {
if (datanodeInfo.equals(dif)) {
- found = null;
+ found = null;
}
}
if (found != null) {
nodes.add(found.getXferAddr());
- dnInfos.add(found);
+ dnInfos.add(dm.getDatanode(found));
}
}
//decommission one of the 3 nodes which have last block
nodes.add(dnInfos4LastBlock[0].getXferAddr());
- dnInfos.add(dnInfos4LastBlock[0]);
+ dnInfos.add(dm.getDatanode(dnInfos4LastBlock[0]));
writeConfigFile(excludeFile, nodes);
refreshNodes(ns, conf);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab934e85/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index 2c65fff..adf31a0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
@@ -31,13 +32,19 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.hdfs.server.protocol.DatanodeInfoWithStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.net.DNSToSwitchMapping;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.Mockito;
+import static org.hamcrest.core.Is.is;
import static org.junit.Assert.*;
public class TestDatanodeManager {
@@ -210,4 +217,81 @@ public class TestDatanodeManager {
public void reloadCachedMappings(List<String> names) {
}
}
+
+ /**
+ * This test creates a LocatedBlock with 5 locations, sorts the locations
+ * based on the network topology, and ensures the locations are still aligned
+ * with the storage ids and storage types.
+ */
+ @Test
+ public void testSortLocatedBlocks() throws IOException {
+ // create the DatanodeManager which will be tested
+ FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
+ Mockito.when(fsn.hasWriteLock()).thenReturn(true);
+ DatanodeManager dm = new DatanodeManager(Mockito.mock(BlockManager.class),
+ fsn, new Configuration());
+
+ // register 5 datanodes, each with different storage ID and type
+ DatanodeInfo[] locs = new DatanodeInfo[5];
+ String[] storageIDs = new String[5];
+ StorageType[] storageTypes = new StorageType[]{
+ StorageType.ARCHIVE,
+ StorageType.DEFAULT,
+ StorageType.DISK,
+ StorageType.RAM_DISK,
+ StorageType.SSD
+ };
+ for(int i = 0; i < 5; i++) {
+ // register new datanode
+ String uuid = "UUID-"+i;
+ String ip = "IP-" + i;
+ DatanodeRegistration dr = Mockito.mock(DatanodeRegistration.class);
+ Mockito.when(dr.getDatanodeUuid()).thenReturn(uuid);
+ Mockito.when(dr.getIpAddr()).thenReturn(ip);
+ Mockito.when(dr.getXferAddr()).thenReturn(ip + ":9000");
+ Mockito.when(dr.getXferPort()).thenReturn(9000);
+ Mockito.when(dr.getSoftwareVersion()).thenReturn("version1");
+ dm.registerDatanode(dr);
+
+ // get location and storage information
+ locs[i] = dm.getDatanode(uuid);
+ storageIDs[i] = "storageID-"+i;
+ }
+
+ // set first 2 locations as decomissioned
+ locs[0].setDecommissioned();
+ locs[1].setDecommissioned();
+
+ // create LocatedBlock with above locations
+ ExtendedBlock b = new ExtendedBlock("somePoolID", 1234);
+ LocatedBlock block = new LocatedBlock(b, locs, storageIDs, storageTypes);
+ List<LocatedBlock> blocks = new ArrayList<>();
+ blocks.add(block);
+
+ final String targetIp = locs[4].getIpAddr();
+
+ // sort block locations
+ dm.sortLocatedBlocks(targetIp, blocks);
+
+ // check that storage IDs/types are aligned with datanode locs
+ DatanodeInfoWithStorage[] sortedLocs = block.getLocations();
+ storageIDs = block.getStorageIDs();
+ storageTypes = block.getStorageTypes();
+ assertThat(sortedLocs.length, is(5));
+ assertThat(storageIDs.length, is(5));
+ assertThat(storageTypes.length, is(5));
+ for(int i = 0; i < sortedLocs.length; i++) {
+ assertThat(sortedLocs[i].getStorageID(), is(storageIDs[i]));
+ assertThat(sortedLocs[i].getStorageType(), is(storageTypes[i]));
+ }
+
+ // Ensure the local node is first.
+ assertThat(sortedLocs[0].getIpAddr(), is(targetIp));
+
+ // Ensure the two decommissioned DNs were moved to the end.
+ assertThat(sortedLocs[sortedLocs.length-1].getAdminState(),
+ is(DatanodeInfo.AdminStates.DECOMMISSIONED));
+ assertThat(sortedLocs[sortedLocs.length-2].getAdminState(),
+ is(DatanodeInfo.AdminStates.DECOMMISSIONED));
+ }
}
[08/50] [abbrv] hadoop git commit: HDFS-7755. httpfs shell code has
hardcoded path to bash (Dmitry Sivachenko via aw)
Posted by zj...@apache.org.
HDFS-7755. httpfs shell code has hardcoded path to bash (Dmitry Sivachenko via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7d732027
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7d732027
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7d732027
Branch: refs/heads/YARN-2928
Commit: 7d73202734e79beaa2db34d6b811beba7b34ee87
Parents: fcad031
Author: Allen Wittenauer <aw...@apache.org>
Authored: Mon Feb 9 13:05:14 2015 -0800
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Mon Feb 9 13:05:14 2015 -0800
----------------------------------------------------------------------
.../hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh | 2 +-
hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 2 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
3 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d732027/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
index a2fe1c2..d4db28f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/libexec/httpfs-config.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d732027/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index a593b67..65903dc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -1,4 +1,4 @@
-#!/bin/bash
+#!/usr/bin/env bash
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7d732027/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 74474ee..fe19a28 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -293,6 +293,9 @@ Trunk (Unreleased)
HDFS-7669. HDFS Design Doc references commands that no longer exist.
(Brahma Reddy Battula via aw)
+ HDFS-7755. httpfs shell code has hardcoded path to bash (Dmitry
+ Sivachenko via aw)
+
Release 2.7.0 - UNRELEASED
INCOMPATIBLE CHANGES
[17/50] [abbrv] hadoop git commit: HDFS-7714. Simultaneous restart of
HA NameNodes and DataNode can cause DataNode to register successfully with
only one NameNode.(Contributed by Vinayakumar B)
Posted by zj...@apache.org.
HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause DataNode to register successfully with only one NameNode.(Contributed by Vinayakumar B)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3d15728f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3d15728f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3d15728f
Branch: refs/heads/YARN-2928
Commit: 3d15728ff5301296801e541d9b23bd1687c4adad
Parents: 23bf6c7
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Feb 10 10:43:08 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Feb 10 10:43:08 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java | 5 +++++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d15728f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 4a6bc11..1ca2263 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -896,6 +896,9 @@ Release 2.7.0 - UNRELEASED
HDFS-7718. Store KeyProvider in ClientContext to avoid leaking key provider
threads when using FileContext (Arun Suresh via Colin P. McCabe)
+ HDFS-7714. Simultaneous restart of HA NameNodes and DataNode can cause
+ DataNode to register successfully with only one NameNode.(vinayakumarb)
+
Release 2.6.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3d15728f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
index e396727..917b5dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BPServiceActor.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.datanode;
import static org.apache.hadoop.util.Time.now;
+import java.io.EOFException;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.net.SocketTimeoutException;
@@ -797,6 +798,10 @@ class BPServiceActor implements Runnable {
// Use returned registration from namenode with updated fields
bpRegistration = bpNamenode.registerDatanode(bpRegistration);
break;
+ } catch(EOFException e) { // namenode might have just restarted
+ LOG.info("Problem connecting to server: " + nnAddr + " :"
+ + e.getLocalizedMessage());
+ sleepAndLogInterrupts(1000, "connecting to server");
} catch(SocketTimeoutException e) { // namenode is busy
LOG.info("Problem connecting to server: " + nnAddr);
sleepAndLogInterrupts(1000, "connecting to server");
[43/50] [abbrv] hadoop git commit: HDFS-7760. Document truncate for
WebHDFS. Contributed by Konstantin Shvachko.
Posted by zj...@apache.org.
HDFS-7760. Document truncate for WebHDFS. Contributed by Konstantin Shvachko.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e42fc1a2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e42fc1a2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e42fc1a2
Branch: refs/heads/YARN-2928
Commit: e42fc1a251e91d25dbc4b3728b3cf4554ca7bee1
Parents: 6f6737b
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Wed Feb 11 09:53:50 2015 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Wed Feb 11 09:53:50 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 +
.../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 46 ++++++++++++++++++++
2 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e42fc1a2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ed5db9b..758bcb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -611,6 +611,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7058. Tests for truncate CLI. (Dasha Boudnik via shv)
+ HDFS-7760. Document truncate for WebHDFS. (shv)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e42fc1a2/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
index 662f8b8..3668286 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/apt/WebHDFS.apt.vm
@@ -137,6 +137,9 @@ WebHDFS REST API
* {{{Concat File(s)}<<<CONCAT>>>}}
(see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.concat)
+ * {{{Truncate a File}<<<TRUNCATE>>>}}
+ (see {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.truncate)
+
* HTTP DELETE
* {{{Delete a File/Directory}<<<DELETE>>>}}
@@ -492,6 +495,31 @@ Transfer-Encoding: chunked
{{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.delete
+** {Truncate a File}
+
+ * Submit a HTTP POST request.
+
++---------------------------------
+curl -i -X POST "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=TRUNCATE&newlength=<LONG>"
++---------------------------------
+
+ The client receives a response with a {{{Boolean JSON Schema}<<<boolean>>> JSON object}}:
+
++---------------------------------
+HTTP/1.1 200 OK
+Content-Type: application/json
+Transfer-Encoding: chunked
+
+{"boolean": true}
++---------------------------------
+
+ []
+
+ See also:
+ {{{New Length}<<<newlength>>>}},
+ {{{../../api/org/apache/hadoop/fs/FileSystem.html}FileSystem}}.truncate
+
+
** {Status of a File/Directory}
* Submit a HTTP GET request.
@@ -2274,6 +2302,24 @@ var tokenProperties =
{{{Set Access or Modification Time}<<<SETTIMES>>>}}
+** {New Length}
+
+*----------------+-------------------------------------------------------------------+
+|| Name | <<<newlength>>> |
+*----------------+-------------------------------------------------------------------+
+|| Description | The size the file is to be truncated to. |
+*----------------+-------------------------------------------------------------------+
+|| Type | long |
+*----------------+-------------------------------------------------------------------+
+|| Valid Values | \>= 0 |
+*----------------+-------------------------------------------------------------------+
+|| Syntax | Any long. |
+*----------------+-------------------------------------------------------------------+
+
+ See also:
+ {{{Truncate a File}<<<TRUNCATE>>>}}
+
+
** {Offset}
*----------------+-------------------------------------------------------------------+
[41/50] [abbrv] hadoop git commit: HADOOP-11580. Remove
SingleNodeSetup.md from trunk. (aajisaka)
Posted by zj...@apache.org.
HADOOP-11580. Remove SingleNodeSetup.md from trunk. (aajisaka)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/22441ab7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/22441ab7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/22441ab7
Branch: refs/heads/YARN-2928
Commit: 22441ab7b29e05033e30dda1615339bd62c76dd2
Parents: b379972
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Feb 11 09:08:19 2015 -0800
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Feb 11 09:08:19 2015 -0800
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/site/markdown/SingleNodeSetup.md | 20 --------------------
2 files changed, 2 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22441ab7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 26d84d8..3035024 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -385,6 +385,8 @@ Trunk (Unreleased)
HADOOP-11581. Multithreaded correctness Warnings
#org.apache.hadoop.fs.shell.Ls (Brahma Reddy Battula via aw)
+ HADOOP-11580. Remove SingleNodeSetup.md from trunk (aajisaka)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/22441ab7/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md
deleted file mode 100644
index fae8b5c..0000000
--- a/hadoop-common-project/hadoop-common/src/site/markdown/SingleNodeSetup.md
+++ /dev/null
@@ -1,20 +0,0 @@
-<!---
- Licensed under the Apache License, Version 2.0 (the "License");
- you may not use this file except in compliance with the License.
- You may obtain a copy of the License at
-
- http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing, software
- distributed under the License is distributed on an "AS IS" BASIS,
- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- See the License for the specific language governing permissions and
- limitations under the License. See accompanying LICENSE file.
--->
-
-Single Node Setup
-=================
-
-This page will be removed in the next major release.
-
-See [Single Cluster Setup](./SingleCluster.html) to set up and configure a single-node Hadoop installation.
[46/50] [abbrv] hadoop git commit: HDFS-7723. Quota By Storage Type
namenode implemenation. (Contributed by Xiaoyu Yao)
Posted by zj...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 7781244..532617f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -250,6 +250,7 @@ import org.apache.hadoop.hdfs.server.protocol.NamenodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.NamespaceInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Text;
@@ -2063,7 +2064,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// update the quota: use the preferred block size for UC block
final long diff =
file.getPreferredBlockSize() - truncatedBlockUC.getNumBytes();
- dir.updateSpaceConsumed(iip, 0, diff * file.getBlockReplication());
+ dir.updateSpaceConsumed(iip, 0, diff, file.getBlockReplication());
return newBlock;
}
@@ -2671,7 +2672,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
if (ret != null) {
// update the quota: use the preferred block size for UC block
final long diff = file.getPreferredBlockSize() - ret.getBlockSize();
- dir.updateSpaceConsumed(iip, 0, diff * file.getBlockReplication());
+ dir.updateSpaceConsumed(iip, 0, diff, file.getBlockReplication());
}
} else {
BlockInfoContiguous lastBlock = file.getLastBlock();
@@ -3803,19 +3804,19 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Set the namespace quota and diskspace quota for a directory.
- * See {@link ClientProtocol#setQuota(String, long, long)} for the
+ * See {@link ClientProtocol#setQuota(String, long, long, StorageType)} for the
* contract.
*
* Note: This does not support ".inodes" relative path.
*/
- void setQuota(String src, long nsQuota, long dsQuota)
+ void setQuota(String src, long nsQuota, long dsQuota, StorageType type)
throws IOException {
checkOperation(OperationCategory.WRITE);
writeLock();
try {
checkOperation(OperationCategory.WRITE);
checkNameNodeSafeMode("Cannot set quota on " + src);
- FSDirAttrOp.setQuota(dir, src, nsQuota, dsQuota);
+ FSDirAttrOp.setQuota(dir, src, nsQuota, dsQuota, type);
} finally {
writeUnlock();
}
@@ -4042,7 +4043,7 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
final long diff = fileINode.getPreferredBlockSize() - commitBlock.getNumBytes();
if (diff > 0) {
try {
- dir.updateSpaceConsumed(iip, 0, -diff*fileINode.getFileReplication());
+ dir.updateSpaceConsumed(iip, 0, -diff, fileINode.getFileReplication());
} catch (IOException e) {
LOG.warn("Unexpected exception while updating disk space.", e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
index bafde45..7f3bf38 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INode.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.Block;
@@ -384,7 +385,9 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* 2.4 To clean {@link INodeDirectory} with snapshot: delete the corresponding
* snapshot in its diff list. Recursively clean its children.
* </pre>
- *
+ *
+ * @param bsps
+ * block storage policy suite to calculate intended storage type usage
* @param snapshotId
* The id of the snapshot to delete.
* {@link Snapshot#CURRENT_STATE_ID} means to delete the current
@@ -401,7 +404,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* inodeMap
* @return quota usage delta when deleting a snapshot
*/
- public abstract Quota.Counts cleanSubtree(final int snapshotId,
+ public abstract QuotaCounts cleanSubtree(final BlockStoragePolicySuite bsps,
+ final int snapshotId,
int priorSnapshotId, BlocksMapUpdateInfo collectedBlocks,
List<INode> removedINodes);
@@ -411,7 +415,11 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* directory, the method goes down the subtree and collects blocks from the
* descents, and clears its parent/children references as well. The method
* also clears the diff list if the INode contains snapshot diff list.
- *
+ *
+ * @param bsps
+ * block storage policy suite to calculate intended storage type usage
+ * This is needed because INodeReference#destroyAndCollectBlocks() needs
+ * to call INode#cleanSubtree(), which calls INode#computeQuotaUsage().
* @param collectedBlocks
* blocks collected from the descents for further block
* deletion/update will be added to this map.
@@ -420,6 +428,7 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* inodeMap
*/
public abstract void destroyAndCollectBlocks(
+ BlockStoragePolicySuite bsps,
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes);
/** Compute {@link ContentSummary}. Blocking call */
@@ -434,11 +443,12 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
public final ContentSummary computeAndConvertContentSummary(
ContentSummaryComputationContext summary) {
Content.Counts counts = computeContentSummary(summary).getCounts();
- final Quota.Counts q = getQuotaCounts();
+ final QuotaCounts q = getQuotaCounts();
return new ContentSummary(counts.get(Content.LENGTH),
counts.get(Content.FILE) + counts.get(Content.SYMLINK),
- counts.get(Content.DIRECTORY), q.get(Quota.NAMESPACE),
- counts.get(Content.DISKSPACE), q.get(Quota.DISKSPACE));
+ counts.get(Content.DIRECTORY), q.getNameSpace(),
+ counts.get(Content.DISKSPACE), q.getDiskSpace());
+ // TODO: storage type quota reporting HDFS-7701.
}
/**
@@ -450,24 +460,24 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
public abstract ContentSummaryComputationContext computeContentSummary(
ContentSummaryComputationContext summary);
-
+
/**
- * Check and add namespace/diskspace consumed to itself and the ancestors.
+ * Check and add namespace/diskspace/storagetype consumed to itself and the ancestors.
* @throws QuotaExceededException if quote is violated.
*/
- public void addSpaceConsumed(long nsDelta, long dsDelta, boolean verify)
- throws QuotaExceededException {
- addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
+ public void addSpaceConsumed(QuotaCounts counts, boolean verify)
+ throws QuotaExceededException {
+ addSpaceConsumed2Parent(counts, verify);
}
/**
- * Check and add namespace/diskspace consumed to itself and the ancestors.
+ * Check and add namespace/diskspace/storagetype consumed to itself and the ancestors.
* @throws QuotaExceededException if quote is violated.
*/
- void addSpaceConsumed2Parent(long nsDelta, long dsDelta, boolean verify)
- throws QuotaExceededException {
+ void addSpaceConsumed2Parent(QuotaCounts counts, boolean verify)
+ throws QuotaExceededException {
if (parent != null) {
- parent.addSpaceConsumed(nsDelta, dsDelta, verify);
+ parent.addSpaceConsumed(counts, verify);
}
}
@@ -475,20 +485,24 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* Get the quota set for this inode
* @return the quota counts. The count is -1 if it is not set.
*/
- public Quota.Counts getQuotaCounts() {
- return Quota.Counts.newInstance(-1, -1);
+ public QuotaCounts getQuotaCounts() {
+ return new QuotaCounts.Builder().
+ nameCount(HdfsConstants.QUOTA_RESET).
+ spaceCount(HdfsConstants.QUOTA_RESET).
+ typeCounts(HdfsConstants.QUOTA_RESET).
+ build();
}
-
+
public final boolean isQuotaSet() {
- final Quota.Counts q = getQuotaCounts();
- return q.get(Quota.NAMESPACE) >= 0 || q.get(Quota.DISKSPACE) >= 0;
+ final QuotaCounts qc = getQuotaCounts();
+ return qc.anyNsSpCountGreaterOrEqual(0) || qc.anyTypeCountGreaterOrEqual(0);
}
-
+
/**
* Count subtree {@link Quota#NAMESPACE} and {@link Quota#DISKSPACE} usages.
*/
- public final Quota.Counts computeQuotaUsage() {
- return computeQuotaUsage(new Quota.Counts(), true);
+ public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps) {
+ return computeQuotaUsage(bsps, new QuotaCounts.Builder().build(), true);
}
/**
@@ -511,7 +525,8 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* creation time of the snapshot associated with the {@link WithName} node.
* We do not count in the size of the diff list.
* <pre>
- *
+ *
+ * @param bsps Block storage policy suite to calculate intended storage type usage
* @param counts The subtree counts for returning.
* @param useCache Whether to use cached quota usage. Note that
* {@link WithName} node never uses cache for its subtree.
@@ -521,14 +536,15 @@ public abstract class INode implements INodeAttributes, Diff.Element<byte[]> {
* {@link WithName} node.
* @return The same objects as the counts parameter.
*/
- public abstract Quota.Counts computeQuotaUsage(Quota.Counts counts,
- boolean useCache, int lastSnapshotId);
+ public abstract QuotaCounts computeQuotaUsage(
+ BlockStoragePolicySuite bsps,
+ QuotaCounts counts, boolean useCache, int lastSnapshotId);
- public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
- boolean useCache) {
- return computeQuotaUsage(counts, useCache, Snapshot.CURRENT_STATE_ID);
+ public final QuotaCounts computeQuotaUsage(
+ BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache) {
+ return computeQuotaUsage(bsps, counts, useCache, Snapshot.CURRENT_STATE_ID);
}
-
+
/**
* @return null if the local name is null; otherwise, return the local name.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
index 6bf57f0..f9d160b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectory.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectorySnapshottableFea
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature.DirectoryDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.util.Diff.ListType;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
@@ -136,35 +137,45 @@ public class INodeDirectory extends INodeWithAdditionalFields
BlockStoragePolicySuite.ID_UNSPECIFIED;
}
- void setQuota(long nsQuota, long dsQuota) {
+ void setQuota(BlockStoragePolicySuite bsps, long nsQuota, long dsQuota, StorageType type) {
DirectoryWithQuotaFeature quota = getDirectoryWithQuotaFeature();
if (quota != null) {
// already has quota; so set the quota to the new values
- quota.setQuota(nsQuota, dsQuota);
+ if (type != null) {
+ quota.setQuota(dsQuota, type);
+ } else {
+ quota.setQuota(nsQuota, dsQuota);
+ }
if (!isQuotaSet() && !isRoot()) {
removeFeature(quota);
}
} else {
- final Quota.Counts c = computeQuotaUsage();
- quota = addDirectoryWithQuotaFeature(nsQuota, dsQuota);
- quota.setSpaceConsumed(c.get(Quota.NAMESPACE), c.get(Quota.DISKSPACE));
+ final QuotaCounts c = computeQuotaUsage(bsps);
+ DirectoryWithQuotaFeature.Builder builder =
+ new DirectoryWithQuotaFeature.Builder().nameSpaceQuota(nsQuota);
+ if (type != null) {
+ builder.typeQuota(type, dsQuota);
+ } else {
+ builder.spaceQuota(dsQuota);
+ }
+ addDirectoryWithQuotaFeature(builder.build()).setSpaceConsumed(c);
}
}
@Override
- public Quota.Counts getQuotaCounts() {
+ public QuotaCounts getQuotaCounts() {
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
return q != null? q.getQuota(): super.getQuotaCounts();
}
@Override
- public void addSpaceConsumed(long nsDelta, long dsDelta, boolean verify)
- throws QuotaExceededException {
+ public void addSpaceConsumed(QuotaCounts counts, boolean verify)
+ throws QuotaExceededException {
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null) {
- q.addSpaceConsumed(this, nsDelta, dsDelta, verify);
+ q.addSpaceConsumed(this, counts, verify);
} else {
- addSpaceConsumed2Parent(nsDelta, dsDelta, verify);
+ addSpaceConsumed2Parent(counts, verify);
}
}
@@ -182,12 +193,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
}
DirectoryWithQuotaFeature addDirectoryWithQuotaFeature(
- long nsQuota, long dsQuota) {
+ DirectoryWithQuotaFeature q) {
Preconditions.checkState(!isWithQuota(), "Directory is already with quota");
- final DirectoryWithQuotaFeature quota = new DirectoryWithQuotaFeature(
- nsQuota, dsQuota);
- addFeature(quota);
- return quota;
+ addFeature(q);
+ return q;
}
int searchChildren(byte[] name) {
@@ -254,10 +263,10 @@ public class INodeDirectory extends INodeWithAdditionalFields
return getDirectorySnapshottableFeature().addSnapshot(this, id, name);
}
- public Snapshot removeSnapshot(String snapshotName,
+ public Snapshot removeSnapshot(BlockStoragePolicySuite bsps, String snapshotName,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes)
throws SnapshotException {
- return getDirectorySnapshottableFeature().removeSnapshot(this,
+ return getDirectorySnapshottableFeature().removeSnapshot(bsps, this,
snapshotName, collectedBlocks, removedINodes);
}
@@ -559,7 +568,7 @@ public class INodeDirectory extends INodeWithAdditionalFields
}
@Override
- public Quota.Counts computeQuotaUsage(Quota.Counts counts, boolean useCache,
+ public QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache,
int lastSnapshotId) {
final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
@@ -570,9 +579,9 @@ public class INodeDirectory extends INodeWithAdditionalFields
&& !(useCache && isQuotaSet())) {
ReadOnlyList<INode> childrenList = getChildrenList(lastSnapshotId);
for (INode child : childrenList) {
- child.computeQuotaUsage(counts, useCache, lastSnapshotId);
+ child.computeQuotaUsage(bsps, counts, useCache, lastSnapshotId);
}
- counts.add(Quota.NAMESPACE, 1);
+ counts.addNameSpace(1);
return counts;
}
@@ -582,27 +591,28 @@ public class INodeDirectory extends INodeWithAdditionalFields
return q.addNamespaceDiskspace(counts);
} else {
useCache = q != null && !q.isQuotaSet() ? false : useCache;
- return computeDirectoryQuotaUsage(counts, useCache, lastSnapshotId);
+ return computeDirectoryQuotaUsage(bsps, counts, useCache, lastSnapshotId);
}
}
- private Quota.Counts computeDirectoryQuotaUsage(Quota.Counts counts,
- boolean useCache, int lastSnapshotId) {
+ private QuotaCounts computeDirectoryQuotaUsage(BlockStoragePolicySuite bsps,
+ QuotaCounts counts, boolean useCache, int lastSnapshotId) {
if (children != null) {
for (INode child : children) {
- child.computeQuotaUsage(counts, useCache, lastSnapshotId);
+ child.computeQuotaUsage(bsps, counts, useCache, lastSnapshotId);
}
}
- return computeQuotaUsage4CurrentDirectory(counts);
+ return computeQuotaUsage4CurrentDirectory(bsps, counts);
}
/** Add quota usage for this inode excluding children. */
- public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) {
- counts.add(Quota.NAMESPACE, 1);
+ public QuotaCounts computeQuotaUsage4CurrentDirectory(
+ BlockStoragePolicySuite bsps, QuotaCounts counts) {
+ counts.addNameSpace(1);
// include the diff list
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
if (sf != null) {
- sf.computeQuotaUsage4CurrentDirectory(counts);
+ sf.computeQuotaUsage4CurrentDirectory(bsps, counts);
}
return counts;
}
@@ -612,7 +622,8 @@ public class INodeDirectory extends INodeWithAdditionalFields
ContentSummaryComputationContext summary) {
final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
if (sf != null) {
- sf.computeContentSummary4Snapshot(summary.getCounts());
+ sf.computeContentSummary4Snapshot(summary.getBlockStoragePolicySuite(),
+ summary.getCounts());
}
final DirectoryWithQuotaFeature q = getDirectoryWithQuotaFeature();
if (q != null) {
@@ -702,7 +713,8 @@ public class INodeDirectory extends INodeWithAdditionalFields
* (with OVERWRITE option) removes a file/dir from the dst tree, add it back
* and delete possible record in the deleted list.
*/
- public void undoRename4DstParent(final INode deletedChild,
+ public void undoRename4DstParent(final BlockStoragePolicySuite bsps,
+ final INode deletedChild,
int latestSnapshotId) throws QuotaExceededException {
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
Preconditions.checkState(sf != null,
@@ -714,9 +726,9 @@ public class INodeDirectory extends INodeWithAdditionalFields
// update quota usage if adding is successfully and the old child has not
// been stored in deleted list before
if (added && !removeDeletedChild) {
- final Quota.Counts counts = deletedChild.computeQuotaUsage();
- addSpaceConsumed(counts.get(Quota.NAMESPACE),
- counts.get(Quota.DISKSPACE), false);
+ final QuotaCounts counts = deletedChild.computeQuotaUsage(bsps);
+ addSpaceConsumed(counts, false);
+
}
}
@@ -732,10 +744,11 @@ public class INodeDirectory extends INodeWithAdditionalFields
}
/** Call cleanSubtree(..) recursively down the subtree. */
- public Quota.Counts cleanSubtreeRecursively(final int snapshot,
+ public QuotaCounts cleanSubtreeRecursively(final BlockStoragePolicySuite bsps,
+ final int snapshot,
int prior, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes, final Map<INode, INode> excludedNodes) {
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
// in case of deletion snapshot, since this call happens after we modify
// the diff list, the snapshot to be deleted has been combined or renamed
// to its latest previous snapshot. (besides, we also need to consider nodes
@@ -748,7 +761,7 @@ public class INodeDirectory extends INodeWithAdditionalFields
&& excludedNodes.containsKey(child)) {
continue;
} else {
- Quota.Counts childCounts = child.cleanSubtree(snapshot, prior,
+ QuotaCounts childCounts = child.cleanSubtree(bsps, snapshot, prior,
collectedBlocks, removedINodes);
counts.add(childCounts);
}
@@ -757,14 +770,15 @@ public class INodeDirectory extends INodeWithAdditionalFields
}
@Override
- public void destroyAndCollectBlocks(final BlocksMapUpdateInfo collectedBlocks,
+ public void destroyAndCollectBlocks(final BlockStoragePolicySuite bsps,
+ final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
final DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
if (sf != null) {
- sf.clear(this, collectedBlocks, removedINodes);
+ sf.clear(bsps, this, collectedBlocks, removedINodes);
}
for (INode child : getChildrenList(Snapshot.CURRENT_STATE_ID)) {
- child.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ child.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
if (getAclFeature() != null) {
AclStorage.removeAclFeature(getAclFeature());
@@ -774,30 +788,30 @@ public class INodeDirectory extends INodeWithAdditionalFields
}
@Override
- public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
+ public QuotaCounts cleanSubtree(final BlockStoragePolicySuite bsps,
+ final int snapshotId, int priorSnapshotId,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
DirectoryWithSnapshotFeature sf = getDirectoryWithSnapshotFeature();
// there is snapshot data
if (sf != null) {
- return sf.cleanDirectory(this, snapshotId, priorSnapshotId,
+ return sf.cleanDirectory(bsps, this, snapshotId, priorSnapshotId,
collectedBlocks, removedINodes);
}
// there is no snapshot data
if (priorSnapshotId == Snapshot.NO_SNAPSHOT_ID
&& snapshotId == Snapshot.CURRENT_STATE_ID) {
// destroy the whole subtree and collect blocks that should be deleted
- Quota.Counts counts = Quota.Counts.newInstance();
- this.computeQuotaUsage(counts, true);
- destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ this.computeQuotaUsage(bsps, counts, true);
+ destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return counts;
} else {
// process recursively down the subtree
- Quota.Counts counts = cleanSubtreeRecursively(snapshotId, priorSnapshotId,
+ QuotaCounts counts = cleanSubtreeRecursively(bsps, snapshotId, priorSnapshotId,
collectedBlocks, removedINodes, null);
if (isQuotaSet()) {
- getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
- -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+ getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(counts.negation());
}
return counts;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
index 26a6678..83649ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeDirectoryAttributes.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.util.EnumCounters;
import com.google.common.base.Preconditions;
@@ -27,7 +29,7 @@ import com.google.common.base.Preconditions;
*/
@InterfaceAudience.Private
public interface INodeDirectoryAttributes extends INodeAttributes {
- public Quota.Counts getQuotaCounts();
+ public QuotaCounts getQuotaCounts();
public boolean metadataEquals(INodeDirectoryAttributes other);
@@ -45,8 +47,9 @@ public interface INodeDirectoryAttributes extends INodeAttributes {
}
@Override
- public Quota.Counts getQuotaCounts() {
- return Quota.Counts.newInstance(-1, -1);
+ public QuotaCounts getQuotaCounts() {
+ return new QuotaCounts.Builder().nameCount(-1).
+ spaceCount(-1).typeCounts(-1).build();
}
@Override
@@ -60,29 +63,26 @@ public interface INodeDirectoryAttributes extends INodeAttributes {
}
public static class CopyWithQuota extends INodeDirectoryAttributes.SnapshotCopy {
- private final long nsQuota;
- private final long dsQuota;
-
+ private QuotaCounts quota;
public CopyWithQuota(byte[] name, PermissionStatus permissions,
AclFeature aclFeature, long modificationTime, long nsQuota,
- long dsQuota, XAttrFeature xAttrsFeature) {
+ long dsQuota, EnumCounters<StorageType> typeQuotas, XAttrFeature xAttrsFeature) {
super(name, permissions, aclFeature, modificationTime, xAttrsFeature);
- this.nsQuota = nsQuota;
- this.dsQuota = dsQuota;
+ this.quota = new QuotaCounts.Builder().nameCount(nsQuota).
+ spaceCount(dsQuota).typeCounts(typeQuotas).build();
}
public CopyWithQuota(INodeDirectory dir) {
super(dir);
Preconditions.checkArgument(dir.isQuotaSet());
- final Quota.Counts q = dir.getQuotaCounts();
- this.nsQuota = q.get(Quota.NAMESPACE);
- this.dsQuota = q.get(Quota.DISKSPACE);
+ final QuotaCounts q = dir.getQuotaCounts();
+ this.quota = new QuotaCounts.Builder().quotaCount(q).build();
}
-
+
@Override
- public Quota.Counts getQuotaCounts() {
- return Quota.Counts.newInstance(nsQuota, dsQuota);
+ public QuotaCounts getQuotaCounts() {
+ return new QuotaCounts.Builder().quotaCount(quota).build();
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
index 81f6ae5..8660947 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeFile.java
@@ -31,6 +31,7 @@ import java.util.Set;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoContiguous;
@@ -42,6 +43,7 @@ import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiff;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileDiffList;
import org.apache.hadoop.hdfs.server.namenode.snapshot.FileWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.util.LongBitFormat;
import com.google.common.annotations.VisibleForTesting;
@@ -488,21 +490,22 @@ public class INodeFile extends INodeWithAdditionalFields
}
@Override
- public Quota.Counts cleanSubtree(final int snapshot, int priorSnapshotId,
+ public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps, final int snapshot,
+ int priorSnapshotId,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) {
- return sf.cleanFile(this, snapshot, priorSnapshotId, collectedBlocks,
+ return sf.cleanFile(bsps, this, snapshot, priorSnapshotId, collectedBlocks,
removedINodes);
}
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
if (snapshot == CURRENT_STATE_ID) {
if (priorSnapshotId == NO_SNAPSHOT_ID) {
// this only happens when deleting the current file and the file is not
// in any snapshot
- computeQuotaUsage(counts, false);
- destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ computeQuotaUsage(bsps, counts, false);
+ destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
} else {
// when deleting the current file and the file is in snapshot, we should
// clean the 0-sized block if the file is UC
@@ -516,8 +519,8 @@ public class INodeFile extends INodeWithAdditionalFields
}
@Override
- public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks,
- final List<INode> removedINodes) {
+ public void destroyAndCollectBlocks(BlockStoragePolicySuite bsps,
+ BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
if (blocks != null && collectedBlocks != null) {
for (BlockInfoContiguous blk : blocks) {
collectedBlocks.addDeleteBlock(blk);
@@ -543,11 +546,15 @@ public class INodeFile extends INodeWithAdditionalFields
return getFullPathName();
}
+ // This is the only place that needs to use the BlockStoragePolicySuite to
+ // derive the intended storage type usage for quota by storage type
@Override
- public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
- boolean useCache, int lastSnapshotId) {
+ public final QuotaCounts computeQuotaUsage(
+ BlockStoragePolicySuite bsps, QuotaCounts counts, boolean useCache,
+ int lastSnapshotId) {
long nsDelta = 1;
- final long dsDelta;
+ final long dsDeltaNoReplication;
+ short dsReplication;
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if (sf != null) {
FileDiffList fileDiffList = sf.getDiffs();
@@ -555,18 +562,33 @@ public class INodeFile extends INodeWithAdditionalFields
if (lastSnapshotId == Snapshot.CURRENT_STATE_ID
|| last == Snapshot.CURRENT_STATE_ID) {
- dsDelta = diskspaceConsumed();
+ dsDeltaNoReplication = diskspaceConsumedNoReplication();
+ dsReplication = getBlockReplication();
} else if (last < lastSnapshotId) {
- dsDelta = computeFileSize(true, false) * getFileReplication();
- } else {
+ dsDeltaNoReplication = computeFileSize(true, false);
+ dsReplication = getFileReplication();
+ } else {
int sid = fileDiffList.getSnapshotById(lastSnapshotId);
- dsDelta = diskspaceConsumed(sid);
+ dsDeltaNoReplication = diskspaceConsumedNoReplication(sid);
+ dsReplication = getReplication(sid);
}
} else {
- dsDelta = diskspaceConsumed();
+ dsDeltaNoReplication = diskspaceConsumedNoReplication();
+ dsReplication = getBlockReplication();
+ }
+ counts.addNameSpace(nsDelta);
+ counts.addDiskSpace(dsDeltaNoReplication * dsReplication);
+
+ if (getStoragePolicyID() != BlockStoragePolicySuite.ID_UNSPECIFIED){
+ BlockStoragePolicy bsp = bsps.getPolicy(getStoragePolicyID());
+ List<StorageType> storageTypes = bsp.chooseStorageTypes(dsReplication);
+ for (StorageType t : storageTypes) {
+ if (!t.supportTypeQuota()) {
+ continue;
+ }
+ counts.addTypeSpace(t, dsDeltaNoReplication);
+ }
}
- counts.add(Quota.NAMESPACE, nsDelta);
- counts.add(Quota.DISKSPACE, dsDelta);
return counts;
}
@@ -660,9 +682,13 @@ public class INodeFile extends INodeWithAdditionalFields
* Use preferred block size for the last block if it is under construction.
*/
public final long diskspaceConsumed() {
+ return diskspaceConsumedNoReplication() * getBlockReplication();
+ }
+
+ public final long diskspaceConsumedNoReplication() {
FileWithSnapshotFeature sf = getFileWithSnapshotFeature();
if(sf == null) {
- return computeFileSize(true, true) * getBlockReplication();
+ return computeFileSize(true, true);
}
// Collect all distinct blocks
@@ -684,18 +710,34 @@ public class INodeFile extends INodeWithAdditionalFields
lastBlock instanceof BlockInfoContiguousUnderConstruction) {
size += getPreferredBlockSize() - lastBlock.getNumBytes();
}
- return size * getBlockReplication();
+ return size;
}
public final long diskspaceConsumed(int lastSnapshotId) {
if (lastSnapshotId != CURRENT_STATE_ID) {
return computeFileSize(lastSnapshotId)
- * getFileReplication(lastSnapshotId);
+ * getFileReplication(lastSnapshotId);
} else {
return diskspaceConsumed();
}
}
-
+
+ public final short getReplication(int lastSnapshotId) {
+ if (lastSnapshotId != CURRENT_STATE_ID) {
+ return getFileReplication(lastSnapshotId);
+ } else {
+ return getBlockReplication();
+ }
+ }
+
+ public final long diskspaceConsumedNoReplication(int lastSnapshotId) {
+ if (lastSnapshotId != CURRENT_STATE_ID) {
+ return computeFileSize(lastSnapshotId);
+ } else {
+ return diskspaceConsumedNoReplication();
+ }
+ }
+
/**
* Return the penultimate allocated block for this file.
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
index 8629bf8..cb270bf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeMap.java
@@ -23,7 +23,6 @@ import java.util.List;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
-import org.apache.hadoop.hdfs.server.namenode.Quota.Counts;
import org.apache.hadoop.util.GSet;
import org.apache.hadoop.util.LightWeightGSet;
@@ -97,17 +96,18 @@ public class INodeMap {
}
@Override
- public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks,
- List<INode> removedINodes) {
+ public void destroyAndCollectBlocks(BlockStoragePolicySuite bsps,
+ BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
// Nothing to do
}
-
+
@Override
- public Counts computeQuotaUsage(Counts counts, boolean useCache,
- int lastSnapshotId) {
+ public QuotaCounts computeQuotaUsage(
+ BlockStoragePolicySuite bsps, QuotaCounts counts,
+ boolean useCache, int lastSnapshotId) {
return null;
}
-
+
@Override
public ContentSummaryComputationContext computeContentSummary(
ContentSummaryComputationContext summary) {
@@ -115,9 +115,10 @@ public class INodeMap {
}
@Override
- public Counts cleanSubtree(int snapshotId, int priorSnapshotId,
+ public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
+ int snapshotId, int priorSnapshotId,
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
- return null;
+ return null;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
index b39151a..f8c813c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeReference.java
@@ -26,6 +26,7 @@ import java.util.List;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.snapshot.DirectoryWithSnapshotFeature;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
@@ -301,17 +302,19 @@ public abstract class INodeReference extends INode {
}
@Override // used by WithCount
- public Quota.Counts cleanSubtree(int snapshot, int prior,
- BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
- return referred.cleanSubtree(snapshot, prior, collectedBlocks,
+ public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps, int snapshot,
+ int prior, BlocksMapUpdateInfo collectedBlocks,
+ final List<INode> removedINodes) {
+ return referred.cleanSubtree(bsps, snapshot, prior, collectedBlocks,
removedINodes);
}
@Override // used by WithCount
public void destroyAndCollectBlocks(
+ BlockStoragePolicySuite bsps,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
if (removeReference(this) <= 0) {
- referred.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ referred.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
}
@@ -322,18 +325,19 @@ public abstract class INodeReference extends INode {
}
@Override
- public Quota.Counts computeQuotaUsage(Quota.Counts counts, boolean useCache,
- int lastSnapshotId) {
- return referred.computeQuotaUsage(counts, useCache, lastSnapshotId);
+ public QuotaCounts computeQuotaUsage(
+ BlockStoragePolicySuite bsps,
+ QuotaCounts counts, boolean useCache, int lastSnapshotId) {
+ return referred.computeQuotaUsage(bsps, counts, useCache, lastSnapshotId);
}
-
+
@Override
public final INodeAttributes getSnapshotINode(int snapshotId) {
return referred.getSnapshotINode(snapshotId);
}
@Override
- public Quota.Counts getQuotaCounts() {
+ public QuotaCounts getQuotaCounts() {
return referred.getQuotaCounts();
}
@@ -506,15 +510,15 @@ public abstract class INodeReference extends INode {
public final ContentSummaryComputationContext computeContentSummary(
ContentSummaryComputationContext summary) {
//only count diskspace for WithName
- final Quota.Counts q = Quota.Counts.newInstance();
- computeQuotaUsage(q, false, lastSnapshotId);
- summary.getCounts().add(Content.DISKSPACE, q.get(Quota.DISKSPACE));
+ final QuotaCounts q = new QuotaCounts.Builder().build();
+ computeQuotaUsage(summary.getBlockStoragePolicySuite(), q, false, lastSnapshotId);
+ summary.getCounts().add(Content.DISKSPACE, q.getDiskSpace());
return summary;
}
@Override
- public final Quota.Counts computeQuotaUsage(Quota.Counts counts,
- boolean useCache, int lastSnapshotId) {
+ public final QuotaCounts computeQuotaUsage(BlockStoragePolicySuite bsps,
+ QuotaCounts counts, boolean useCache, int lastSnapshotId) {
// if this.lastSnapshotId < lastSnapshotId, the rename of the referred
// node happened before the rename of its ancestor. This should be
// impossible since for WithName node we only count its children at the
@@ -529,12 +533,12 @@ public abstract class INodeReference extends INode {
// been updated by changes in the current tree.
int id = lastSnapshotId != Snapshot.CURRENT_STATE_ID ?
lastSnapshotId : this.lastSnapshotId;
- return referred.computeQuotaUsage(counts, false, id);
+ return referred.computeQuotaUsage(bsps, counts, false, id);
}
@Override
- public Quota.Counts cleanSubtree(final int snapshot, int prior,
- final BlocksMapUpdateInfo collectedBlocks,
+ public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
+ final int snapshot, int prior, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
// since WithName node resides in deleted list acting as a snapshot copy,
// the parameter snapshot must be non-null
@@ -547,16 +551,15 @@ public abstract class INodeReference extends INode {
if (prior != Snapshot.NO_SNAPSHOT_ID
&& Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
- return Quota.Counts.newInstance();
+ return new QuotaCounts.Builder().build();
}
- Quota.Counts counts = getReferredINode().cleanSubtree(snapshot, prior,
+ QuotaCounts counts = getReferredINode().cleanSubtree(bsps, snapshot, prior,
collectedBlocks, removedINodes);
INodeReference ref = getReferredINode().getParentReference();
if (ref != null) {
try {
- ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
- -counts.get(Quota.DISKSPACE), true);
+ ref.addSpaceConsumed(counts.negation(), true);
} catch (QuotaExceededException e) {
Log.warn("Should not have QuotaExceededException");
}
@@ -567,17 +570,18 @@ public abstract class INodeReference extends INode {
// in all the nodes existing at the time of the corresponding rename op.
// Thus if we are deleting a snapshot before/at the snapshot associated
// with lastSnapshotId, we do not need to update the quota upwards.
- counts = Quota.Counts.newInstance();
+ counts = new QuotaCounts.Builder().build();
}
return counts;
}
@Override
- public void destroyAndCollectBlocks(BlocksMapUpdateInfo collectedBlocks,
+ public void destroyAndCollectBlocks(BlockStoragePolicySuite bsps,
+ BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
int snapshot = getSelfSnapshot();
if (removeReference(this) <= 0) {
- getReferredINode().destroyAndCollectBlocks(collectedBlocks,
+ getReferredINode().destroyAndCollectBlocks(bsps, collectedBlocks,
removedINodes);
} else {
int prior = getPriorSnapshot(this);
@@ -597,12 +601,11 @@ public abstract class INodeReference extends INode {
return;
}
try {
- Quota.Counts counts = referred.cleanSubtree(snapshot, prior,
+ QuotaCounts counts = referred.cleanSubtree(bsps, snapshot, prior,
collectedBlocks, removedINodes);
INodeReference ref = getReferredINode().getParentReference();
if (ref != null) {
- ref.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
- -counts.get(Quota.DISKSPACE), true);
+ ref.addSpaceConsumed(counts.negation(), true);
}
} catch (QuotaExceededException e) {
LOG.error("should not exceed quota while snapshot deletion", e);
@@ -653,13 +656,13 @@ public abstract class INodeReference extends INode {
}
@Override
- public Quota.Counts cleanSubtree(int snapshot, int prior,
+ public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps, int snapshot, int prior,
BlocksMapUpdateInfo collectedBlocks, List<INode> removedINodes) {
if (snapshot == Snapshot.CURRENT_STATE_ID
&& prior == Snapshot.NO_SNAPSHOT_ID) {
- Quota.Counts counts = Quota.Counts.newInstance();
- this.computeQuotaUsage(counts, true);
- destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ this.computeQuotaUsage(bsps, counts, true);
+ destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
return counts;
} else {
// if prior is NO_SNAPSHOT_ID, we need to check snapshot belonging to
@@ -673,9 +676,9 @@ public abstract class INodeReference extends INode {
if (snapshot != Snapshot.CURRENT_STATE_ID
&& prior != Snapshot.NO_SNAPSHOT_ID
&& Snapshot.ID_INTEGER_COMPARATOR.compare(snapshot, prior) <= 0) {
- return Quota.Counts.newInstance();
+ return new QuotaCounts.Builder().build();
}
- return getReferredINode().cleanSubtree(snapshot, prior,
+ return getReferredINode().cleanSubtree(bsps, snapshot, prior,
collectedBlocks, removedINodes);
}
}
@@ -691,10 +694,10 @@ public abstract class INodeReference extends INode {
* WithName nodes.
*/
@Override
- public void destroyAndCollectBlocks(
+ public void destroyAndCollectBlocks(BlockStoragePolicySuite bsps,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
if (removeReference(this) <= 0) {
- getReferredINode().destroyAndCollectBlocks(collectedBlocks,
+ getReferredINode().destroyAndCollectBlocks(bsps, collectedBlocks,
removedINodes);
} else {
// we will clean everything, including files, directories, and
@@ -717,7 +720,7 @@ public abstract class INodeReference extends INode {
// when calling cleanSubtree of the referred node, since we
// compute quota usage updates before calling this destroy
// function, we use true for countDiffChange
- referred.cleanSubtree(snapshot, prior, collectedBlocks,
+ referred.cleanSubtree(bsps, snapshot, prior, collectedBlocks,
removedINodes);
} else if (referred.isDirectory()) {
// similarly, if referred is a directory, it must be an
@@ -725,7 +728,7 @@ public abstract class INodeReference extends INode {
INodeDirectory dir = referred.asDirectory();
Preconditions.checkState(dir.isWithSnapshot());
try {
- DirectoryWithSnapshotFeature.destroyDstSubtree(dir, snapshot,
+ DirectoryWithSnapshotFeature.destroyDstSubtree(bsps, dir, snapshot,
prior, collectedBlocks, removedINodes);
} catch (QuotaExceededException e) {
LOG.error("should not exceed quota while snapshot deletion", e);
@@ -733,7 +736,7 @@ public abstract class INodeReference extends INode {
}
}
}
-
+
private int getSelfSnapshot(final int prior) {
WithCount wc = (WithCount) getReferredINode().asReference();
INode referred = wc.getReferredINode();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
index 617c99a..ef30ed7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/INodeSymlink.java
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.fs.permission.PermissionStatus;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
/**
@@ -72,26 +73,29 @@ public class INodeSymlink extends INodeWithAdditionalFields {
}
@Override
- public Quota.Counts cleanSubtree(final int snapshotId, int priorSnapshotId,
+ public QuotaCounts cleanSubtree(BlockStoragePolicySuite bsps,
+ final int snapshotId, int priorSnapshotId,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
if (snapshotId == Snapshot.CURRENT_STATE_ID
&& priorSnapshotId == Snapshot.NO_SNAPSHOT_ID) {
- destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
- return Quota.Counts.newInstance(1, 0);
+ return new QuotaCounts.Builder().nameCount(1).build();
}
@Override
- public void destroyAndCollectBlocks(final BlocksMapUpdateInfo collectedBlocks,
+ public void destroyAndCollectBlocks(final BlockStoragePolicySuite bsps,
+ final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
removedINodes.add(this);
}
@Override
- public Quota.Counts computeQuotaUsage(Quota.Counts counts,
- boolean updateCache, int lastSnapshotId) {
- counts.add(Quota.NAMESPACE, 1);
+ public QuotaCounts computeQuotaUsage(
+ BlockStoragePolicySuite bsps,
+ QuotaCounts counts, boolean useCache, int lastSnapshotId) {
+ counts.addNameSpace(1);
return counts;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
index 848fa33..d235e2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeLayoutVersion.java
@@ -71,7 +71,8 @@ public class NameNodeLayoutVersion {
XATTRS_NAMESPACE_EXT(-59, "Increase number of xattr namespaces"),
BLOCK_STORAGE_POLICY(-60, "Block Storage policy"),
TRUNCATE(-61, "Truncate"),
- APPEND_NEW_BLOCK(-62, "Support appending to new block");
+ APPEND_NEW_BLOCK(-62, "Support appending to new block"),
+ QUOTA_BY_STORAGE_TYPE(-63, "Support quota for specific storage types");
private final FeatureInfo info;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 1c434e8..9d8199a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -68,7 +68,6 @@ import org.apache.hadoop.ha.protocolPB.HAServiceProtocolServerSideTranslatorPB;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
-import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.hdfs.inotify.EventBatch;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
import org.apache.hadoop.hdfs.protocol.AclException;
@@ -137,6 +136,7 @@ import org.apache.hadoop.hdfs.server.protocol.RemoteEditLogManifest;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.hdfs.StorageType;
import org.apache.hadoop.io.EnumSetWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
@@ -1196,11 +1196,7 @@ class NameNodeRpcServer implements NamenodeProtocols {
StorageType type)
throws IOException {
checkNNStartup();
- if (type != null) {
- throw new UnsupportedActionException(
- "Quota by storage type support is not fully supported by namenode yet.");
- }
- namesystem.setQuota(path, namespaceQuota, diskspaceQuota);
+ namesystem.setQuota(path, namespaceQuota, diskspaceQuota, type);
}
@Override // ClientProtocol
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
index 7abd017..6121bcb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/Quota.java
@@ -39,7 +39,7 @@ public enum Quota {
public static Counts newInstance() {
return newInstance(0, 0);
}
-
+
Counts() {
super(Quota.class);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java
new file mode 100644
index 0000000..d115acc
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaByStorageTypeEntry.java
@@ -0,0 +1,85 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.base.Objects;
+import org.apache.hadoop.hdfs.StorageType;
+
+ public class QuotaByStorageTypeEntry {
+ private StorageType type;
+ private long quota;
+
+ public StorageType getStorageType() {
+ return type;
+ }
+
+ public long getQuota() {
+ return quota;
+ }
+
+ @Override
+ public boolean equals(Object o){
+ if (o == null) {
+ return false;
+ }
+ if (getClass() != o.getClass()) {
+ return false;
+ }
+ QuotaByStorageTypeEntry other = (QuotaByStorageTypeEntry)o;
+ return Objects.equal(type, other.type) && Objects.equal(quota, other.quota);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(type, quota);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ assert (type != null);
+ sb.append(type.toString().toLowerCase());
+ sb.append(':');
+ sb.append(quota);
+ return sb.toString();
+ }
+
+ public static class Builder {
+ private StorageType type;
+ private long quota;
+
+ public Builder setStorageType(StorageType type) {
+ this.type = type;
+ return this;
+ }
+
+ public Builder setQuota(long quota) {
+ this.quota = quota;
+ return this;
+ }
+
+ public QuotaByStorageTypeEntry build() {
+ return new QuotaByStorageTypeEntry(type, quota);
+ }
+ }
+
+ private QuotaByStorageTypeEntry(StorageType type, long quota) {
+ this.type = type;
+ this.quota = quota;
+ }
+ }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java
new file mode 100644
index 0000000..9b306b0
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/QuotaCounts.java
@@ -0,0 +1,179 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.hdfs.StorageType;
+import org.apache.hadoop.hdfs.util.EnumCounters;
+
+/**
+ * Counters for namespace, space and storage type quota and usage.
+ */
+public class QuotaCounts {
+
+ private EnumCounters<Quota> nsSpCounts;
+ private EnumCounters<StorageType> typeCounts;
+
+ public static class Builder {
+ private EnumCounters<Quota> nsSpCounts;
+ private EnumCounters<StorageType> typeCounts;
+
+ public Builder() {
+ this.nsSpCounts = new EnumCounters<Quota>(Quota.class);
+ this.typeCounts = new EnumCounters<StorageType>(StorageType.class);
+ }
+
+ public Builder nameCount(long val) {
+ this.nsSpCounts.set(Quota.NAMESPACE, val);
+ return this;
+ }
+
+ public Builder spaceCount(long val) {
+ this.nsSpCounts.set(Quota.DISKSPACE, val);
+ return this;
+ }
+
+ public Builder typeCounts(EnumCounters<StorageType> val) {
+ if (val != null) {
+ this.typeCounts.set(val);
+ }
+ return this;
+ }
+
+ public Builder typeCounts(long val) {
+ this.typeCounts.reset(val);
+ return this;
+ }
+
+ public Builder quotaCount(QuotaCounts that) {
+ this.nsSpCounts.set(that.nsSpCounts);
+ this.typeCounts.set(that.typeCounts);
+ return this;
+ }
+
+ public QuotaCounts build() {
+ return new QuotaCounts(this);
+ }
+ }
+
+ private QuotaCounts(Builder builder) {
+ this.nsSpCounts = builder.nsSpCounts;
+ this.typeCounts = builder.typeCounts;
+ }
+
+ public void add(QuotaCounts that) {
+ this.nsSpCounts.add(that.nsSpCounts);
+ this.typeCounts.add(that.typeCounts);
+ }
+
+ public void subtract(QuotaCounts that) {
+ this.nsSpCounts.subtract(that.nsSpCounts);
+ this.typeCounts.subtract(that.typeCounts);
+ }
+
+ /**
+ * Returns a QuotaCounts whose value is {@code (-this)}.
+ *
+ * @return {@code -this}
+ */
+ public QuotaCounts negation() {
+ QuotaCounts ret = new QuotaCounts.Builder().quotaCount(this).build();
+ ret.nsSpCounts.negation();
+ ret.typeCounts.negation();
+ return ret;
+ }
+
+ public long getNameSpace(){
+ return nsSpCounts.get(Quota.NAMESPACE);
+ }
+
+ public void setNameSpace(long nameSpaceCount) {
+ this.nsSpCounts.set(Quota.NAMESPACE, nameSpaceCount);
+ }
+
+ public void addNameSpace(long nsDelta) {
+ this.nsSpCounts.add(Quota.NAMESPACE, nsDelta);
+ }
+
+ public long getDiskSpace(){
+ return nsSpCounts.get(Quota.DISKSPACE);
+ }
+
+ public void setDiskSpace(long spaceCount) {
+ this.nsSpCounts.set(Quota.DISKSPACE, spaceCount);
+ }
+
+ public void addDiskSpace(long dsDelta) {
+ this.nsSpCounts.add(Quota.DISKSPACE, dsDelta);
+ }
+
+ public EnumCounters<StorageType> getTypeSpaces() {
+ EnumCounters<StorageType> ret =
+ new EnumCounters<StorageType>(StorageType.class);
+ ret.set(typeCounts);
+ return ret;
+ }
+
+ void setTypeSpaces(EnumCounters<StorageType> that) {
+ if (that != null) {
+ this.typeCounts.set(that);
+ }
+ }
+
+ long getTypeSpace(StorageType type) {
+ return this.typeCounts.get(type);
+ }
+
+ void setTypeSpace(StorageType type, long spaceCount) {
+ this.typeCounts.set(type, spaceCount);
+ }
+
+ public void addTypeSpace(StorageType type, long delta) {
+ this.typeCounts.add(type, delta);
+ }
+
+ public void addTypeSpaces(EnumCounters<StorageType> deltas) {
+ this.typeCounts.add(deltas);
+ }
+
+ public boolean anyNsSpCountGreaterOrEqual(long val) {
+ return nsSpCounts.anyGreaterOrEqual(val);
+ }
+
+ public boolean anyTypeCountGreaterOrEqual(long val) {
+ return typeCounts.anyGreaterOrEqual(val);
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (obj == this) {
+ return true;
+ } else if (obj == null || !(obj instanceof QuotaCounts)) {
+ return false;
+ }
+ final QuotaCounts that = (QuotaCounts)obj;
+ return this.nsSpCounts.equals(that.nsSpCounts)
+ && this.typeCounts.equals(that.typeCounts);
+ }
+
+ @Override
+ public int hashCode() {
+ assert false : "hashCode not designed";
+ return 42; // any arbitrary constant will do
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
index 04e4fc9..691d717 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiff.java
@@ -21,10 +21,11 @@ import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import com.google.common.base.Preconditions;
@@ -114,17 +115,21 @@ abstract class AbstractINodeDiff<N extends INode,
}
/** Combine the posterior diff and collect blocks for deletion. */
- abstract Quota.Counts combinePosteriorAndCollectBlocks(final N currentINode,
+ abstract QuotaCounts combinePosteriorAndCollectBlocks(
+ final BlockStoragePolicySuite bsps, final N currentINode,
final D posterior, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes);
/**
* Delete and clear self.
+ * @param bsps The block storage policy suite used to retrieve storage policy
* @param currentINode The inode where the deletion happens.
* @param collectedBlocks Used to collect blocks for deletion.
+ * @param removedINodes INodes removed
* @return quota usage delta
*/
- abstract Quota.Counts destroyDiffAndCollectBlocks(final N currentINode,
+ abstract QuotaCounts destroyDiffAndCollectBlocks(
+ final BlockStoragePolicySuite bsps, final N currentINode,
final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes);
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
index 8187d0b..5bd4ed5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/AbstractINodeDiffList.java
@@ -22,10 +22,11 @@ import java.util.Collections;
import java.util.Iterator;
import java.util.List;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.INode;
import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.server.namenode.INodeAttributes;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
/**
* A list of snapshot diffs for storing snapshot data.
@@ -66,13 +67,14 @@ abstract class AbstractINodeDiffList<N extends INode,
* @param collectedBlocks Used to collect information for blocksMap update
* @return delta in namespace.
*/
- public final Quota.Counts deleteSnapshotDiff(final int snapshot,
+ public final QuotaCounts deleteSnapshotDiff(BlockStoragePolicySuite bsps,
+ final int snapshot,
final int prior, final N currentINode,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
int snapshotIndex = Collections.binarySearch(diffs, snapshot);
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
D removed = null;
if (snapshotIndex == 0) {
if (prior != Snapshot.NO_SNAPSHOT_ID) { // there is still snapshot before
@@ -80,7 +82,7 @@ abstract class AbstractINodeDiffList<N extends INode,
diffs.get(snapshotIndex).setSnapshotId(prior);
} else { // there is no snapshot before
removed = diffs.remove(0);
- counts.add(removed.destroyDiffAndCollectBlocks(currentINode,
+ counts.add(removed.destroyDiffAndCollectBlocks(bsps, currentINode,
collectedBlocks, removedINodes));
}
} else if (snapshotIndex > 0) {
@@ -95,7 +97,7 @@ abstract class AbstractINodeDiffList<N extends INode,
}
counts.add(previous.combinePosteriorAndCollectBlocks(
- currentINode, removed, collectedBlocks, removedINodes));
+ bsps, currentINode, removed, collectedBlocks, removedINodes));
previous.setPosterior(removed.getPosterior());
removed.setPosterior(null);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
index 0ed99e6..5168f0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectorySnapshottableFeature.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
import org.apache.hadoop.hdfs.server.namenode.INode;
@@ -39,7 +40,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithCount;
import org.apache.hadoop.hdfs.server.namenode.INodeReference.WithName;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
import org.apache.hadoop.hdfs.util.Diff.ListType;
import org.apache.hadoop.hdfs.util.ReadOnlyList;
import org.apache.hadoop.util.Time;
@@ -202,7 +203,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
* @return The removed snapshot. Null if no snapshot with the given name
* exists.
*/
- public Snapshot removeSnapshot(INodeDirectory snapshotRoot,
+ public Snapshot removeSnapshot(BlockStoragePolicySuite bsps, INodeDirectory snapshotRoot,
String snapshotName, BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) throws SnapshotException {
final int i = searchSnapshot(DFSUtil.string2Bytes(snapshotName));
@@ -214,14 +215,13 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
final Snapshot snapshot = snapshotsByNames.get(i);
int prior = Snapshot.findLatestSnapshot(snapshotRoot, snapshot.getId());
try {
- Quota.Counts counts = snapshotRoot.cleanSubtree(snapshot.getId(),
+ QuotaCounts counts = snapshotRoot.cleanSubtree(bsps, snapshot.getId(),
prior, collectedBlocks, removedINodes);
INodeDirectory parent = snapshotRoot.getParent();
if (parent != null) {
// there will not be any WithName node corresponding to the deleted
// snapshot, thus only update the quota usage in the current tree
- parent.addSpaceConsumed(-counts.get(Quota.NAMESPACE),
- -counts.get(Quota.DISKSPACE), true);
+ parent.addSpaceConsumed(counts.negation(), true);
}
} catch(QuotaExceededException e) {
INode.LOG.error("BUG: removeSnapshot increases namespace usage.", e);
@@ -233,6 +233,7 @@ public class DirectorySnapshottableFeature extends DirectoryWithSnapshotFeature
}
public ContentSummaryComputationContext computeContentSummary(
+ final BlockStoragePolicySuite bsps,
final INodeDirectory snapshotRoot,
final ContentSummaryComputationContext summary) {
snapshotRoot.computeContentSummary(summary);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5dae97a5/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
index a0008de..07ff744 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/snapshot/DirectoryWithSnapshotFeature.java
@@ -28,6 +28,7 @@ import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.namenode.AclStorage;
import org.apache.hadoop.hdfs.server.namenode.Content;
import org.apache.hadoop.hdfs.server.namenode.ContentSummaryComputationContext;
@@ -38,7 +39,7 @@ import org.apache.hadoop.hdfs.server.namenode.INodeDirectory;
import org.apache.hadoop.hdfs.server.namenode.INodeDirectoryAttributes;
import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.INodeReference;
-import org.apache.hadoop.hdfs.server.namenode.Quota;
+import org.apache.hadoop.hdfs.server.namenode.QuotaCounts;
import org.apache.hadoop.hdfs.server.namenode.snapshot.SnapshotFSImageFormat.ReferenceMap;
import org.apache.hadoop.hdfs.util.Diff;
import org.apache.hadoop.hdfs.util.Diff.Container;
@@ -94,14 +95,16 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
/** clear the created list */
- private Quota.Counts destroyCreatedList(final INodeDirectory currentINode,
+ private QuotaCounts destroyCreatedList(
+ final BlockStoragePolicySuite bsps,
+ final INodeDirectory currentINode,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
final List<INode> createdList = getList(ListType.CREATED);
for (INode c : createdList) {
- c.computeQuotaUsage(counts, true);
- c.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ c.computeQuotaUsage(bsps, counts, true);
+ c.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
// c should be contained in the children list, remove it
currentINode.removeChild(c);
}
@@ -110,14 +113,15 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
/** clear the deleted list */
- private Quota.Counts destroyDeletedList(
+ private QuotaCounts destroyDeletedList(
+ final BlockStoragePolicySuite bsps,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
final List<INode> deletedList = getList(ListType.DELETED);
for (INode d : deletedList) {
- d.computeQuotaUsage(counts, false);
- d.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ d.computeQuotaUsage(bsps, counts, false);
+ d.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
deletedList.clear();
return counts;
@@ -204,18 +208,19 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
@Override
- Quota.Counts combinePosteriorAndCollectBlocks(
+ QuotaCounts combinePosteriorAndCollectBlocks(
+ final BlockStoragePolicySuite bsps,
final INodeDirectory currentDir, final DirectoryDiff posterior,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
- final Quota.Counts counts = Quota.Counts.newInstance();
+ final QuotaCounts counts = new QuotaCounts.Builder().build();
diff.combinePosterior(posterior.diff, new Diff.Processor<INode>() {
/** Collect blocks for deleted files. */
@Override
public void process(INode inode) {
if (inode != null) {
- inode.computeQuotaUsage(counts, false);
- inode.destroyAndCollectBlocks(collectedBlocks, removedINodes);
+ inode.computeQuotaUsage(bsps, counts, false);
+ inode.destroyAndCollectBlocks(bsps, collectedBlocks, removedINodes);
}
}
});
@@ -313,11 +318,12 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
@Override
- Quota.Counts destroyDiffAndCollectBlocks(INodeDirectory currentINode,
+ QuotaCounts destroyDiffAndCollectBlocks(
+ BlockStoragePolicySuite bsps, INodeDirectory currentINode,
BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
// this diff has been deleted
- Quota.Counts counts = Quota.Counts.newInstance();
- counts.add(diff.destroyDeletedList(collectedBlocks, removedINodes));
+ QuotaCounts counts = new QuotaCounts.Builder().build();
+ counts.add(diff.destroyDeletedList(bsps, collectedBlocks, removedINodes));
INodeDirectoryAttributes snapshotINode = getSnapshotINode();
if (snapshotINode != null && snapshotINode.getAclFeature() != null) {
AclStorage.removeAclFeature(snapshotINode.getAclFeature());
@@ -401,7 +407,8 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
/**
* Destroy a subtree under a DstReference node.
*/
- public static void destroyDstSubtree(INode inode, final int snapshot,
+ public static void destroyDstSubtree(
+ final BlockStoragePolicySuite bsps, INode inode, final int snapshot,
final int prior, final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) throws QuotaExceededException {
Preconditions.checkArgument(prior != Snapshot.NO_SNAPSHOT_ID);
@@ -410,14 +417,14 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
&& snapshot != Snapshot.CURRENT_STATE_ID) {
// this inode has been renamed before the deletion of the DstReference
// subtree
- inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes);
+ inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
} else {
// for DstReference node, continue this process to its subtree
- destroyDstSubtree(inode.asReference().getReferredINode(), snapshot,
+ destroyDstSubtree(bsps, inode.asReference().getReferredINode(), snapshot,
prior, collectedBlocks, removedINodes);
}
} else if (inode.isFile()) {
- inode.cleanSubtree(snapshot, prior, collectedBlocks, removedINodes);
+ inode.cleanSubtree(bsps, snapshot, prior, collectedBlocks, removedINodes);
} else if (inode.isDirectory()) {
Map<INode, INode> excludedNodes = null;
INodeDirectory dir = inode.asDirectory();
@@ -431,12 +438,12 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
if (snapshot != Snapshot.CURRENT_STATE_ID) {
- diffList.deleteSnapshotDiff(snapshot, prior, dir, collectedBlocks,
+ diffList.deleteSnapshotDiff(bsps, snapshot, prior, dir, collectedBlocks,
removedINodes);
}
priorDiff = diffList.getDiffById(prior);
if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
- priorDiff.diff.destroyCreatedList(dir, collectedBlocks,
+ priorDiff.diff.destroyCreatedList(bsps, dir, collectedBlocks,
removedINodes);
}
}
@@ -444,7 +451,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
if (excludedNodes != null && excludedNodes.containsKey(child)) {
continue;
}
- destroyDstSubtree(child, snapshot, prior, collectedBlocks,
+ destroyDstSubtree(bsps, child, snapshot, prior, collectedBlocks,
removedINodes);
}
}
@@ -453,17 +460,19 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
/**
* Clean an inode while we move it from the deleted list of post to the
* deleted list of prior.
+ * @param bsps The block storage policy suite.
* @param inode The inode to clean.
* @param post The post snapshot.
* @param prior The id of the prior snapshot.
* @param collectedBlocks Used to collect blocks for later deletion.
* @return Quota usage update.
*/
- private static Quota.Counts cleanDeletedINode(INode inode,
+ private static QuotaCounts cleanDeletedINode(
+ final BlockStoragePolicySuite bsps, INode inode,
final int post, final int prior,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
Deque<INode> queue = new ArrayDeque<INode>();
queue.addLast(inode);
while (!queue.isEmpty()) {
@@ -471,13 +480,13 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
if (topNode instanceof INodeReference.WithName) {
INodeReference.WithName wn = (INodeReference.WithName) topNode;
if (wn.getLastSnapshotId() >= post) {
- wn.cleanSubtree(post, prior, collectedBlocks, removedINodes);
+ wn.cleanSubtree(bsps, post, prior, collectedBlocks, removedINodes);
}
// For DstReference node, since the node is not in the created list of
// prior, we should treat it as regular file/dir
} else if (topNode.isFile() && topNode.asFile().isWithSnapshot()) {
INodeFile file = topNode.asFile();
- counts.add(file.getDiffs().deleteSnapshotDiff(post, prior, file,
+ counts.add(file.getDiffs().deleteSnapshotDiff(bsps, post, prior, file,
collectedBlocks, removedINodes));
} else if (topNode.isDirectory()) {
INodeDirectory dir = topNode.asDirectory();
@@ -489,7 +498,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
DirectoryDiff priorDiff = sf.getDiffs().getDiffById(prior);
if (priorDiff != null && priorDiff.getSnapshotId() == prior) {
priorChildrenDiff = priorDiff.getChildrenDiff();
- counts.add(priorChildrenDiff.destroyCreatedList(dir,
+ counts.add(priorChildrenDiff.destroyCreatedList(bsps, dir,
collectedBlocks, removedINodes));
}
}
@@ -619,27 +628,29 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
diff.diff.modify(snapshotCopy, child);
return child;
}
-
- public void clear(INodeDirectory currentINode,
+
+ public void clear(BlockStoragePolicySuite bsps, INodeDirectory currentINode,
final BlocksMapUpdateInfo collectedBlocks, final List<INode> removedINodes) {
// destroy its diff list
for (DirectoryDiff diff : diffs) {
- diff.destroyDiffAndCollectBlocks(currentINode, collectedBlocks,
- removedINodes);
+ diff.destroyDiffAndCollectBlocks(bsps, currentINode, collectedBlocks,
+ removedINodes);
}
diffs.clear();
}
-
- public Quota.Counts computeQuotaUsage4CurrentDirectory(Quota.Counts counts) {
+
+ public QuotaCounts computeQuotaUsage4CurrentDirectory(
+ BlockStoragePolicySuite bsps, QuotaCounts counts) {
for(DirectoryDiff d : diffs) {
for(INode deleted : d.getChildrenDiff().getList(ListType.DELETED)) {
- deleted.computeQuotaUsage(counts, false, Snapshot.CURRENT_STATE_ID);
+ deleted.computeQuotaUsage(bsps, counts, false, Snapshot.CURRENT_STATE_ID);
}
}
return counts;
}
-
- public void computeContentSummary4Snapshot(final Content.Counts counts) {
+
+ public void computeContentSummary4Snapshot(final BlockStoragePolicySuite bsps,
+ final Content.Counts counts) {
// Create a new blank summary context for blocking processing of subtree.
ContentSummaryComputationContext summary =
new ContentSummaryComputationContext();
@@ -706,11 +717,11 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
}
- public Quota.Counts cleanDirectory(final INodeDirectory currentINode,
+ public QuotaCounts cleanDirectory(final BlockStoragePolicySuite bsps, final INodeDirectory currentINode,
final int snapshot, int prior,
final BlocksMapUpdateInfo collectedBlocks,
final List<INode> removedINodes) {
- Quota.Counts counts = Quota.Counts.newInstance();
+ QuotaCounts counts = new QuotaCounts.Builder().build();
Map<INode, INode> priorCreated = null;
Map<INode, INode> priorDeleted = null;
if (snapshot == Snapshot.CURRENT_STATE_ID) { // delete the current directory
@@ -718,10 +729,10 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
// delete everything in created list
DirectoryDiff lastDiff = diffs.getLast();
if (lastDiff != null) {
- counts.add(lastDiff.diff.destroyCreatedList(currentINode,
+ counts.add(lastDiff.diff.destroyCreatedList(bsps, currentINode,
collectedBlocks, removedINodes));
}
- counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
+ counts.add(currentINode.cleanSubtreeRecursively(bsps, snapshot, prior,
collectedBlocks, removedINodes, priorDeleted));
} else {
// update prior
@@ -738,9 +749,9 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
}
}
- counts.add(getDiffs().deleteSnapshotDiff(snapshot, prior,
+ counts.add(getDiffs().deleteSnapshotDiff(bsps, snapshot, prior,
currentINode, collectedBlocks, removedINodes));
- counts.add(currentINode.cleanSubtreeRecursively(snapshot, prior,
+ counts.add(currentINode.cleanSubtreeRecursively(bsps, snapshot, prior,
collectedBlocks, removedINodes, priorDeleted));
// check priorDiff again since it may be created during the diff deletion
@@ -757,7 +768,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
for (INode cNode : priorDiff.getChildrenDiff().getList(
ListType.CREATED)) {
if (priorCreated.containsKey(cNode)) {
- counts.add(cNode.cleanSubtree(snapshot, Snapshot.NO_SNAPSHOT_ID,
+ counts.add(cNode.cleanSubtree(bsps, snapshot, Snapshot.NO_SNAPSHOT_ID,
collectedBlocks, removedINodes));
}
}
@@ -774,7 +785,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
for (INode dNode : priorDiff.getChildrenDiff().getList(
ListType.DELETED)) {
if (priorDeleted == null || !priorDeleted.containsKey(dNode)) {
- counts.add(cleanDeletedINode(dNode, snapshot, prior,
+ counts.add(cleanDeletedINode(bsps, dNode, snapshot, prior,
collectedBlocks, removedINodes));
}
}
@@ -784,7 +795,7 @@ public class DirectoryWithSnapshotFeature implements INode.Feature {
if (currentINode.isQuotaSet()) {
currentINode.getDirectoryWithQuotaFeature().addSpaceConsumed2Cache(
- -counts.get(Quota.NAMESPACE), -counts.get(Quota.DISKSPACE));
+ counts.negation());
}
return counts;
}
[18/50] [abbrv] hadoop git commit: HDFS-7058. Tests for truncate CLI.
Contributed by Dasha Boudnik.
Posted by zj...@apache.org.
HDFS-7058. Tests for truncate CLI. Contributed by Dasha Boudnik.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b73956fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b73956fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b73956fd
Branch: refs/heads/YARN-2928
Commit: b73956fdad87e03e9875a96465579a1a3b17c498
Parents: 3d15728
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Mon Feb 9 21:26:59 2015 -0800
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Mon Feb 9 21:26:59 2015 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../src/test/resources/testHDFSConf.xml | 36 ++++++++++++++++++++
2 files changed, 38 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b73956fd/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 1ca2263..59d498d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -606,6 +606,8 @@ Release 2.7.0 - UNRELEASED
HDFS-7743. Code cleanup of BlockInfo and rename BlockInfo to
BlockInfoContiguous. (jing9)
+ HDFS-7058. Tests for truncate CLI. (Dasha Boudnik via shv)
+
OPTIMIZATIONS
HDFS-7454. Reduce memory footprint for AclEntries in NameNode.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b73956fd/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
index 8b221d6..05546a2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testHDFSConf.xml
@@ -17266,5 +17266,41 @@ $</expected-output>
</comparator>
</comparators>
</test>
+ <test> <!-- TESTED -->
+ <description>truncate to 5 bytes after waiting for block recovery to complete</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /user/USERNAME/dir0</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data120bytes /user/USERNAME/dir0</command>
+ <command>-fs NAMENODE -truncate -w 5 /user/USERNAME/dir0/data120bytes</command>
+ <command>-fs NAMENODE -cat /user/USERNAME/dir0/data120bytes</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /user/USERNAME/dir0</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>12345</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test> <!-- TESTED -->
+ <description>truncate to greater length than length of file</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /user/USERNAME/dir0</command>
+ <command>-fs NAMENODE -put CLITEST_DATA/data15bytes /user/USERNAME/dir0</command>
+ <command>-fs NAMENODE -truncate -w 50 /user/USERNAME/dir0/data15bytes</command>
+ <command>-fs NAMENODE -cat /user/USERNAME/dir0/data15bytes</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r /user/USERNAME/dir0</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>12345678901234</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>
[30/50] [abbrv] hadoop git commit: YARN-2246. Made the proxy tracking
URL always be http(s)://proxy addr:port/proxy/ to avoid duplicate
sections. Contributed by Devaraj K.
Posted by zj...@apache.org.
YARN-2246. Made the proxy tracking URL always be http(s)://proxy addr:port/proxy/<appId> to avoid duplicate sections. Contributed by Devaraj K.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5855c0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5855c0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5855c0e
Branch: refs/heads/YARN-2928
Commit: d5855c0e46404cfc1b5a63e59015e68ba668f0ea
Parents: e9d26fe
Author: Zhijie Shen <zj...@apache.org>
Authored: Tue Feb 10 15:24:01 2015 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Tue Feb 10 15:24:01 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 4 ++++
.../rmapp/attempt/RMAppAttemptImpl.java | 18 ++++++------------
.../attempt/TestRMAppAttemptTransitions.java | 9 ++-------
3 files changed, 12 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5855c0e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ee58b24..92e1803 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -533,6 +533,10 @@ Release 2.7.0 - UNRELEASED
YARN-2809. Implement workaround for linux kernel panic when removing
cgroup (Nathan Roberts via jlowe)
+ YARN-2246. Made the proxy tracking URL always be
+ http(s)://proxy addr:port/proxy/<appId> to avoid duplicate sections. (Devaraj
+ K via zjshen)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5855c0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 4c52d29..e1218ad 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -441,7 +441,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
this.readLock = lock.readLock();
this.writeLock = lock.writeLock();
- this.proxiedTrackingUrl = generateProxyUriWithScheme(null);
+ this.proxiedTrackingUrl = generateProxyUriWithScheme();
this.maybeLastAttempt = maybeLastAttempt;
this.stateMachine = stateMachineFactory.make(this);
@@ -534,21 +534,19 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
}
}
- private String generateProxyUriWithScheme(
- final String trackingUriWithoutScheme) {
+ private String generateProxyUriWithScheme() {
this.readLock.lock();
try {
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
- URI trackingUri = StringUtils.isEmpty(trackingUriWithoutScheme) ? null :
- ProxyUriUtils.getUriFromAMUrl(scheme, trackingUriWithoutScheme);
String proxy = WebAppUtils.getProxyHostAndPort(conf);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
+ URI result = ProxyUriUtils.getProxyUri(null, proxyUri,
applicationAttemptId.getApplicationId());
return result.toASCIIString();
} catch (URISyntaxException e) {
- LOG.warn("Could not proxify "+trackingUriWithoutScheme,e);
- return trackingUriWithoutScheme;
+ LOG.warn("Could not proxify the uri for "
+ + applicationAttemptId.getApplicationId(), e);
+ return null;
} finally {
this.readLock.unlock();
}
@@ -811,7 +809,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
recoverAppAttemptCredentials(credentials, attemptState.getState());
this.recoveredFinalState = attemptState.getState();
this.originalTrackingUrl = attemptState.getFinalTrackingUrl();
- this.proxiedTrackingUrl = generateProxyUriWithScheme(originalTrackingUrl);
this.finalStatus = attemptState.getFinalApplicationStatus();
this.startTime = attemptState.getStartTime();
this.finishTime = attemptState.getFinishTime();
@@ -1365,8 +1362,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
appAttempt.rpcPort = registrationEvent.getRpcport();
appAttempt.originalTrackingUrl =
sanitizeTrackingUrl(registrationEvent.getTrackingurl());
- appAttempt.proxiedTrackingUrl =
- appAttempt.generateProxyUriWithScheme(appAttempt.originalTrackingUrl);
// Let the app know
appAttempt.eventHandler.handle(new RMAppEvent(appAttempt
@@ -1584,7 +1579,6 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
(RMAppAttemptUnregistrationEvent) event;
diagnostics.append(unregisterEvent.getDiagnostics());
originalTrackingUrl = sanitizeTrackingUrl(unregisterEvent.getFinalTrackingUrl());
- proxiedTrackingUrl = generateProxyUriWithScheme(originalTrackingUrl);
finalStatus = unregisterEvent.getFinalApplicationStatus();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5855c0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index fc653dc..fee40e7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -44,7 +44,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -345,14 +344,10 @@ public class TestRMAppAttemptTransitions {
String url = null;
final String scheme = WebAppUtils.getHttpSchemePrefix(conf);
try {
- URI trackingUri =
- StringUtils.isEmpty(appAttempt.getOriginalTrackingUrl()) ? null :
- ProxyUriUtils
- .getUriFromAMUrl(scheme, appAttempt.getOriginalTrackingUrl());
String proxy = WebAppUtils.getProxyHostAndPort(conf);
URI proxyUri = ProxyUriUtils.getUriFromAMUrl(scheme, proxy);
- URI result = ProxyUriUtils.getProxyUri(trackingUri, proxyUri,
- appAttempt.getAppAttemptId().getApplicationId());
+ URI result = ProxyUriUtils.getProxyUri(null, proxyUri, appAttempt
+ .getAppAttemptId().getApplicationId());
url = result.toASCIIString();
} catch (URISyntaxException ex) {
Assert.fail();
[13/50] [abbrv] hadoop git commit: YARN-3155. Refactor the exception
handling code for TimelineClientImpl's retryOn method (Li Lu via wangda)
Posted by zj...@apache.org.
YARN-3155. Refactor the exception handling code for TimelineClientImpl's retryOn method (Li Lu via wangda)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/00a748d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/00a748d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/00a748d2
Branch: refs/heads/YARN-2928
Commit: 00a748d24a565bce0cc8cfa2bdcf165778cea395
Parents: af08425
Author: Wangda Tan <wa...@apache.org>
Authored: Mon Feb 9 17:20:25 2015 -0800
Committer: Wangda Tan <wa...@apache.org>
Committed: Mon Feb 9 17:20:34 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/client/api/impl/TimelineClientImpl.java | 15 +--------------
2 files changed, 4 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a748d2/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 634a0e7..2f0ef7a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -255,6 +255,9 @@ Release 2.7.0 - UNRELEASED
YARN-3144. Configuration for making delegation token failures to timeline
server not-fatal (Jonathan Eagles via jlowe)
+ YARN-3155. Refactor the exception handling code for TimelineClientImpl's
+ retryOn method (Li Lu via wangda)
+
OPTIMIZATIONS
YARN-2990. FairScheduler's delay-scheduling always waits for node-local and
http://git-wip-us.apache.org/repos/asf/hadoop/blob/00a748d2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 0b88632..af68492 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -178,20 +178,7 @@ public class TimelineClientImpl extends TimelineClient {
try {
// try perform the op, if fail, keep retrying
return op.run();
- } catch (IOException e) {
- // We may only throw runtime and IO exceptions. After switching to
- // Java 1.7, we can merge these two catch blocks into one.
-
- // break if there's no retries left
- if (leftRetries == 0) {
- break;
- }
- if (op.shouldRetryOn(e)) {
- logException(e, leftRetries);
- } else {
- throw e;
- }
- } catch (RuntimeException e) {
+ } catch (IOException | RuntimeException e) {
// break if there's no retries left
if (leftRetries == 0) {
break;
[09/50] [abbrv] hadoop git commit: YARN-3094. Reset timer for
liveness monitors after RM recovery. Contributed by Jun Gong
Posted by zj...@apache.org.
YARN-3094. Reset timer for liveness monitors after RM recovery. Contributed by Jun Gong
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0af6a99a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0af6a99a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0af6a99a
Branch: refs/heads/YARN-2928
Commit: 0af6a99a3fcfa4b47d3bcba5e5cc5fe7b312a152
Parents: 7d73202
Author: Jian He <ji...@apache.org>
Authored: Mon Feb 9 13:47:08 2015 -0800
Committer: Jian He <ji...@apache.org>
Committed: Mon Feb 9 13:47:08 2015 -0800
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/util/AbstractLivelinessMonitor.java | 8 ++
.../server/resourcemanager/ResourceManager.java | 2 +
.../rmapp/attempt/AMLivelinessMonitor.java | 6 ++
.../rmapp/attempt/TestAMLivelinessMonitor.java | 81 ++++++++++++++++++++
5 files changed, 100 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af6a99a/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cc2f34b..578a8cc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -516,6 +516,9 @@ Release 2.7.0 - UNRELEASED
YARN-3143. RM Apps REST API can return NPE or entries missing id and other
fields (jlowe)
+ YARN-3094. Reset timer for liveness monitors after RM recovery. (Jun Gong
+ via jianhe)
+
Release 2.6.0 - 2014-11-18
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af6a99a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java
index c182531..4f587b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/AbstractLivelinessMonitor.java
@@ -59,6 +59,7 @@ public abstract class AbstractLivelinessMonitor<O> extends AbstractService {
@Override
protected void serviceStart() throws Exception {
assert !stopped : "starting when already stopped";
+ resetTimer();
checkerThread = new Thread(new PingChecker());
checkerThread.setName("Ping Checker");
checkerThread.start();
@@ -99,6 +100,13 @@ public abstract class AbstractLivelinessMonitor<O> extends AbstractService {
running.remove(ob);
}
+ public synchronized void resetTimer() {
+ long time = clock.getTime();
+ for (O ob : running.keySet()) {
+ running.put(ob, time);
+ }
+ }
+
private class PingChecker implements Runnable {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af6a99a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
index 4f242e93..a93372a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
@@ -564,12 +564,14 @@ public class ResourceManager extends CompositeService implements Recoverable {
if(recoveryEnabled) {
try {
+ LOG.info("Recovery started");
rmStore.checkVersion();
if (rmContext.isWorkPreservingRecoveryEnabled()) {
rmContext.setEpoch(rmStore.getAndIncrementEpoch());
}
RMState state = rmStore.loadState();
recover(state);
+ LOG.info("Recovery ended");
} catch (Exception e) {
// the Exception from loadState() needs to be handled for
// HA and we need to give up master status if we got fenced
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af6a99a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java
index 2c1f7f1..76331bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/AMLivelinessMonitor.java
@@ -24,6 +24,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.util.AbstractLivelinessMonitor;
+import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.SystemClock;
public class AMLivelinessMonitor extends AbstractLivelinessMonitor<ApplicationAttemptId> {
@@ -35,6 +36,11 @@ public class AMLivelinessMonitor extends AbstractLivelinessMonitor<ApplicationAt
this.dispatcher = d.getEventHandler();
}
+ public AMLivelinessMonitor(Dispatcher d, Clock clock) {
+ super("AMLivelinessMonitor", clock);
+ this.dispatcher = d.getEventHandler();
+ }
+
public void serviceInit(Configuration conf) throws Exception {
super.serviceInit(conf);
int expireIntvl = conf.getInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0af6a99a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java
new file mode 100644
index 0000000..e0e6aee
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestAMLivelinessMonitor.java
@@ -0,0 +1,81 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt;
+
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.service.Service;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.util.ControlledClock;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.mockito.Mockito.mock;
+
+public class TestAMLivelinessMonitor {
+
+ @Test(timeout = 10000)
+ public void testResetTimer() throws Exception {
+ YarnConfiguration conf = new YarnConfiguration();
+ UserGroupInformation.setConfiguration(conf);
+ conf.set(YarnConfiguration.RECOVERY_ENABLED, "true");
+ conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+ conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED, true);
+ conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 6000);
+ final ControlledClock clock = new ControlledClock(new SystemClock());
+ clock.setTime(0);
+ MemoryRMStateStore memStore = new MemoryRMStateStore() {
+ @Override
+ public synchronized RMState loadState() throws Exception {
+ clock.setTime(8000);
+ return super.loadState();
+ }
+ };
+ memStore.init(conf);
+ final ApplicationAttemptId attemptId = mock(ApplicationAttemptId.class);
+ final Dispatcher dispatcher = mock(Dispatcher.class);
+ final boolean[] expired = new boolean[]{false};
+ final AMLivelinessMonitor monitor = new AMLivelinessMonitor(
+ dispatcher, clock) {
+ @Override
+ protected void expire(ApplicationAttemptId id) {
+ Assert.assertEquals(id, attemptId);
+ expired[0] = true;
+ }
+ };
+ monitor.register(attemptId);
+ MockRM rm = new MockRM(conf, memStore) {
+ @Override
+ protected AMLivelinessMonitor createAMLivelinessMonitor() {
+ return monitor;
+ }
+ };
+ rm.start();
+ // make sure that monitor has started
+ while (monitor.getServiceState() != Service.STATE.STARTED) {
+ Thread.sleep(100);
+ }
+ // expired[0] would be set to true without resetTimer
+ Assert.assertFalse(expired[0]);
+ rm.stop();
+ }
+}
[50/50] [abbrv] hadoop git commit: Merge remote-tracking branch
'apache/trunk' into YARN-2928
Posted by zj...@apache.org.
Merge remote-tracking branch 'apache/trunk' into YARN-2928
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e2786151
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e2786151
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e2786151
Branch: refs/heads/YARN-2928
Commit: e2786151da326ab763288c001677c17ca0de8dd3
Parents: f26941b f80c988
Author: Zhijie Shen <zj...@apache.org>
Authored: Wed Feb 11 11:40:48 2015 -0800
Committer: Zhijie Shen <zj...@apache.org>
Committed: Wed Feb 11 11:40:48 2015 -0800
----------------------------------------------------------------------
dev-support/determine-flaky-tests-hadoop.py | 204 +++
.../main/resources/assemblies/hadoop-dist.xml | 5 +
.../assemblies/hadoop-mapreduce-dist.xml | 6 +
.../resources/assemblies/hadoop-yarn-dist.xml | 5 +
hadoop-common-project/hadoop-common/CHANGES.txt | 139 ++
.../hadoop-common/dev-support/checkstyle.xml | 1 -
hadoop-common-project/hadoop-common/pom.xml | 2 +-
.../hadoop-common/src/CMakeLists.txt | 2 +-
.../hadoop-common/src/main/bin/hadoop | 12 +-
.../hadoop-common/src/main/bin/hadoop-config.sh | 26 +-
.../src/main/bin/hadoop-functions.sh | 344 ++---
.../hadoop-common/src/main/bin/hadoop.cmd | 6 +-
.../src/main/conf/log4j.properties | 4 +-
.../src/main/conf/shellprofile.d/example | 106 ++
.../org/apache/hadoop/conf/Configuration.java | 142 +-
.../apache/hadoop/fs/AbstractFileSystem.java | 19 +-
.../java/org/apache/hadoop/fs/ChecksumFs.java | 5 +
.../hadoop/fs/CommonConfigurationKeys.java | 3 -
.../fs/CommonConfigurationKeysPublic.java | 7 +-
.../org/apache/hadoop/fs/ContentSummary.java | 57 +-
.../java/org/apache/hadoop/fs/CreateFlag.java | 24 +-
.../apache/hadoop/fs/DelegateToFileSystem.java | 6 +
.../org/apache/hadoop/fs/FSDataInputStream.java | 2 +-
.../org/apache/hadoop/fs/FSOutputSummer.java | 2 +-
.../java/org/apache/hadoop/fs/FileContext.java | 43 +
.../java/org/apache/hadoop/fs/FilterFs.java | 8 +
.../java/org/apache/hadoop/fs/shell/Count.java | 32 +-
.../java/org/apache/hadoop/fs/shell/Ls.java | 183 ++-
.../org/apache/hadoop/fs/viewfs/ChRootedFs.java | 6 +
.../org/apache/hadoop/fs/viewfs/ViewFs.java | 17 +-
.../io/serializer/SerializationFactory.java | 2 +-
.../main/java/org/apache/hadoop/ipc/Client.java | 2 +-
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 27 +-
.../apache/hadoop/ipc/WritableRpcEngine.java | 4 +-
.../hadoop/metrics/ganglia/GangliaContext.java | 33 +-
.../apache/hadoop/metrics/ganglia/package.html | 6 +
.../hadoop/metrics2/impl/MetricsSystemImpl.java | 2 +-
.../sink/ganglia/AbstractGangliaSink.java | 36 +-
.../org/apache/hadoop/net/NetworkTopology.java | 12 +-
.../net/NetworkTopologyWithNodeGroup.java | 2 +-
.../hadoop/security/LdapGroupsMapping.java | 2 +
.../apache/hadoop/security/SaslRpcClient.java | 18 +-
.../ZKDelegationTokenSecretManager.java | 26 +-
.../apache/hadoop/tracing/SpanReceiverHost.java | 67 +-
.../hadoop/tracing/TraceSamplerFactory.java | 53 -
.../org/apache/hadoop/tracing/TraceUtils.java | 65 +
.../apache/hadoop/util/ComparableVersion.java | 2 -
.../java/org/apache/hadoop/util/FindClass.java | 388 ++++++
.../java/org/apache/hadoop/util/LogAdapter.java | 74 +
.../java/org/apache/hadoop/util/ProtoUtil.java | 4 +-
.../org/apache/hadoop/util/SignalLogger.java | 8 +-
.../org/apache/hadoop/util/StringUtils.java | 16 +
.../src/main/native/src/exception.c | 6 +
.../src/main/native/src/exception.h | 20 +-
.../hadoop/io/compress/bzip2/Bzip2Compressor.c | 28 +-
.../io/compress/bzip2/Bzip2Decompressor.c | 28 +-
.../org/apache/hadoop/io/nativeio/NativeIO.c | 7 +-
.../src/main/resources/core-default.xml | 157 ++-
.../src/site/apt/CLIMiniCluster.apt.vm | 83 --
.../src/site/apt/ClusterSetup.apt.vm | 651 ---------
.../src/site/apt/CommandsManual.apt.vm | 327 -----
.../src/site/apt/Compatibility.apt.vm | 541 --------
.../src/site/apt/DeprecatedProperties.apt.vm | 552 --------
.../src/site/apt/FileSystemShell.apt.vm | 746 ----------
.../src/site/apt/HttpAuthentication.apt.vm | 98 --
.../src/site/apt/InterfaceClassification.apt.vm | 239 ----
.../hadoop-common/src/site/apt/Metrics.apt.vm | 879 ------------
.../src/site/apt/NativeLibraries.apt.vm | 205 ---
.../src/site/apt/RackAwareness.apt.vm | 140 --
.../src/site/apt/SecureMode.apt.vm | 689 ---------
.../src/site/apt/ServiceLevelAuth.apt.vm | 216 ---
.../src/site/apt/SingleCluster.apt.vm | 286 ----
.../src/site/apt/SingleNodeSetup.apt.vm | 24 -
.../src/site/apt/Superusers.apt.vm | 144 --
.../hadoop-common/src/site/apt/Tracing.apt.vm | 233 ----
.../src/site/markdown/CLIMiniCluster.md.vm | 68 +
.../src/site/markdown/ClusterSetup.md | 339 +++++
.../src/site/markdown/CommandsManual.md | 237 ++++
.../src/site/markdown/Compatibility.md | 313 +++++
.../src/site/markdown/DeprecatedProperties.md | 288 ++++
.../src/site/markdown/FileSystemShell.md | 709 ++++++++++
.../src/site/markdown/HttpAuthentication.md | 58 +
.../site/markdown/InterfaceClassification.md | 221 +++
.../hadoop-common/src/site/markdown/Metrics.md | 456 ++++++
.../src/site/markdown/NativeLibraries.md.vm | 145 ++
.../src/site/markdown/RackAwareness.md | 136 ++
.../src/site/markdown/SecureMode.md | 377 +++++
.../src/site/markdown/ServiceLevelAuth.md | 144 ++
.../src/site/markdown/SingleCluster.md.vm | 232 ++++
.../src/site/markdown/Superusers.md | 106 ++
.../hadoop-common/src/site/markdown/Tracing.md | 182 +++
.../src/site/markdown/filesystem/filesystem.md | 35 +
.../apache/hadoop/conf/TestConfiguration.java | 109 +-
.../apache/hadoop/fs/FileSystemTestHelper.java | 27 +-
.../org/apache/hadoop/fs/SymlinkBaseTest.java | 2 +-
.../org/apache/hadoop/fs/TestAfsCheckPath.java | 6 +
.../apache/hadoop/fs/TestContentSummary.java | 6 +-
.../org/apache/hadoop/fs/shell/TestCount.java | 115 +-
.../java/org/apache/hadoop/fs/shell/TestLs.java | 1308 ++++++++++++++++++
.../io/serializer/TestSerializationFactory.java | 10 +
.../java/org/apache/hadoop/ipc/TestIPC.java | 51 +
.../metrics/ganglia/TestGangliaContext.java | 41 +
.../hadoop/metrics2/impl/MetricsRecords.java | 92 ++
.../metrics2/impl/TestMetricsSystemImpl.java | 2 +-
.../metrics2/sink/ganglia/TestGangliaSink.java | 81 ++
.../org/apache/hadoop/net/TestTableMapping.java | 43 +-
.../org/apache/hadoop/test/TestJUnitSetup.java | 39 +
.../apache/hadoop/tracing/TestTraceUtils.java | 51 +
.../org/apache/hadoop/util/TestFindClass.java | 218 +++
.../src/test/resources/testConf.xml | 50 +-
.../hadoop-kms/src/main/conf/kms-env.sh | 4 -
.../hadoop/crypto/key/kms/server/KMSACLs.java | 7 +-
.../crypto/key/kms/server/KMSConfiguration.java | 1 +
.../hadoop-kms/src/main/libexec/kms-config.sh | 7 +-
.../crypto/key/kms/server/TestKMSACLs.java | 18 +-
.../hadoop/fs/http/client/HttpFSUtils.java | 3 +-
.../http/server/HttpFSAuthenticationFilter.java | 8 +-
.../service/hadoop/FileSystemAccessService.java | 2 +-
.../apache/hadoop/lib/wsrs/JSONMapProvider.java | 3 +-
.../apache/hadoop/lib/wsrs/JSONProvider.java | 3 +-
.../src/main/libexec/httpfs-config.sh | 2 +-
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 2 +-
.../hadoop/hdfs/nfs/nfs3/AsyncDataService.java | 16 +-
.../hadoop/hdfs/nfs/nfs3/OpenFileCtx.java | 78 +-
.../hadoop/hdfs/nfs/nfs3/RpcProgramNfs3.java | 147 +-
.../hadoop/hdfs/nfs/nfs3/WriteManager.java | 31 +-
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 144 +-
.../hadoop-hdfs/dev-support/checkstyle.xml | 1 -
hadoop-hdfs-project/hadoop-hdfs/pom.xml | 7 +-
.../contrib/libwebhdfs/src/hdfs_http_client.c | 11 +-
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 35 +-
.../main/java/org/apache/hadoop/fs/Hdfs.java | 6 +
.../apache/hadoop/hdfs/BlockReaderLocal.java | 6 +-
.../hadoop/hdfs/BlockReaderLocalLegacy.java | 6 +-
.../hadoop/hdfs/BlockStorageLocationUtil.java | 8 +-
.../org/apache/hadoop/hdfs/ClientContext.java | 14 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 143 +-
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 16 +-
.../hadoop/hdfs/DFSInotifyEventInputStream.java | 6 +-
.../org/apache/hadoop/hdfs/DFSInputStream.java | 90 +-
.../org/apache/hadoop/hdfs/DFSOutputStream.java | 86 +-
.../hadoop/hdfs/DistributedFileSystem.java | 55 +-
.../apache/hadoop/hdfs/KeyProviderCache.java | 109 ++
.../apache/hadoop/hdfs/RemoteBlockReader.java | 6 +-
.../apache/hadoop/hdfs/RemoteBlockReader2.java | 6 +-
.../org/apache/hadoop/hdfs/StorageType.java | 32 +-
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 27 +
.../hadoop/hdfs/client/HdfsDataInputStream.java | 8 +-
.../hdfs/client/HdfsDataOutputStream.java | 8 +-
.../org/apache/hadoop/hdfs/inotify/Event.java | 12 +
.../hdfs/protocol/CacheDirectiveIterator.java | 6 +-
.../hadoop/hdfs/protocol/CachePoolIterator.java | 6 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 18 +-
.../hdfs/protocol/DatanodeInfoWithStorage.java | 63 +
.../hdfs/protocol/EncryptionZoneIterator.java | 6 +-
.../hadoop/hdfs/protocol/LocatedBlock.java | 75 +-
.../QuotaByStorageTypeExceededException.java | 56 +
.../datatransfer/DataTransferProtoUtil.java | 8 +-
.../hdfs/protocol/datatransfer/PipelineAck.java | 98 +-
.../hdfs/protocol/datatransfer/Receiver.java | 2 +-
.../hdfs/protocol/datatransfer/Sender.java | 4 +-
...tNamenodeProtocolServerSideTranslatorPB.java | 18 +-
.../ClientNamenodeProtocolTranslatorPB.java | 31 +-
.../apache/hadoop/hdfs/protocolPB/PBHelper.java | 25 +-
.../hadoop/hdfs/server/balancer/Balancer.java | 28 +-
.../hdfs/server/balancer/NameNodeConnector.java | 26 +-
.../server/blockmanagement/BlockCollection.java | 8 +-
.../hdfs/server/blockmanagement/BlockInfo.java | 403 ------
.../blockmanagement/BlockInfoContiguous.java | 412 ++++++
.../BlockInfoContiguousUnderConstruction.java | 400 ++++++
.../BlockInfoUnderConstruction.java | 402 ------
.../server/blockmanagement/BlockManager.java | 343 +++--
.../hdfs/server/blockmanagement/BlocksMap.java | 32 +-
.../CacheReplicationMonitor.java | 14 +-
.../blockmanagement/CorruptReplicasMap.java | 19 +-
.../blockmanagement/DatanodeDescriptor.java | 28 +-
.../server/blockmanagement/DatanodeManager.java | 8 +-
.../blockmanagement/DatanodeStorageInfo.java | 22 +-
.../blockmanagement/InvalidateBlocks.java | 8 +-
.../PendingReplicationBlocks.java | 4 +-
.../blockmanagement/UnderReplicatedBlocks.java | 48 +-
.../hdfs/server/datanode/BPServiceActor.java | 5 +
.../server/datanode/BlockPoolSliceStorage.java | 12 +-
.../hdfs/server/datanode/BlockReceiver.java | 49 +-
.../hdfs/server/datanode/BlockSender.java | 6 +-
.../hadoop/hdfs/server/datanode/DataNode.java | 19 +
.../hdfs/server/datanode/DataStorage.java | 15 +-
.../hdfs/server/datanode/VolumeScanner.java | 52 +-
.../datanode/fsdataset/ReplicaInputStreams.java | 8 +-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 57 +-
.../datanode/web/webhdfs/WebHdfsHandler.java | 3 +-
.../apache/hadoop/hdfs/server/mover/Mover.java | 7 +-
.../hdfs/server/namenode/BackupImage.java | 31 +-
.../hadoop/hdfs/server/namenode/BackupNode.java | 2 +-
.../ContentSummaryComputationContext.java | 5 +
.../namenode/DirectoryWithQuotaFeature.java | 228 ++-
.../hdfs/server/namenode/FSDirAttrOp.java | 64 +-
.../hdfs/server/namenode/FSDirConcatOp.java | 280 ++--
.../hdfs/server/namenode/FSDirDeleteOp.java | 21 +-
.../hdfs/server/namenode/FSDirRenameOp.java | 64 +-
.../hdfs/server/namenode/FSDirectory.java | 222 ++-
.../hadoop/hdfs/server/namenode/FSEditLog.java | 37 +-
.../hdfs/server/namenode/FSEditLogLoader.java | 105 +-
.../hdfs/server/namenode/FSEditLogOp.java | 186 ++-
.../hdfs/server/namenode/FSEditLogOpCodes.java | 2 +
.../hadoop/hdfs/server/namenode/FSImage.java | 60 +-
.../hdfs/server/namenode/FSImageFormat.java | 33 +-
.../server/namenode/FSImageFormatPBINode.java | 94 +-
.../server/namenode/FSImageSerialization.java | 16 +-
.../hdfs/server/namenode/FSNamesystem.java | 220 +--
.../namenode/FileUnderConstructionFeature.java | 17 +-
.../hadoop/hdfs/server/namenode/INode.java | 124 +-
.../hdfs/server/namenode/INodeDirectory.java | 129 +-
.../namenode/INodeDirectoryAttributes.java | 30 +-
.../hadoop/hdfs/server/namenode/INodeFile.java | 183 ++-
.../hadoop/hdfs/server/namenode/INodeMap.java | 27 +-
.../hdfs/server/namenode/INodeReference.java | 117 +-
.../hdfs/server/namenode/INodeSymlink.java | 25 +-
.../namenode/INodeWithAdditionalFields.java | 4 +-
.../namenode/InotifyFSEditLogOpTranslator.java | 4 +
.../hdfs/server/namenode/LeaseManager.java | 6 +-
.../hadoop/hdfs/server/namenode/NameNode.java | 92 +-
.../server/namenode/NameNodeLayoutVersion.java | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 26 +-
.../hdfs/server/namenode/NamenodeFsck.java | 4 +-
.../hadoop/hdfs/server/namenode/Namesystem.java | 4 +-
.../hadoop/hdfs/server/namenode/Quota.java | 2 +-
.../namenode/QuotaByStorageTypeEntry.java | 85 ++
.../hdfs/server/namenode/QuotaCounts.java | 179 +++
.../namenode/snapshot/AbstractINodeDiff.java | 11 +-
.../snapshot/AbstractINodeDiffList.java | 54 +-
.../snapshot/DirectorySnapshottableFeature.java | 13 +-
.../snapshot/DirectoryWithSnapshotFeature.java | 135 +-
.../snapshot/FSImageFormatPBSnapshot.java | 45 +-
.../hdfs/server/namenode/snapshot/FileDiff.java | 22 +-
.../server/namenode/snapshot/FileDiffList.java | 27 +-
.../snapshot/FileWithSnapshotFeature.java | 79 +-
.../namenode/snapshot/SnapshotManager.java | 3 +-
.../web/resources/NamenodeWebHdfsMethods.java | 30 +-
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 50 +-
.../org/apache/hadoop/hdfs/tools/DFSck.java | 2 +-
.../tools/offlineImageViewer/FSImageLoader.java | 4 +-
.../OfflineImageViewerPB.java | 21 +-
.../PBImageDelimitedTextWriter.java | 132 ++
.../offlineImageViewer/PBImageTextWriter.java | 586 ++++++++
.../hdfs/tools/snapshot/LsSnapshottableDir.java | 2 +-
.../hdfs/tools/snapshot/SnapshotDiff.java | 2 +-
.../apache/hadoop/hdfs/util/EnumCounters.java | 36 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 8 +
.../hdfs/web/resources/NewLengthParam.java | 49 +
.../hadoop/hdfs/web/resources/PostOpParam.java | 2 +
.../src/main/native/libhdfs/exception.c | 5 +
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.c | 33 +-
.../hadoop-hdfs/src/main/native/libhdfs/hdfs.h | 13 +
.../main/native/libhdfs/test_libhdfs_threaded.c | 4 +
.../src/main/proto/ClientNamenodeProtocol.proto | 3 +
.../src/main/proto/datatransfer.proto | 3 +-
.../hadoop-hdfs/src/main/proto/fsimage.proto | 10 +
.../hadoop-hdfs/src/main/proto/inotify.proto | 1 +
.../src/main/resources/hdfs-default.xml | 35 +-
.../hadoop-hdfs/src/main/shellprofile.d/hdfs | 36 +
.../src/main/webapps/hdfs/dfshealth.html | 9 +-
.../src/site/apt/HDFSCommands.apt.vm | 5 +-
.../hadoop-hdfs/src/site/apt/HdfsDesign.apt.vm | 12 +-
.../src/site/apt/HdfsQuotaAdminGuide.apt.vm | 10 +-
.../hadoop-hdfs/src/site/apt/WebHDFS.apt.vm | 46 +
.../fs/TestHDFSFileContextMainOperations.java | 32 +-
.../org/apache/hadoop/fs/TestSymlinkHdfs.java | 3 +-
.../org/apache/hadoop/hdfs/AppendTestUtil.java | 56 +
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 49 +-
.../org/apache/hadoop/hdfs/MiniDFSCluster.java | 4 +-
.../hdfs/TestDFSInotifyEventInputStream.java | 9 +-
.../hadoop/hdfs/TestDataTransferProtocol.java | 10 +-
.../apache/hadoop/hdfs/TestDatanodeDeath.java | 17 +-
.../apache/hadoop/hdfs/TestDecommission.java | 10 +-
.../apache/hadoop/hdfs/TestEncryptionZones.java | 10 +-
.../hadoop/hdfs/TestEncryptionZonesWithHA.java | 2 +-
.../hadoop/hdfs/TestEncryptionZonesWithKMS.java | 2 +-
.../org/apache/hadoop/hdfs/TestFileAppend.java | 165 ++-
.../org/apache/hadoop/hdfs/TestFileAppend2.java | 207 ++-
.../org/apache/hadoop/hdfs/TestFileAppend3.java | 227 ++-
.../org/apache/hadoop/hdfs/TestFileAppend4.java | 11 +-
.../hadoop/hdfs/TestFileAppendRestart.java | 10 +-
.../apache/hadoop/hdfs/TestFileCorruption.java | 15 +-
.../apache/hadoop/hdfs/TestFileCreation.java | 5 +-
.../hadoop/hdfs/TestFileCreationDelete.java | 9 +-
.../java/org/apache/hadoop/hdfs/TestHFlush.java | 132 +-
.../hadoop/hdfs/TestKeyProviderCache.java | 124 ++
.../apache/hadoop/hdfs/TestLeaseRecovery.java | 6 +-
.../org/apache/hadoop/hdfs/TestPipelines.java | 10 +-
.../apache/hadoop/hdfs/TestRenameWhileOpen.java | 9 +-
.../hadoop/hdfs/TestReservedRawPaths.java | 4 +-
.../org/apache/hadoop/hdfs/TestSafeMode.java | 17 +-
.../hdfs/server/balancer/TestBalancer.java | 6 +-
.../TestBalancerWithMultipleNameNodes.java | 8 +-
.../server/blockmanagement/TestBlockInfo.java | 14 +-
.../TestBlockInfoUnderConstruction.java | 4 +-
.../blockmanagement/TestBlockManager.java | 20 +-
.../blockmanagement/TestDatanodeDescriptor.java | 4 +-
.../blockmanagement/TestDatanodeManager.java | 84 ++
.../blockmanagement/TestHeartbeatHandling.java | 6 +-
.../TestPendingInvalidateBlock.java | 4 +-
.../blockmanagement/TestReplicationPolicy.java | 12 +-
.../server/datanode/BlockReportTestBase.java | 8 +-
.../hdfs/server/datanode/TestBlockScanner.java | 36 +-
.../server/datanode/TestCachingStrategy.java | 30 +
.../datanode/TestDataNodeHotSwapVolumes.java | 34 +
.../datanode/TestIncrementalBrVariations.java | 13 +-
.../extdataset/ExternalDatasetImpl.java | 2 +-
.../fsdataset/impl/LazyPersistTestCase.java | 8 +-
.../fsdataset/impl/TestLazyPersistFiles.java | 5 +-
.../hadoop/hdfs/server/mover/TestMover.java | 8 +-
.../hdfs/server/namenode/CreateEditsLog.java | 8 +-
.../hdfs/server/namenode/FSImageTestUtil.java | 2 +-
.../server/namenode/NNThroughputBenchmark.java | 14 +-
.../hdfs/server/namenode/TestAddBlock.java | 12 +-
.../namenode/TestBlockUnderConstruction.java | 10 +-
.../server/namenode/TestCacheDirectives.java | 2 +-
.../TestCommitBlockSynchronization.java | 16 +-
.../hdfs/server/namenode/TestDeleteRace.java | 32 +-
.../namenode/TestDiskspaceQuotaUpdate.java | 34 +-
.../hdfs/server/namenode/TestEditLog.java | 4 +-
.../hdfs/server/namenode/TestFSImage.java | 4 +-
.../namenode/TestFSImageWithSnapshot.java | 2 +-
.../hdfs/server/namenode/TestFileTruncate.java | 219 ++-
.../hadoop/hdfs/server/namenode/TestFsck.java | 4 +-
.../namenode/TestFsckWithMultipleNameNodes.java | 5 +-
.../hdfs/server/namenode/TestHDFSConcat.java | 78 +-
.../hdfs/server/namenode/TestINodeFile.java | 4 +-
.../namenode/TestListCorruptFileBlocks.java | 7 +-
.../server/namenode/TestNamenodeRetryCache.java | 16 +-
.../server/namenode/TestQuotaByStorageType.java | 524 +++++++
.../hdfs/server/namenode/ha/TestDNFencing.java | 11 +-
.../hdfs/server/namenode/ha/TestHAAppend.java | 51 +-
.../hdfs/server/namenode/ha/TestHASafeMode.java | 5 +-
.../namenode/ha/TestPipelinesFailover.java | 15 +-
.../namenode/ha/TestRetryCacheWithHA.java | 16 +-
.../namenode/ha/TestStandbyBlockManagement.java | 20 +-
.../server/namenode/ha/TestStandbyIsHot.java | 7 +-
.../namenode/snapshot/SnapshotTestHelper.java | 10 +-
.../namenode/snapshot/TestAclWithSnapshot.java | 3 -
.../namenode/snapshot/TestNestedSnapshots.java | 41 +-
.../snapshot/TestRenameWithSnapshots.java | 107 +-
.../snapshot/TestSnapshotBlocksMap.java | 24 +-
.../namenode/snapshot/TestSnapshotDeletion.java | 178 +--
.../snapshot/TestXAttrWithSnapshot.java | 71 -
.../web/resources/TestWebHdfsDataLocality.java | 8 +-
.../TestOfflineImageViewer.java | 54 +-
.../hdfs/web/TestFSMainOperationsWebHdfs.java | 29 +
.../web/TestWebHdfsWithMultipleNameNodes.java | 13 +-
.../apache/hadoop/tracing/TestTraceAdmin.java | 4 +-
.../org/apache/hadoop/tracing/TestTracing.java | 16 +-
.../TestTracingShortCircuitLocalRead.java | 8 +-
....apache.hadoop.crypto.key.KeyProviderFactory | 16 +
.../hadoop-hdfs/src/test/resources/editsStored | Bin 5586 -> 5850 bytes
.../src/test/resources/editsStored.xml | 442 +++---
.../src/test/resources/testHDFSConf.xml | 246 +++-
hadoop-mapreduce-project/CHANGES.txt | 49 +
hadoop-mapreduce-project/bin/mapred-config.sh | 30 +-
hadoop-mapreduce-project/conf/mapred-env.sh | 35 +-
.../v2/app/rm/RMContainerAllocator.java | 5 +-
.../v2/app/speculate/DefaultSpeculator.java | 62 +-
.../mapreduce/v2/app/webapp/AppController.java | 1 -
.../mapreduce/v2/app/webapp/NavBlock.java | 2 +-
.../mapreduce/v2/app/TestRuntimeEstimators.java | 16 +
.../v2/app/rm/TestRMContainerAllocator.java | 93 ++
.../v2/api/protocolrecords/package-info.java | 2 -
.../apache/hadoop/mapred/jobcontrol/Job.java | 4 +-
.../mapred/lib/CombineFileInputFormat.java | 4 +-
.../apache/hadoop/mapred/lib/InputSampler.java | 2 +-
.../hadoop/mapred/lib/db/DBInputFormat.java | 2 +-
.../java/org/apache/hadoop/mapreduce/Job.java | 4 +-
.../apache/hadoop/mapreduce/MRJobConfig.java | 29 +
.../hadoop/mapreduce/counters/package-info.java | 2 -
.../lib/aggregate/ValueAggregatorJob.java | 2 +-
.../hadoop/mapreduce/lib/db/DBInputFormat.java | 31 +-
.../lib/db/DataDrivenDBInputFormat.java | 5 +-
.../lib/db/OracleDataDrivenDBInputFormat.java | 2 +-
.../lib/input/DelegatingInputFormat.java | 2 +-
.../mapreduce/lib/jobcontrol/ControlledJob.java | 2 +-
.../hadoop/mapreduce/lib/join/Parser.java | 2 +-
.../mapreduce/lib/output/MultipleOutputs.java | 2 +-
.../mapreduce/lib/partition/InputSampler.java | 2 +-
.../lib/partition/TotalOrderPartitioner.java | 2 +-
.../hadoop/mapreduce/util/ConfigUtil.java | 4 +-
.../src/main/resources/mapred-default.xml | 41 +-
.../src/site/apt/HadoopStreaming.apt.vm | 792 -----------
.../src/site/markdown/DistCp.md.vm | 512 -------
.../src/site/markdown/HadoopArchives.md.vm | 162 ---
.../mapreduce/v2/hs/HistoryFileManager.java | 8 +
.../hadoop/mapreduce/v2/hs/package-info.java | 2 -
.../mapreduce/v2/hs/webapp/HsNavBlock.java | 2 +-
.../resources/job_1329348432655_0001_conf.xml | 2 -
.../org/apache/hadoop/RandomTextWriterJob.java | 2 +-
.../java/org/apache/hadoop/fs/TestDFSIO.java | 91 +-
.../hadoop/mapred/TestMiniMRChildTask.java | 4 +-
.../hadoop/mapred/TestMiniMRClientCluster.java | 2 +-
.../apache/hadoop/mapreduce/LargeSorter.java | 3 +-
.../hadoop/mapreduce/RandomTextWriter.java | 2 +-
.../apache/hadoop/mapreduce/RandomWriter.java | 2 +-
.../apache/hadoop/mapreduce/TestLargeSort.java | 1 +
.../apache/hadoop/mapreduce/v2/TestMRJobs.java | 2 +-
.../test/java/testjar/UserNamePermission.java | 2 +-
.../hadoop/examples/BaileyBorweinPlouffe.java | 2 +-
.../java/org/apache/hadoop/examples/Grep.java | 6 +-
.../java/org/apache/hadoop/examples/Join.java | 2 +-
.../hadoop/examples/MultiFileWordCount.java | 2 +-
.../apache/hadoop/examples/QuasiMonteCarlo.java | 2 +-
.../hadoop/examples/RandomTextWriter.java | 2 +-
.../apache/hadoop/examples/RandomWriter.java | 2 +-
.../apache/hadoop/examples/SecondarySort.java | 2 +-
.../java/org/apache/hadoop/examples/Sort.java | 2 +-
.../org/apache/hadoop/examples/WordCount.java | 2 +-
.../org/apache/hadoop/examples/WordMean.java | 3 +-
.../org/apache/hadoop/examples/WordMedian.java | 3 +-
.../hadoop/examples/WordStandardDeviation.java | 3 +-
.../examples/dancing/DistributedPentomino.java | 2 +-
.../org/apache/hadoop/examples/pi/DistSum.java | 3 +-
.../shellprofile.d/mapreduce | 41 +
hadoop-project/pom.xml | 62 +-
hadoop-project/src/site/site.xml | 17 +-
.../org/apache/hadoop/tools/HadoopArchives.java | 16 +-
.../src/site/markdown/HadoopArchives.md.vm | 162 +++
.../src/site/resources/css/site.css | 30 +
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 30 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 2 +-
.../org/apache/hadoop/fs/s3a/S3ATestUtils.java | 3 +
hadoop-tools/hadoop-azure/pom.xml | 13 +
.../hadoop-azure/src/config/checkstyle.xml | 1 -
.../fs/azure/AzureNativeFileSystemStore.java | 5 +-
.../hadoop/fs/azure/NativeAzureFileSystem.java | 32 +-
.../fs/azure/AzureBlobStorageTestAccount.java | 2 +-
.../src/site/markdown/DistCp.md.vm | 512 +++++++
.../src/site/resources/css/site.css | 30 +
.../hadoop/mapred/gridmix/GridmixJob.java | 6 +-
.../src/site/markdown/GridMix.md.vm | 107 +-
.../gridmix/TestCompressionEmulationUtils.java | 2 +-
.../hadoop/tools/rumen/Hadoop20JHParser.java | 5 -
.../hadoop/tools/rumen/HadoopLogsAnalyzer.java | 2 +-
.../MapAttempt20LineHistoryEventEmitter.java | 2 +-
.../hadoop/tools/rumen/ParsedConfigFile.java | 11 +-
.../hadoop/tools/rumen/RandomSeedGenerator.java | 4 +-
.../ReduceAttempt20LineHistoryEventEmitter.java | 2 +-
.../hadoop-rumen/src/site/markdown/Rumen.md.vm | 135 +-
hadoop-tools/hadoop-sls/README | 24 +-
.../hadoop-sls/src/main/bin/rumen2sls.sh | 130 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 144 +-
.../src/main/data/2jobs2min-rumen-jh.json | 6 -
.../src/site/apt/SchedulerLoadSimulator.apt.vm | 5 +-
.../src/site/apt/HadoopStreaming.apt.vm | 792 +++++++++++
.../src/site/resources/css/site.css | 30 +
.../mapreduce/TestStreamXmlRecordReader.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 140 +-
.../hadoop-yarn/bin/start-yarn.sh | 6 +-
.../hadoop-yarn/bin/stop-yarn.sh | 6 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 88 +-
.../hadoop-yarn/bin/yarn-config.sh | 52 +-
.../hadoop-yarn/conf/yarn-env.sh | 56 +-
.../yarn/api/records/ResourceRequest.java | 21 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 20 +-
.../hadoop/yarn/client/api/AMRMClient.java | 3 +-
.../yarn/client/api/async/AMRMClientAsync.java | 11 +
.../api/async/impl/AMRMClientAsyncImpl.java | 13 +
.../yarn/client/api/impl/AMRMClientImpl.java | 45 +-
.../yarn/client/api/impl/YarnClientImpl.java | 105 +-
.../hadoop/yarn/client/cli/RMAdminCLI.java | 37 +-
.../yarn/client/api/impl/TestAMRMClient.java | 72 +-
.../yarn/client/api/impl/TestYarnClient.java | 161 ++-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 48 +-
.../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 +
.../client/api/impl/TimelineClientImpl.java | 37 +-
.../nodelabels/CommonNodeLabelsManager.java | 143 +-
.../hadoop/yarn/nodelabels/NodeLabel.java | 17 +
.../apache/hadoop/yarn/security/AccessType.java | 33 +
.../hadoop/yarn/security/AdminACLsManager.java | 31 -
.../yarn/security/ConfiguredYarnAuthorizer.java | 97 ++
.../hadoop/yarn/security/PrivilegedEntity.java | 83 ++
.../security/YarnAuthorizationProvider.java | 112 ++
.../yarn/util/AbstractLivelinessMonitor.java | 8 +
.../apache/hadoop/yarn/util/ConverterUtils.java | 2 +-
.../org/apache/hadoop/yarn/util/FSDownload.java | 5 +-
.../org/apache/hadoop/yarn/webapp/WebApps.java | 7 +-
.../hadoop/yarn/webapp/view/NavBlock.java | 2 +-
.../src/main/resources/yarn-default.xml | 131 +-
.../client/api/impl/TestTimelineClient.java | 14 +-
.../yarn/nodelabels/NodeLabelTestBase.java | 33 +
.../nodelabels/TestCommonNodeLabelsManager.java | 207 ++-
.../TestFileSystemNodeLabelsStore.java | 22 +-
.../apache/hadoop/yarn/util/TestFSDownload.java | 33 +-
.../apache/hadoop/registry/cli/RegistryCli.java | 425 +++---
.../hadoop/registry/cli/TestRegistryCli.java | 197 +++
.../ApplicationHistoryClientService.java | 27 +-
...pplicationHistoryManagerOnTimelineStore.java | 12 +-
.../ApplicationHistoryServer.java | 2 +
.../timeline/webapp/CrossOriginFilter.java | 18 +
.../TestApplicationHistoryClientService.java | 4 +
...pplicationHistoryManagerOnTimelineStore.java | 9 +
.../TestApplicationHistoryServer.java | 26 +
.../metrics/ApplicationMetricsConstants.java | 6 +
.../server/nodemanager/DeletionService.java | 40 +-
.../container/ContainerImpl.java | 3 +-
.../localizer/ResourceLocalizationService.java | 29 +-
.../monitor/ContainerMetrics.java | 42 +-
.../monitor/ContainerStartMonitoringEvent.java | 7 +-
.../monitor/ContainersMonitorImpl.java | 30 +-
.../util/CgroupsLCEResourcesHandler.java | 73 +-
.../server/nodemanager/webapp/NavBlock.java | 2 +-
.../impl/container-executor.c | 28 +-
.../test/test-container-executor.c | 21 +
.../TestLocalResourcesTrackerImpl.java | 5 +-
.../TestResourceLocalizationService.java | 105 +-
.../monitor/TestContainerMetrics.java | 40 +-
.../metrics/TestNodeManagerMetrics.java | 6 +
.../util/TestCgroupsLCEResourcesHandler.java | 80 +-
.../server/resourcemanager/AdminService.java | 34 +-
.../ApplicationMasterService.java | 7 +-
.../server/resourcemanager/ClientRMService.java | 19 +-
.../server/resourcemanager/RMAppManager.java | 3 +-
.../server/resourcemanager/RMServerUtils.java | 25 +-
.../server/resourcemanager/ResourceManager.java | 2 +
.../metrics/ApplicationFinishedEvent.java | 9 +-
.../metrics/SystemMetricsPublisher.java | 10 +-
.../ProportionalCapacityPreemptionPolicy.java | 20 +-
.../nodelabels/RMNodeLabelsManager.java | 23 +-
.../recovery/ZKRMStateStore.java | 17 +-
.../rmapp/attempt/AMLivelinessMonitor.java | 6 +
.../rmapp/attempt/RMAppAttemptImpl.java | 18 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 5 +-
.../scheduler/AbstractYarnScheduler.java | 70 +-
.../scheduler/ResourceUsage.java | 80 +-
.../scheduler/SchedulerUtils.java | 35 +-
.../scheduler/YarnScheduler.java | 11 +-
.../scheduler/capacity/AbstractCSQueue.java | 145 +-
.../scheduler/capacity/CSQueue.java | 6 +
.../scheduler/capacity/CapacityScheduler.java | 37 +-
.../CapacitySchedulerConfiguration.java | 90 +-
.../capacity/CapacitySchedulerContext.java | 2 +
.../scheduler/capacity/LeafQueue.java | 174 ++-
.../scheduler/capacity/ParentQueue.java | 72 +-
.../scheduler/capacity/QueueCapacities.java | 191 +++
.../scheduler/fair/FSAppAttempt.java | 19 +-
.../scheduler/fair/FairScheduler.java | 13 +-
.../security/AMRMTokenSecretManager.java | 2 +-
.../security/NMTokenSecretManagerInRM.java | 2 +-
.../security/RMContainerTokenSecretManager.java | 4 +-
.../webapp/CapacitySchedulerPage.java | 3 +-
.../server/resourcemanager/webapp/NavBlock.java | 2 +-
.../resourcemanager/webapp/RMWebServices.java | 10 +-
.../dao/CapacitySchedulerLeafQueueInfo.java | 6 +
.../yarn/server/resourcemanager/MockAM.java | 17 +-
.../server/resourcemanager/TestAppManager.java | 3 +-
.../server/resourcemanager/TestRMRestart.java | 6 +-
.../TestWorkPreservingRMRestart.java | 2 +-
.../metrics/TestSystemMetricsPublisher.java | 17 +-
...estProportionalCapacityPreemptionPolicy.java | 68 +-
.../nodelabels/TestRMNodeLabelsManager.java | 77 +-
.../recovery/TestZKRMStateStore.java | 4 +-
.../rmapp/attempt/TestAMLivelinessMonitor.java | 81 ++
.../attempt/TestRMAppAttemptTransitions.java | 9 +-
.../scheduler/TestAbstractYarnScheduler.java | 62 +
.../scheduler/TestSchedulerUtils.java | 54 +-
.../scheduler/capacity/TestCSQueueUtils.java | 24 +-
.../capacity/TestCapacityScheduler.java | 355 ++++-
.../capacity/TestContainerAllocation.java | 32 +-
.../scheduler/capacity/TestParentQueue.java | 5 +-
.../scheduler/capacity/TestQueueCapacities.java | 127 ++
.../fair/TestContinuousScheduling.java | 114 ++
.../scheduler/fair/TestFairScheduler.java | 112 +-
.../webapp/TestRMWebServices.java | 55 +
.../webapp/TestRMWebServicesCapacitySched.java | 2 +-
.../webapp/TestRMWebServicesNodeLabels.java | 17 +-
.../SCMAdminProtocolService.java | 10 +-
.../src/site/apt/CapacityScheduler.apt.vm | 12 +
.../src/site/markdown/registry/index.md | 1 +
.../markdown/registry/registry-configuration.md | 453 ++++++
.../site/markdown/registry/registry-security.md | 4 +-
.../registry/using-the-yarn-service-registry.md | 235 +++-
.../src/site/markdown/registry/yarn-registry.md | 73 +-
.../hadoop-yarn/shellprofile.d/yarn | 62 +
pom.xml | 40 +-
581 files changed, 24402 insertions(+), 13496 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2786151/hadoop-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e2786151/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------