You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by aw...@apache.org on 2016/09/08 14:57:26 UTC
[01/48] hadoop git commit: HDFS-10729. Improve log message for edit
loading failures caused by FS limit checks. Contributed by Wei-Chiu Chuang.
[Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/HADOOP-13341 536ad247b -> 5b7a3df75 (forced update)
HDFS-10729. Improve log message for edit loading failures caused by FS limit checks. Contributed by Wei-Chiu Chuang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/01721dd8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/01721dd8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/01721dd8
Branch: refs/heads/HADOOP-13341
Commit: 01721dd88ee532d20eda841254437da4dfd69db5
Parents: 20ae1fa
Author: Kihwal Lee <ki...@apache.org>
Authored: Wed Aug 31 14:02:37 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Wed Aug 31 14:02:37 2016 -0500
----------------------------------------------------------------------
.../hadoop/hdfs/server/namenode/FSDirWriteFileOp.java | 12 ++++++++----
1 file changed, 8 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/01721dd8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index cb639b1..077f04f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -40,6 +40,7 @@ import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
+import org.apache.hadoop.hdfs.protocol.FSLimitException;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
@@ -506,10 +507,13 @@ class FSDirWriteFileOp {
return newNode;
}
} catch (IOException e) {
- if(NameNode.stateChangeLog.isDebugEnabled()) {
- NameNode.stateChangeLog.debug(
- "DIR* FSDirectory.unprotectedAddFile: exception when add "
- + existing.getPath() + " to the file system", e);
+ NameNode.stateChangeLog.warn(
+ "DIR* FSDirectory.unprotectedAddFile: exception when add " + existing
+ .getPath() + " to the file system", e);
+ if (e instanceof FSLimitException.MaxDirectoryItemsExceededException) {
+ NameNode.stateChangeLog.warn("Please increase "
+ + "dfs.namenode.fs-limits.max-directory-items and make it "
+ + "consistent across all NameNodes.");
}
}
return null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/48] hadoop git commit: HADOOP-13562. Change
hadoop_subcommand_opts to use only uppercase
Posted by aw...@apache.org.
HADOOP-13562. Change hadoop_subcommand_opts to use only uppercase
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d03c6f83
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d03c6f83
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d03c6f83
Branch: refs/heads/HADOOP-13341
Commit: d03c6f835dad5963e90d45e047077a7c270c1d5e
Parents: 6bc45fa
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 08:52:41 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../src/main/bin/hadoop-functions.sh | 31 +++++++-------------
.../test/scripts/hadoop_subcommand_opts.bats | 18 ++++--------
2 files changed, 16 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d03c6f83/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index dd82347..695fd41 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -2007,8 +2007,8 @@ function hadoop_subcommand_opts
{
declare program=$1
declare command=$2
- declare var
declare uvar
+ declare depvar
declare uprogram
declare ucommand
@@ -2029,34 +2029,25 @@ function hadoop_subcommand_opts
ucommand=${command^^}
fi
- # HDFS_namenode_OPTS
- # HADOOP_distcp_OPTS
- # MAPRED_distcp_OPTS
- # YARN_sharedcachemanger_OPTS
- # ...
- var="${uprogram}_${command}_OPTS"
+ uvar="${uprogram}_${ucommand}_OPTS"
# Let's handle all of the deprecation cases early
- # HADOOP_NAMENODE_OPTS -> HDFS_namenode_OPTS
- # YARN_RESOURCEMANAGER_OPTS -> YARN_resourcemanager_OPTS
+ # HADOOP_NAMENODE_OPTS -> HDFS_NAMENODE_OPTS
- uvar="${uprogram}_${ucommand}_OPTS"
- if [[ -n ${!uvar} ]]; then
- hadoop_deprecate_envvar "${uvar}" "${var}"
- fi
+ depvar="HADOOP_${ucommand}_OPTS"
- uvar="HADOOP_${ucommand}_OPTS"
- if [[ -n ${!uvar} ]]; then
- hadoop_deprecate_envvar "${uvar}" "${var}"
+ if [[ "${depvar}" != "${uvar}" ]]; then
+ if [[ -n "${!depvar}" ]]; then
+ hadoop_deprecate_envvar "${depvar}" "${uvar}"
+ fi
fi
- if [[ -n ${!var} ]]; then
- hadoop_debug "Appending ${!var} onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${!var}"
+ if [[ -n ${!uvar} ]]; then
+ hadoop_debug "Appending ${!uvar} onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
return 0
fi
}
-
## @description Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS.
## @description This *does not* handle the pre-3.x deprecated cases
## @audience public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d03c6f83/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
index 1fbf343..0010a58 100644
--- a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
@@ -29,7 +29,7 @@ load hadoop-functions_test_helper
@test "hadoop_subcommand_opts (hadoop simple exist)" {
HADOOP_OPTS="x"
- HADOOP_test_OPTS="y"
+ HADOOP_TEST_OPTS="y"
hadoop_subcommand_opts hadoop test
echo "${HADOOP_OPTS}"
[ "${HADOOP_OPTS}" = "x y" ]
@@ -37,7 +37,7 @@ load hadoop-functions_test_helper
@test "hadoop_subcommand_opts (hadoop complex exist)" {
HADOOP_OPTS="x"
- HADOOP_test_OPTS="y z"
+ HADOOP_TEST_OPTS="y z"
hadoop_subcommand_opts hadoop test
echo "${HADOOP_OPTS}"
[ "${HADOOP_OPTS}" = "x y z" ]
@@ -45,7 +45,7 @@ load hadoop-functions_test_helper
@test "hadoop_subcommand_opts (hdfs simple exist)" {
HADOOP_OPTS="x"
- HDFS_test_OPTS="y"
+ HDFS_TEST_OPTS="y"
hadoop_subcommand_opts hdfs test
echo "${HADOOP_OPTS}"
[ "${HADOOP_OPTS}" = "x y" ]
@@ -53,24 +53,16 @@ load hadoop-functions_test_helper
@test "hadoop_subcommand_opts (yarn simple exist)" {
HADOOP_OPTS="x"
- YARN_test_OPTS="y"
+ YARN_TEST_OPTS="y"
hadoop_subcommand_opts yarn test
echo "${HADOOP_OPTS}"
[ "${HADOOP_OPTS}" = "x y" ]
}
-@test "hadoop_subcommand_opts (deprecation case #1)" {
+@test "hadoop_subcommand_opts (deprecation case)" {
HADOOP_OPTS="x"
HADOOP_NAMENODE_OPTS="y"
hadoop_subcommand_opts hdfs namenode
echo "${HADOOP_OPTS}"
[ "${HADOOP_OPTS}" = "x y" ]
}
-
-@test "hadoop_subcommand_opts (deprecation case #2)" {
- HADOOP_OPTS="x"
- YARN_RESOURCEMANAGER_OPTS="y"
- hadoop_subcommand_opts yarn resourcemanager
- echo "${HADOOP_OPTS}"
- [ "${HADOOP_OPTS}" = "x y" ]
-}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/48] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f388a209/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
new file mode 100644
index 0000000..edf47ca
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md
@@ -0,0 +1,1854 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" 3.0.0-alpha1 Release Notes
+
+These release notes cover new developer and user-facing incompatibilities, important issues, features, and major improvements.
+
+
+---
+
+* [HDFS-46](https://issues.apache.org/jira/browse/HDFS-46) | *Major* | **The namespace quota of root directory should not be Integer.MAX\_VALUE**
+
+Change default namespace quota of root directory from Integer.MAX\_VALUE to Long.MAX\_VALUE.
+
+
+---
+
+* [HADOOP-8124](https://issues.apache.org/jira/browse/HADOOP-8124) | *Major* | **Remove the deprecated Syncable.sync() method**
+
+Remove the deprecated FSDataOutputStream constructor, FSDataOutputStream.sync() and Syncable.sync().
+
+
+---
+
+* [HDFS-3034](https://issues.apache.org/jira/browse/HDFS-3034) | *Major* | **Remove the deprecated Syncable.sync() method**
+
+Remove the deprecated DFSOutputStream.sync() method.
+
+
+---
+
+* [HADOOP-7659](https://issues.apache.org/jira/browse/HADOOP-7659) | *Minor* | **fs -getmerge isn't guaranteed to work well over non-HDFS filesystems**
+
+Documented that the "fs -getmerge" shell command may not work properly over non HDFS-filesystem implementations due to platform-varying file list ordering.
+
+
+---
+
+* [HADOOP-8776](https://issues.apache.org/jira/browse/HADOOP-8776) | *Minor* | **Provide an option in test-patch that can enable / disable compiling native code**
+
+test-patch.sh adds a new option "--build-native". When set to false native
+components are not built. When set to true native components are built. The
+default value is true.
+
+
+---
+
+* [HDFS-5079](https://issues.apache.org/jira/browse/HDFS-5079) | *Major* | **Cleaning up NNHAStatusHeartbeat.State DatanodeProtocolProtos.**
+
+This change affects wire-compatibility of the NameNode/DataNode heartbeat protocol.
+
+
+---
+
+* [HDFS-5570](https://issues.apache.org/jira/browse/HDFS-5570) | *Major* | **Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs**
+
+Support for hftp and hsftp has been removed. They have superseded by webhdfs and swebhdfs.
+
+
+---
+
+* [HADOOP-10485](https://issues.apache.org/jira/browse/HADOOP-10485) | *Major* | **Remove dead classes in hadoop-streaming**
+
+Deprecated and unused classes in the org.apache.hadoop.record package have been removed from hadoop-streaming.
+
+
+---
+
+* [HDFS-6246](https://issues.apache.org/jira/browse/HDFS-6246) | *Minor* | **Remove 'dfs.support.append' flag from trunk code**
+
+Appends in HDFS can no longer be disabled.
+
+
+---
+
+* [HADOOP-10474](https://issues.apache.org/jira/browse/HADOOP-10474) | *Major* | **Move o.a.h.record to hadoop-streaming**
+
+The classes in org.apache.hadoop.record are moved from hadoop-common to a new hadoop-streaming artifact within the hadoop-tools module.
+
+
+---
+
+* [HDFS-2538](https://issues.apache.org/jira/browse/HDFS-2538) | *Minor* | **option to disable fsck dots**
+
+fsck does not print out dots for progress reporting by default. To print out dots, you should specify '-showprogress' option.
+
+
+---
+
+* [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902) | *Major* | **Shell script rewrite**
+
+<!-- markdown -->
+The Hadoop shell scripts have been rewritten to fix many long standing bugs and include some new features. While an eye has been kept towards compatibility, some changes may break existing installations.
+
+INCOMPATIBLE CHANGES:
+
+* The pid and out files for secure daemons have been renamed to include the appropriate ${HADOOP\_IDENT\_STR}. This should allow, with proper configurations in place, for multiple versions of the same secure daemon to run on a host. Additionally, pid files are now created when daemons are run in interactive mode. This will also prevent the accidental starting of two daemons with the same configuration prior to launching java (i.e., "fast fail" without having to wait for socket opening).
+* All Hadoop shell script subsystems now execute hadoop-env.sh, which allows for all of the environment variables to be in one location. This was not the case previously.
+* The default content of *-env.sh has been significantly altered, with the majority of defaults moved into more protected areas inside the code. Additionally, these files do not auto-append anymore; setting a variable on the command line prior to calling a shell command must contain the entire content, not just any extra settings. This brings Hadoop more in-line with the vast majority of other software packages.
+* All HDFS\_\*, YARN\_\*, and MAPRED\_\* environment variables act as overrides to their equivalent HADOOP\_\* environment variables when 'hdfs', 'yarn', 'mapred', and related commands are executed. Previously, these were separated out which meant a significant amount of duplication of common settings.
+* hdfs-config.sh and hdfs-config.cmd were inadvertently duplicated into libexec and sbin. The sbin versions have been removed.
+* The log4j settings forcibly set by some *-daemon.sh commands have been removed. These settings are now configurable in the \*-env.sh files via \*\_OPT.
+* Support for various undocumented YARN log4j.properties files has been removed.
+* Support for ${HADOOP\_MASTER} and the related rsync code have been removed.
+* The undocumented and unused yarn.id.str Java property has been removed.
+* The unused yarn.policy.file Java property has been removed.
+* We now require bash v3 (released July 27, 2004) or better in order to take advantage of better regex handling and ${BASH\_SOURCE}. POSIX sh will not work.
+* Support for --script has been removed. We now use ${HADOOP\_\*\_PATH} or ${HADOOP\_PREFIX} to find the necessary binaries. (See other note regarding ${HADOOP\_PREFIX} auto discovery.)
+* Non-existent classpaths, ld.so library paths, JNI library paths, etc, will be ignored and stripped from their respective environment settings.
+
+NEW FEATURES:
+
+* Daemonization has been moved from *-daemon.sh to the bin commands via the --daemon option. Simply use --daemon start to start a daemon, --daemon stop to stop a daemon, and --daemon status to set $? to the daemon's status. The return code for status is LSB-compatible. For example, 'hdfs --daemon start namenode'.
+* It is now possible to override some of the shell code capabilities to provide site specific functionality without replacing the shipped versions. Replacement functions should go into the new hadoop-user-functions.sh file.
+* A new option called --buildpaths will attempt to add developer build directories to the classpath to allow for in source tree testing.
+* Operations which trigger ssh connections can now use pdsh if installed. ${HADOOP\_SSH\_OPTS} still gets applied.
+* Added distch and jnipath subcommands to the hadoop command.
+* Shell scripts now support a --debug option which will report basic information on the construction of various environment variables, java options, classpath, etc. to help in configuration debugging.
+
+BUG FIXES:
+
+* ${HADOOP\_CONF\_DIR} is now properly honored everywhere, without requiring symlinking and other such tricks.
+* ${HADOOP\_CONF\_DIR}/hadoop-layout.sh is now documented with a provided hadoop-layout.sh.example file.
+* Shell commands should now work properly when called as a relative path, without ${HADOOP\_PREFIX} being defined, and as the target of bash -x for debugging. If ${HADOOP\_PREFIX} is not set, it will be automatically determined based upon the current location of the shell library. Note that other parts of the extended Hadoop ecosystem may still require this environment variable to be configured.
+* Operations which trigger ssh will now limit the number of connections to run in parallel to ${HADOOP\_SSH\_PARALLEL} to prevent memory and network exhaustion. By default, this is set to 10.
+* ${HADOOP\_CLIENT\_OPTS} support has been added to a few more commands.
+* Some subcommands were not listed in the usage.
+* Various options on hadoop command lines were supported inconsistently. These have been unified into hadoop-config.sh. --config is still required to be first, however.
+* ulimit logging for secure daemons no longer assumes /bin/bash but does assume bash is on the command line path.
+* Removed references to some Yahoo! specific paths.
+* Removed unused slaves.sh from YARN build tree.
+* Many exit states have been changed to reflect reality.
+* Shell level errors now go to STDERR. Before, many of them went incorrectly to STDOUT.
+* CDPATH with a period (.) should no longer break the scripts.
+* The scripts no longer try to chown directories.
+* If ${JAVA\_HOME} is not set on OS X, it now properly detects it instead of throwing an error.
+
+IMPROVEMENTS:
+
+* The *.out files are now appended instead of overwritten to allow for external log rotation.
+* The style and layout of the scripts is much more consistent across subprojects.
+* More of the shell code is now commented.
+* Significant amounts of redundant code have been moved into a new file called hadoop-functions.sh.
+* The various *-env.sh have been massively changed to include documentation and examples on what can be set, ramifications of setting, etc. for all variables that are expected to be set by a user.
+* There is now some trivial de-duplication and sanitization of the classpath and JVM options. This allows, amongst other things, for custom settings in \*\_OPTS for Hadoop daemons to override defaults and other generic settings (i.e., ${HADOOP\_OPTS}). This is particularly relevant for Xmx settings, as one can now set them in _OPTS and ignore the heap specific options for daemons which force the size in megabytes.
+* Subcommands have been alphabetized in both usage and in the code.
+* All/most of the functionality provided by the sbin/* commands has been moved to either their bin/ equivalents or made into functions. The rewritten versions of these commands are now wrappers to maintain backward compatibility.
+* Usage information is given with the following options/subcommands for all scripts using the common framework: --? -? ? --help -help -h help
+* Several generic environment variables have been added to provide a common configuration for pids, logs, and their security equivalents. The older versions still act as overrides to these generic versions.
+* Groundwork has been laid to allow for custom secure daemon setup using something other than jsvc (e.g., pfexec on Solaris).
+* Scripts now test and report better error messages for various states of the log and pid dirs on daemon startup. Before, unprotected shell errors would be displayed to the user.
+
+
+---
+
+* [HADOOP-11041](https://issues.apache.org/jira/browse/HADOOP-11041) | *Minor* | **VersionInfo output specifies subversion**
+
+This changes the output of the 'hadoop version' command to generically say 'Source code repository' rather than specify which type of repo.
+
+
+---
+
+* [MAPREDUCE-5972](https://issues.apache.org/jira/browse/MAPREDUCE-5972) | *Trivial* | **Fix typo 'programatically' in job.xml (and a few other places)**
+
+Fix a typo. If a configuration is set through program, the source of the configuration is set to 'programmatically' instead of 'programatically' now.
+
+
+---
+
+* [MAPREDUCE-2841](https://issues.apache.org/jira/browse/MAPREDUCE-2841) | *Major* | **Task level native optimization**
+
+Adds a native implementation of the map output collector. The native library will build automatically with -Pnative. Users may choose the new collector on a job-by-job basis by setting mapreduce.job.map.output.collector.class=org.apache.hadoop.mapred.
+nativetask.NativeMapOutputCollectorDelegator in their job configuration. For shuffle-intensive jobs this may provide speed-ups of 30% or more.
+
+
+---
+
+* [HADOOP-11356](https://issues.apache.org/jira/browse/HADOOP-11356) | *Major* | **Removed deprecated o.a.h.fs.permission.AccessControlException**
+
+org.apache.hadoop.fs.permission.AccessControlException was deprecated in the last major release, and has been removed in favor of org.apache.hadoop.security.AccessControlException
+
+
+---
+
+* [HADOOP-10950](https://issues.apache.org/jira/browse/HADOOP-10950) | *Major* | **rework heap management vars**
+
+<!-- markdown -->
+
+* HADOOP\_HEAPSIZE variable has been deprecated (It will still be honored if set, but expect it to go away in the future). In its place, HADOOP\_HEAPSIZE\_MAX and HADOOP\_HEAPSIZE\_MIN have been introduced to set Xmx and Xms, respectively.
+
+* The internal variable JAVA\_HEAP\_MAX has been removed.
+
+* Default heap sizes have been removed. This will allow for the JVM to use auto-tuning based upon the memory size of the host. To re-enable the old default, configure HADOOP\_HEAPSIZE_MAX="1g" in hadoop-env.sh.
+
+* All global and daemon-specific heap size variables now support units. If the variable is only a number, the size is assumed to be in megabytes.
+
+
+---
+
+* [HADOOP-11353](https://issues.apache.org/jira/browse/HADOOP-11353) | *Major* | **Add support for .hadooprc**
+
+.hadooprc allows users a convenient way to set and/or override the shell level settings.
+
+
+---
+
+* [MAPREDUCE-5785](https://issues.apache.org/jira/browse/MAPREDUCE-5785) | *Major* | **Derive heap size or mapreduce.\*.memory.mb automatically**
+
+The memory values for mapreduce.map/reduce.memory.mb keys, if left to their default values of -1, will now be automatically inferred from the heap size value system property (-Xmx) specified for mapreduce.map/reduce.java.opts keys.
+
+The converse is also done, i.e. if mapreduce.map/reduce.memory.mb values are specified, but no -Xmx is supplied for mapreduce.map/reduce.java.opts keys, then the -Xmx value will be derived from the former's value.
+
+If neither is specified, then a default value of 1024 MB gets used.
+
+For both these conversions, a scaling factor specified by property mapreduce.job.heap.memory-mb.ratio is used, to account for overheads between heap usage vs. actual physical memory usage.
+
+Existing configs or job code that already specify both the set of properties explicitly would not be affected by this inferring change.
+
+
+---
+
+* [YARN-2428](https://issues.apache.org/jira/browse/YARN-2428) | *Trivial* | **LCE default banned user list should have yarn**
+
+The user 'yarn' is no longer allowed to run tasks for security reasons.
+
+
+---
+
+* [HADOOP-11460](https://issues.apache.org/jira/browse/HADOOP-11460) | *Major* | **Deprecate shell vars**
+
+<!-- markdown -->
+
+| Old | New |
+|:---- |:---- |
+| HADOOP\_HDFS\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| HADOOP\_HDFS\_LOGFILE | HADOOP\_LOGFILE |
+| HADOOP\_HDFS\_NICENESS | HADOOP\_NICENESS |
+| HADOOP\_HDFS\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| HADOOP\_HDFS\_PID\_DIR | HADOOP\_PID\_DIR |
+| HADOOP\_HDFS\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| HADOOP\_HDFS\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| HADOOP\_MAPRED\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| HADOOP\_MAPRED\_LOGFILE | HADOOP\_LOGFILE |
+| HADOOP\_MAPRED\_NICENESS | HADOOP\_NICENESS |
+| HADOOP\_MAPRED\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| HADOOP\_MAPRED\_PID\_DIR | HADOOP\_PID\_DIR |
+| HADOOP\_MAPRED\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| HADOOP\_MAPRED\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| YARN\_CONF\_DIR | HADOOP\_CONF\_DIR |
+| YARN\_LOG\_DIR | HADOOP\_LOG\_DIR |
+| YARN\_LOGFILE | HADOOP\_LOGFILE |
+| YARN\_NICENESS | HADOOP\_NICENESS |
+| YARN\_STOP\_TIMEOUT | HADOOP\_STOP\_TIMEOUT |
+| YARN\_PID\_DIR | HADOOP\_PID\_DIR |
+| YARN\_ROOT\_LOGGER | HADOOP\_ROOT\_LOGGER |
+| YARN\_IDENT\_STRING | HADOOP\_IDENT\_STRING |
+| YARN\_OPTS | HADOOP\_OPTS |
+| YARN\_SLAVES | HADOOP\_SLAVES |
+| YARN\_USER\_CLASSPATH | HADOOP\_CLASSPATH |
+| YARN\_USER\_CLASSPATH\_FIRST | HADOOP\_USER\_CLASSPATH\_FIRST |
+| KMS\_CONFIG | HADOOP\_CONF\_DIR |
+| KMS\_LOG | HADOOP\_LOG\_DIR |
+
+
+---
+
+* [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | *Trivial* | **dfs -count -q should label output column**
+
+Added -v option to fs -count command to display a header record in the report.
+
+
+---
+
+* [HADOOP-11485](https://issues.apache.org/jira/browse/HADOOP-11485) | *Major* | **Pluggable shell integration**
+
+Support for shell profiles has been added. They allow for easy integration of additional functionality, classpaths, and more from inside the bash scripts rather than relying upon modifying hadoop-env.sh, etc. See the Unix Shell Guide for more information.
+
+
+---
+
+* [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | *Minor* | **Shell command ls should include sort options**
+
+Options to sort output of fs -ls comment: -t (mtime), -S (size), -u (atime), -r (reverse)
+
+
+---
+
+* [HADOOP-11554](https://issues.apache.org/jira/browse/HADOOP-11554) | *Major* | **Expose HadoopKerberosName as a hadoop subcommand**
+
+The hadoop kerbname subcommand has been added to ease operational pain in determining the output of auth\_to\_local rules.
+
+
+---
+
+* [HDFS-7460](https://issues.apache.org/jira/browse/HDFS-7460) | *Major* | **Rewrite httpfs to use new shell framework**
+
+<!-- markdown -->
+This deprecates the following environment variables:
+
+| Old | New |
+|:---- |:---- |
+| HTTPFS_LOG | HADOOP_LOG_DIR|
+| HTTPFS_CONFG | HADOOP_CONF_DIR |
+
+
+---
+
+* [MAPREDUCE-5653](https://issues.apache.org/jira/browse/MAPREDUCE-5653) | *Major* | **DistCp does not honour config-overrides for mapreduce.[map,reduce].memory.mb**
+
+Prior to this change, distcp had hard-coded values for memory usage. Now distcp will honor memory settings in a way compatible with the rest of MapReduce.
+
+
+---
+
+* [HADOOP-11657](https://issues.apache.org/jira/browse/HADOOP-11657) | *Minor* | **Align the output of `hadoop fs -du` to be more Unix-like**
+
+The output of du has now been made more Unix-like, with aligned output.
+
+
+---
+
+* [HDFS-7302](https://issues.apache.org/jira/browse/HDFS-7302) | *Major* | **namenode -rollingUpgrade downgrade may finalize a rolling upgrade**
+
+Remove "downgrade" from "namenode -rollingUpgrade" startup option since it may incorrectly finalize an ongoing rolling upgrade.
+
+
+---
+
+* [HADOOP-6857](https://issues.apache.org/jira/browse/HADOOP-6857) | *Major* | **FsShell should report raw disk usage including replication factor**
+
+The output format of hadoop fs -du has been changed. It shows not only the file size but also the raw disk usage including the replication factor.
+
+
+---
+
+* [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | *Major* | **Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY**
+
+Use low latency TCP connections for hadoop IPC
+
+
+---
+
+* [HADOOP-10115](https://issues.apache.org/jira/browse/HADOOP-10115) | *Major* | **Exclude duplicate jars in hadoop package under different component's lib**
+
+Jars in the various subproject lib directories are now de-duplicated against Hadoop common. Users who interact directly with those directories must be sure to pull in common's dependencies as well.
+
+
+---
+
+* [YARN-3154](https://issues.apache.org/jira/browse/YARN-3154) | *Blocker* | **Should not upload partial logs for MR jobs or other "short-running' applications**
+
+Applications which made use of the LogAggregationContext in their application will need to revisit this code in order to make sure that their logs continue to get rolled out.
+
+
+---
+
+* [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | *Major* | **Add posixGroups support for LDAP groups mapping service**
+
+Add posixGroups support for LDAP groups mapping service. The change in LDAPGroupMapping is compatible with previous scenario. In LDAP, the group mapping between {{posixAccount}} and {{posixGroup}} is different from the general LDAPGroupMapping, one of the differences is the {{"memberUid"}} will be used to mapping {{posixAccount}} and {{posixGroup}}. The feature will handle the mapping in internal when configuration {{hadoop.security.group.mapping.ldap.search.filter.user}} is set as "posixAccount" and {{hadoop.security.group.mapping.ldap.search.filter.group}} is "posixGroup".
+
+
+---
+
+* [MAPREDUCE-4424](https://issues.apache.org/jira/browse/MAPREDUCE-4424) | *Minor* | **'mapred job -list' command should show the job name as well**
+
+Now "mapred job -list" command displays the Job Name as well.
+
+
+---
+
+* [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | *Major* | **FairScheduler handles "invalid" queue names inconsistently**
+
+FairScheduler does not allow queue names with leading or tailing spaces or empty sub-queue names anymore.
+
+
+---
+
+* [HDFS-7985](https://issues.apache.org/jira/browse/HDFS-7985) | *Major* | **WebHDFS should be always enabled**
+
+WebHDFS is mandatory and cannot be disabled.
+
+
+---
+
+* [HDFS-6353](https://issues.apache.org/jira/browse/HDFS-6353) | *Major* | **Check and make checkpoint before stopping the NameNode**
+
+Stopping the namenode on secure systems now requires the user be authenticated.
+
+
+---
+
+* [HADOOP-11553](https://issues.apache.org/jira/browse/HADOOP-11553) | *Blocker* | **Formalize the shell API**
+
+Python is now required to build the documentation.
+
+
+---
+
+* [HDFS-7501](https://issues.apache.org/jira/browse/HDFS-7501) | *Major* | **TransactionsSinceLastCheckpoint can be negative on SBNs**
+
+Fixed a bug where the StandbyNameNode's TransactionsSinceLastCheckpoint metric may slide into a negative number after every subsequent checkpoint.
+
+
+---
+
+* [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | *Minor* | **Add support for hardware crc of HDFS checksums on ARM aarch64 architecture**
+
+Add support for aarch64 CRC instructions
+
+
+---
+
+* [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | *Major* | **Rework the changelog and releasenotes**
+
+<!-- markdown -->
+* The release notes now only contains JIRA issues with incompatible changes and actual release notes. The generated format has been changed from HTML to markdown.
+
+* The changelog is now automatically generated from data stored in JIRA rather than manually maintained. The format has been changed from pure text to markdown as well as containing more of the information that was previously stored in the release notes.
+
+* In order to generate the changes file, python must be installed.
+
+* New -Preleasedocs profile added to maven in order to trigger this functionality.
+
+
+---
+
+* [YARN-3365](https://issues.apache.org/jira/browse/YARN-3365) | *Major* | **Add support for using the 'tc' tool via container-executor**
+
+Adding support for using the 'tc' tool in batch mode via container-executor. This is a prerequisite for traffic-shaping functionality that is necessary to support outbound bandwidth as a resource in YARN.
+
+
+---
+
+* [HADOOP-11781](https://issues.apache.org/jira/browse/HADOOP-11781) | *Major* | **fix race conditions and add URL support to smart-apply-patch.sh**
+
+Now auto-downloads patch from issue-id; fixed race conditions; fixed bug affecting some patches.
+
+
+---
+
+* [YARN-3443](https://issues.apache.org/jira/browse/YARN-3443) | *Major* | **Create a 'ResourceHandler' subsystem to ease addition of support for new resource types on the NM**
+
+The current cgroups implementation is closely tied to supporting CPU as a resource . This patch separates out CGroups implementation into a reusable class as well as provides a simple ResourceHandler subsystem that will enable us to add support for new resource types on the NM - e.g Network, Disk etc.
+
+
+---
+
+* [HDFS-6666](https://issues.apache.org/jira/browse/HDFS-6666) | *Minor* | **Abort NameNode and DataNode startup if security is enabled but block access token is not enabled.**
+
+NameNode and DataNode now abort during startup if attempting to run in secure mode, but block access tokens are not enabled by setting configuration property dfs.block.access.token.enable to true in hdfs-site.xml. Previously, this case logged a warning, because this would be an insecure configuration.
+
+
+---
+
+* [YARN-3021](https://issues.apache.org/jira/browse/YARN-3021) | *Major* | **YARN's delegation-token handling disallows certain trust setups to operate properly over DistCp**
+
+ResourceManager renews delegation tokens for applications. This behavior has been changed to renew tokens only if the token's renewer is a non-empty string. MapReduce jobs can instruct ResourceManager to skip renewal of tokens obtained from certain hosts by specifying the hosts with configuration mapreduce.job.hdfs-servers.token-renewal.exclude=\<host1\>,\<host2\>,..,\<hostN\>.
+
+
+---
+
+* [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | *Major* | **rewrite test-patch.sh**
+
+<!-- markdown -->
+* test-patch.sh now has new output that is different than the previous versions
+* test-patch.sh is now pluggable via the test-patch.d directory, with checkstyle and shellcheck tests included
+* JIRA comments now use much more markup to improve readability
+* test-patch.sh now supports either a file name, a URL, or a JIRA issue as input in developer mode
+* If part of the patch testing code is changed, test-patch.sh will now attempt to re-executing itself using the new version.
+* Some logic to try and reduce the amount of unnecessary tests. For example, patches that only modify markdown should not run the Java compilation tests.
+* Plugins for checkstyle, shellcheck, and whitespace now execute as necessary.
+* New test code for mvn site
+* A breakdown of the times needed to execute certain blocks as well as a total runtime is now reported to assist in fixing long running tests and optimize the entire process.
+* Several new options
+ * --resetrepo will put test-patch.sh in destructive mode, similar to a normal Jenkins run
+ * --testlist allows one to provide a comma delimited list of test subsystems to forcibly execute
+ * --modulelist to provide a comma delimited list of module tests to execute in addition to the ones that are automatically detected
+ * --offline mode to attempt to stop connecting to the Internet for certain operations
+* test-patch.sh now defaults to the POSIX equivalents on Solaris and Illumos-based operating systems
+* shelldocs.py may be used to generate test-patch.sh API information
+* FindBugs output is now listed on the JIRA comment
+* lots of general code cleanup, including attempts to remove any local state files to reduce potential race conditions
+* Some logic to determine if a patch is for a given major branch using several strategies as well as a particular git ref (using git+ref as part of the name).
+* Some logic to determine if a patch references a particular JIRA issue.
+* Unit tests are only flagged as necessary with native or Java code, since Hadoop has no framework in place yet for other types of unit tests.
+* test-patch now exits with a failure status if problems arise trying to do git checkouts. Previously the exit code was success.
+
+
+---
+
+* [YARN-3366](https://issues.apache.org/jira/browse/YARN-3366) | *Major* | **Outbound network bandwidth : classify/shape traffic originating from YARN containers**
+
+1) A TrafficController class that provides an implementation for traffic shaping using tc.
+2) A ResourceHandler implementation for OutboundBandwidth as a resource - isolation/enforcement using cgroups and tc.
+
+
+---
+
+* [HADOOP-11861](https://issues.apache.org/jira/browse/HADOOP-11861) | *Major* | **test-patch.sh rewrite addendum patch**
+
+<!-- markdown -->
+* --build-native=false should work now
+* --branch option lets one specify a branch to test against on the command line
+* On certain Jenkins machines, the artifact directory sometimes gets deleted from outside the test-patch script. There is now some code to try to detect, alert, and quick exit if that happens.
+* Various semi-critical output and bug fixes
+
+
+---
+
+* [HADOOP-10597](https://issues.apache.org/jira/browse/HADOOP-10597) | *Major* | **RPC Server signals backoff to clients when all request queues are full**
+
+This change introduces a new configuration key used by RPC server to decide whether to send backoff signal to RPC Client when RPC call queue is full. When the feature is enabled, RPC server will no longer block on the processing of RPC requests when RPC call queue is full. It helps to improve quality of service when the service is under heavy load. The configuration key is in the format of "ipc.#port#.backoff.enable" where #port# is the port number that RPC server listens on. For example, if you want to enable the feature for the RPC server that listens on 8020, set ipc.8020.backoff.enable to true.
+
+
+---
+
+* [HADOOP-11627](https://issues.apache.org/jira/browse/HADOOP-11627) | *Major* | **Remove io.native.lib.available**
+
+io.native.lib.available was removed. Always use native libraries if they exist.
+
+
+---
+
+* [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | *Major* | **Make setting up the build environment easier**
+
+Includes a docker based solution for setting up a build environment with minimal effort.
+
+
+---
+
+* [HDFS-7281](https://issues.apache.org/jira/browse/HDFS-7281) | *Major* | **Missing block is marked as corrupted block**
+
+The patch improves the reporting around missing blocks and corrupted blocks.
+
+1. A block is missing if and only if all DNs of its expected replicas are dead.
+2. A block is corrupted if and only if all its available replicas are corrupted. So if a block has 3 replicas; one of the DN is dead, the other two replicas are corrupted; it will be marked as corrupted.
+3. A new line is added to fsck output to display the corrupt block size per file.
+4. A new line is added to fsck output to display the number of missing blocks in the summary section.
+
+
+---
+
+* [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | *Minor* | **releasedocmaker.py should use today's date instead of unreleased**
+
+Use today instead of 'Unreleased' in releasedocmaker.py when --usetoday is given as an option.
+
+
+---
+
+* [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | *Blocker* | **Non-HA rollback compatibility broken**
+
+Non-HA rollback steps have been changed. Run the rollback command on the namenode (`bin/hdfs namenode -rollback`) before starting cluster with '-rollback' option using (sbin/start-dfs.sh -rollback).
+
+
+---
+
+* [MAPREDUCE-2632](https://issues.apache.org/jira/browse/MAPREDUCE-2632) | *Major* | **Avoid calling the partitioner when the numReduceTasks is 1.**
+
+A partitioner is now only created if there are multiple reducers.
+
+
+---
+
+* [HDFS-8241](https://issues.apache.org/jira/browse/HDFS-8241) | *Minor* | **Remove unused NameNode startup option -finalize**
+
+Remove -finalize option from hdfs namenode command.
+
+
+---
+
+* [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | *Major* | **Allow selectively audit logging ops**
+
+Specific HDFS ops can be selectively excluded from audit logging via 'dfs.namenode.audit.log.debug.cmdlist' configuration.
+
+
+---
+
+* [HDFS-8157](https://issues.apache.org/jira/browse/HDFS-8157) | *Major* | **Writes to RAM DISK reserve locked memory for block files**
+
+This change requires setting the dfs.datanode.max.locked.memory configuration key to use the HDFS Lazy Persist feature. Its value limits the combined off-heap memory for blocks in RAM via caching and lazy persist writes.
+
+
+---
+
+* [HDFS-8332](https://issues.apache.org/jira/browse/HDFS-8332) | *Major* | **DFS client API calls should check filesystem closed**
+
+Users may need special attention for this change while upgrading to this version. Previously user could call some APIs(example: setReplication) wrongly even after closing the fs object. With this change DFS client will not allow any operations to call on closed fs objects. As calling fs operations on closed fs is not right thing to do, users need to correct the usage if any.
+
+
+---
+
+* [HADOOP-11698](https://issues.apache.org/jira/browse/HADOOP-11698) | *Major* | **Remove DistCpV1 and Logalyzer**
+
+Removed DistCpV1 and Logalyzer.
+
+
+---
+
+* [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | *Major* | **RPC Invoker relies on static ClientCache which has synchronized(this) blocks**
+
+The Client#call() methods that are deprecated since 0.23 have been removed.
+
+
+---
+
+* [YARN-3684](https://issues.apache.org/jira/browse/YARN-3684) | *Major* | **Change ContainerExecutor's primary lifecycle methods to use a more extensible mechanism for passing information.**
+
+Modifying key methods in ContainerExecutor to use context objects instead of an argument list. This is more extensible and less brittle.
+
+
+---
+
+* [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | *Major* | **Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HDFS-8135](https://issues.apache.org/jira/browse/HDFS-8135) | *Major* | **Remove the deprecated FSConstants class**
+
+The FSConstants class has been deprecated since 0.23 and it is removed in the release.
+
+
+---
+
+* [MAPREDUCE-6336](https://issues.apache.org/jira/browse/MAPREDUCE-6336) | *Major* | **Enable v2 FileOutputCommitter by default**
+
+mapreduce.fileoutputcommitter.algorithm.version now defaults to 2.
+
+In algorithm version 1:
+
+ 1. commitTask renames directory
+ $joboutput/\_temporary/$appAttemptID/\_temporary/$taskAttemptID/
+ to
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+
+ 2. recoverTask renames
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+ to
+ $joboutput/\_temporary/($appAttemptID + 1)/$taskID/
+
+ 3. commitJob merges every task output file in
+ $joboutput/\_temporary/$appAttemptID/$taskID/
+ to
+ $joboutput/, then it will delete $joboutput/\_temporary/
+ and write $joboutput/\_SUCCESS
+
+commitJob's run time, number of RPC, is O(n) in terms of output files, which is discussed in MAPREDUCE-4815, and can take minutes.
+
+Algorithm version 2 changes the behavior of commitTask, recoverTask, and commitJob.
+
+ 1. commitTask renames all files in
+ $joboutput/\_temporary/$appAttemptID/\_temporary/$taskAttemptID/
+ to $joboutput/
+
+ 2. recoverTask is a nop strictly speaking, but for
+ upgrade from version 1 to version 2 case, it checks if there
+ are any files in
+ $joboutput/\_temporary/($appAttemptID - 1)/$taskID/
+ and renames them to $joboutput/
+
+ 3. commitJob deletes $joboutput/\_temporary and writes
+ $joboutput/\_SUCCESS
+
+Algorithm 2 takes advantage of task parallelism and makes commitJob itself O(1). However, the window of vulnerability for having incomplete output in $jobOutput directory is much larger. Therefore, pipeline logic for consuming job outputs should be built on checking for existence of \_SUCCESS marker.
+
+
+---
+
+* [YARN-2355](https://issues.apache.org/jira/browse/YARN-2355) | *Major* | **MAX\_APP\_ATTEMPTS\_ENV may no longer be a useful env var for a container**
+
+Removed consumption of the MAX\_APP\_ATTEMPTS\_ENV environment variable
+
+
+---
+
+* [HDFS-5033](https://issues.apache.org/jira/browse/HDFS-5033) | *Minor* | **Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source**
+
+"Permission denied" error message when unable to read local file for -put/copyFromLocal
+
+
+---
+
+* [HADOOP-9905](https://issues.apache.org/jira/browse/HADOOP-9905) | *Major* | **remove dependency of zookeeper for hadoop-client**
+
+Zookeeper jar removed from hadoop-client dependency tree.
+
+
+---
+
+* [HDFS-8486](https://issues.apache.org/jira/browse/HDFS-8486) | *Blocker* | **DN startup may cause severe data loss**
+
+<!-- markdown -->
+Public service notice:
+* Every restart of a 2.6.x or 2.7.0 DN incurs a risk of unwanted block deletion.
+* Apply this patch if you are running a pre-2.7.1 release.
+
+
+---
+
+* [HDFS-8270](https://issues.apache.org/jira/browse/HDFS-8270) | *Major* | **create() always retried with hardcoded timeout when file already exists with open lease**
+
+Proxy level retries will not be done on AlreadyBeingCreatedExeption for create() op.
+
+
+---
+
+* [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | *Major* | **The RM should handle the graceful shutdown of the NM.**
+
+The behavior of shutdown a NM could be different (if NM work preserving is not enabled): NM will unregister to RM immediately rather than waiting for timeout to be LOST. A new status of NodeStatus - SHUTDOWN is involved which could affect UI, CLI and ClusterMetrics for node's status.
+
+
+---
+
+* [HDFS-8591](https://issues.apache.org/jira/browse/HDFS-8591) | *Minor* | **Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval**
+
+Related to the decommission enhancements in HDFS-7411, this change removes the deprecated configuration key "dfs.namenode.decommission.nodes.per.interval" which has been subsumed by the configuration key "dfs.namenode.decommission.blocks.per.interval".
+
+
+---
+
+* [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | *Major* | **Allow appending to existing SequenceFiles**
+
+Existing sequence files can be appended.
+
+
+---
+
+* [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | *Minor* | **Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings**
+
+Add a new option "properties" to the "dfsadmin -reconfig" command to get a list of reconfigurable properties.
+
+
+---
+
+* [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | *Major* | **Use slf4j instead of common-logging in hdfs-client**
+
+Users may need special attention for this change while upgrading to this version. Previously hdfs client was using commons-logging as the logging framework. With this change it will use slf4j framework. For more details about slf4j, please see: http://www.slf4j.org/manual.html. Also, org.apache.hadoop.hdfs.protocol.CachePoolInfo#LOG public static member variable has been removed as it is not used anywhere. Users need to correct their code if any one has a reference to this variable. One can retrieve the named logger via the logging framework of their choice directly like, org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(org.apache.hadoop.hdfs.protocol.CachePoolInfo.class);
+
+
+---
+
+* [HDFS-6440](https://issues.apache.org/jira/browse/HDFS-6440) | *Major* | **Support more than 2 NameNodes**
+
+This feature adds support for running additional standby NameNodes, which provides additional fault-tolerance. It is designed for a total of 3-5 NameNodes.
+
+
+---
+
+* [YARN-3823](https://issues.apache.org/jira/browse/YARN-3823) | *Minor* | **Fix mismatch in default values for yarn.scheduler.maximum-allocation-vcores property**
+
+Default value for 'yarn.scheduler.maximum-allocation-vcores' changed from 32 to 4.
+
+
+---
+
+* [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | *Minor* | **Add SFTP FileSystem**
+
+Added SFTP filesystem by using the JSch library.
+
+
+---
+
+* [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | *Major* | **Document missing properties in yarn-default.xml**
+
+Documented missing properties and added the regression test to verify that there are no missing properties in yarn-default.xml.
+
+
+---
+
+* [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | *Minor* | **Fix typo in JobHistoryEventHandler**
+
+There is a typo in the event string "WORKFLOW\_ID" (as "WORKLFOW\_ID"). The branch-2 change will publish both event strings for compatibility with consumers, but the misspelled metric will be removed in trunk.
+
+
+---
+
+* [HDFS-7582](https://issues.apache.org/jira/browse/HDFS-7582) | *Major* | **Enforce maximum number of ACL entries separately per access and default.**
+
+Limit on Maximum number of ACL entries(32) will be enforced separately on access and default ACLs. So in total, max. 64 ACL entries can be present in a ACL spec.
+
+
+---
+
+* [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | *Major* | **Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3**
+
+The Maven dependency on aws-sdk has been changed to aws-sdk-s3 and the version bumped. Applications depending on transitive dependencies pulled in by aws-sdk and not aws-sdk-s3 might not work.
+
+
+---
+
+* [HDFS-8818](https://issues.apache.org/jira/browse/HDFS-8818) | *Major* | **Allow Balancer to run faster**
+
+Add a new conf "dfs.balancer.max-size-to-move" so that Balancer.MAX\_SIZE\_TO\_MOVE becomes configurable.
+
+
+---
+
+* [HDFS-8895](https://issues.apache.org/jira/browse/HDFS-8895) | *Major* | **Remove deprecated BlockStorageLocation APIs**
+
+This removes the deprecated DistributedFileSystem#getFileBlockStorageLocations API used for getting VolumeIds of block replicas. Applications interested in the volume of a replica can instead consult BlockLocation#getStorageIds to obtain equivalent information.
+
+
+---
+
+* [HADOOP-12352](https://issues.apache.org/jira/browse/HADOOP-12352) | *Trivial* | **Delay in checkpointing Trash can leave trash for 2 intervals before deleting**
+
+Fixes an Trash related issue wherein a delay in the periodic checkpointing of one user's directory causes the subsequent user directory checkpoints to carry a newer timestamp, thereby delaying their eventual deletion.
+
+
+---
+
+* [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | *Major* | **Compact XAttrs to optimize memory footprint.**
+
+The config key "dfs.namenode.fs-limits.max-xattr-size" can no longer be set to a value of 0 (previously used to indicate unlimited) or a value greater than 32KB. This is a constraint on xattr size similar to many local filesystems.
+
+
+---
+
+* [HDFS-8890](https://issues.apache.org/jira/browse/HDFS-8890) | *Major* | **Allow admin to specify which blockpools the balancer should run on**
+
+Adds a new blockpools flag to the balancer. This allows admins to specify which blockpools the balancer will run on.
+Usage:
+-blockpools \<comma-separated list of blockpool ids\>
+The balancer will only run on blockpools included in this list.
+
+
+---
+
+* [HDFS-8981](https://issues.apache.org/jira/browse/HDFS-8981) | *Minor* | **Adding revision to data node jmx getVersion() method**
+
+getSoftwareVersion method would replace original getVersion method, which returns the version string.
+
+The new getVersion method would return both version string and revision string.
+
+
+---
+
+* [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | *Major* | **Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [HADOOP-12384](https://issues.apache.org/jira/browse/HADOOP-12384) | *Major* | **Add "-direct" flag option for fs copy so that user can choose not to create ".\_COPYING\_" file**
+
+An option '-d' added for all command-line copy commands to skip intermediate '.COPYING' file creation.
+
+
+---
+
+* [HDFS-8929](https://issues.apache.org/jira/browse/HDFS-8929) | *Major* | **Add a metric to expose the timestamp of the last journal**
+
+Exposed a metric 'LastJournalTimestamp' for JournalNode
+
+
+---
+
+* [HDFS-7116](https://issues.apache.org/jira/browse/HDFS-7116) | *Major* | **Add a command to get the balancer bandwidth**
+
+Exposed command "-getBalancerBandwidth" in dfsadmin to get the bandwidth of balancer.
+
+
+---
+
+* [HDFS-8829](https://issues.apache.org/jira/browse/HDFS-8829) | *Major* | **Make SO\_RCVBUF and SO\_SNDBUF size configurable for DataTransferProtocol sockets and allow configuring auto-tuning**
+
+HDFS-8829 introduces two new configuration settings: dfs.datanode.transfer.socket.send.buffer.size and dfs.datanode.transfer.socket.recv.buffer.size. These settings can be used to control the socket send buffer and receive buffer sizes respectively on the DataNode for client-DataNode and DataNode-DataNode connections. The default values of both settings are 128KB for backwards compatibility. For optimum performance it is recommended to set these values to zero to enable the OS networking stack to auto-tune buffer sizes.
+
+
+---
+
+* [YARN-313](https://issues.apache.org/jira/browse/YARN-313) | *Critical* | **Add Admin API for supporting node resource configuration in command line**
+
+After this patch, the feature to support NM resource dynamically configuration is completed, so that user can configure NM with new resource without bring NM down or decommissioned.
+Two CLIs are provided to support update resources on individual node or a batch of nodes:
+1. Update resource on single node: yarn rmadmin -updateNodeResource [NodeID] [MemSize] [vCores]
+2. Update resource on a batch of nodes: yarn rmadmin -refreshNodesResources, that reflect nodes' resource configuration defined in dynamic-resources.xml which is loaded by RM dynamically (like capacity-scheduler.xml or fair-scheduler.xml).
+The first version of configuration format is:
+\<configuration\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.nodes\</name\>
+ \<value\>h1:1234\</value\>
+ \</property\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.h1:1234.vcores\</name\>
+ \<value\>16\</value\>
+ \</property\>
+ \<property\>
+ \<name\>yarn.resource.dynamic.h1:1234.memory\</name\>
+ \<value\>1024\</value\>
+ \</property\>
+\</configuration\>
+
+
+---
+
+* [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | *Major* | **Trash messages should be handled by Logger instead of being delivered on System.out**
+
+Now trash message is not printed to System.out. It is handled by Logger instead.
+
+
+---
+
+* [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | *Minor* | **Create StatsD metrics2 sink**
+
+Added StatsD metrics2 sink
+
+
+---
+
+* [HDFS-9013](https://issues.apache.org/jira/browse/HDFS-9013) | *Major* | **Deprecate NameNodeMXBean#getNNStarted in branch2 and remove from trunk**
+
+NameNodeMXBean#getNNStarted() metric is deprecated in branch-2 and removed from trunk.
+
+
+---
+
+* [HADOOP-12437](https://issues.apache.org/jira/browse/HADOOP-12437) | *Major* | **Allow SecurityUtil to lookup alternate hostnames**
+
+HADOOP-12437 introduces two new configuration settings: hadoop.security.dns.interface and hadoop.security.dns.nameserver. These settings can be used to control how Hadoop service instances look up their own hostname and may be required in some multi-homed environments where hosts are configured with multiple hostnames in DNS or hosts files. They supersede the existing settings dfs.datanode.dns.interface and dfs.datanode.dns.nameserver.
+
+
+---
+
+* [HADOOP-12446](https://issues.apache.org/jira/browse/HADOOP-12446) | *Major* | **Undeprecate createNonRecursive()**
+
+FileSystem#createNonRecursive() is undeprecated.
+
+
+---
+
+* [HDFS-8696](https://issues.apache.org/jira/browse/HDFS-8696) | *Major* | **Make the lower and higher watermark in the DN Netty server configurable**
+
+Introduced two new configuration dfs.webhdfs.netty.low.watermark and dfs.webhdfs.netty.high.watermark to enable tuning the size of the buffers of the Netty server inside Datanodes.
+
+
+---
+
+* [HDFS-7285](https://issues.apache.org/jira/browse/HDFS-7285) | *Major* | **Erasure Coding Support inside HDFS**
+
+<!-- markdown -->
+HDFS now provides native support for erasure coding (EC) to store data more efficiently. Each individual directory can be configured with an EC policy with command `hdfs erasurecode -setPolicy`. When a file is created, it will inherit the EC policy from its nearest ancestor directory to determine how its blocks are stored. Compared to 3-way replication, the default EC policy saves 50% of storage space while also tolerating more storage failures.
+
+To support small files, the currently phase of HDFS-EC stores blocks in _striped_ layout, where a logical file block is divided into small units (64KB by default) and distributed to a set of DataNodes. This enables parallel I/O but also decreases data locality. Therefore, the cluster environment and I/O workloads should be considered before configuring EC policies.
+
+
+---
+
+* [HDFS-9085](https://issues.apache.org/jira/browse/HDFS-9085) | *Trivial* | **Show renewer information in DelegationTokenIdentifier#toString**
+
+The output of the "hdfs fetchdt --print" command now includes the token renewer appended to the end of the existing token information. This change may be incompatible with tools that parse the output of the command.
+
+
+---
+
+* [HADOOP-12493](https://issues.apache.org/jira/browse/HADOOP-12493) | *Major* | **bash unit tests are failing**
+
+In the extremely rare event that HADOOP\_USER\_IDENT and USER environment variables are not defined, we now fall back to use 'hadoop' as the identification string.
+
+
+---
+
+* [HADOOP-12495](https://issues.apache.org/jira/browse/HADOOP-12495) | *Major* | **Fix posix\_spawn error on OS X**
+
+When Hadoop JVMs create other processes on OS X, it will always use posix\_spawn.
+
+
+---
+
+* [HDFS-9070](https://issues.apache.org/jira/browse/HDFS-9070) | *Major* | **Allow fsck display pending replica location information for being-written blocks**
+
+The output of fsck command for being written hdfs files had been changed. When using fsck against being written hdfs files with {{-openforwrite}} and {{-files -blocks -locations}}, the fsck output will include the being written block for replication files or being written block group for erasure code files.
+
+
+---
+
+* [HDFS-9278](https://issues.apache.org/jira/browse/HDFS-9278) | *Trivial* | **Fix preferredBlockSize typo in OIV XML output**
+
+The preferred block size XML element has been corrected from "\\<perferredBlockSize\>" to "\\<preferredBlockSize\>".
+
+
+---
+
+* [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | *Major* | **Logging HDFS operation's caller context into audit logs**
+
+The feature needs to enabled by setting "hadoop.caller.context.enabled" to true. When the feature is used, additional fields are written into namenode audit log records.
+
+
+---
+
+* [HDFS-9259](https://issues.apache.org/jira/browse/HDFS-9259) | *Major* | **Make SO\_SNDBUF size configurable at DFSClient side for hdfs write scenario**
+
+Introduces a new configuration setting dfs.client.socket.send.buffer.size to control the socket send buffer size for writes. Setting it to zero enables TCP auto-tuning on systems that support it.
+
+
+---
+
+* [HDFS-9311](https://issues.apache.org/jira/browse/HDFS-9311) | *Major* | **Support optional offload of NameNode HA service health checks to a separate RPC server.**
+
+There is now support for offloading HA health check RPC activity to a separate RPC server endpoint running within the NameNode process. This may improve reliability of HA health checks and prevent spurious failovers in highly overloaded conditions. For more details, please refer to the hdfs-default.xml documentation for properties dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host and dfs.namenode.lifeline.handler.count.
+
+
+---
+
+* [HDFS-6200](https://issues.apache.org/jira/browse/HDFS-6200) | *Major* | **Create a separate jar for hdfs-client**
+
+Projects that access HDFS can depend on the hadoop-hdfs-client module instead of the hadoop-hdfs module to avoid pulling in unnecessary dependency.
+
+
+---
+
+* [HADOOP-10787](https://issues.apache.org/jira/browse/HADOOP-10787) | *Blocker* | **Rename/remove non-HADOOP\_\*, etc from the shell scripts**
+
+<!-- markdown -->
+The following shell environment variables have been deprecated:
+
+| Old | New |
+|:---- |:---- |
+| DEFAULT\_LIBEXEC\_DIR | HADOOP\_DEFAULT\_LIBEXEC\_DIR |
+| SLAVE\_NAMES | HADOOP\_SLAVE\_NAMES |
+| TOOL\_PATH | HADOOP\_TOOLS\_PATH |
+
+In addition:
+
+* DEFAULT\_LIBEXEC\_DIR will NOT be automatically transitioned to HADOOP\_DEFAULT\_LIBEXEC\_DIR and will require changes to any scripts setting that value. A warning will be printed to the screen if DEFAULT\_LIBEXEC\_DIR has been configured.
+* HADOOP\_TOOLS\_PATH is now properly handled as a multi-valued, Java classpath-style variable. Prior, multiple values assigned to TOOL\_PATH would not work a predictable manner.
+
+
+---
+
+* [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | *Major* | **allow/disallow snapshots via webhdfs**
+
+Snapshots can be allowed/disallowed on a directory via WebHdfs from users with superuser privilege.
+
+
+---
+
+* [MAPREDUCE-5485](https://issues.apache.org/jira/browse/MAPREDUCE-5485) | *Critical* | **Allow repeating job commit by extending OutputCommitter API**
+
+Previously, the MR job will get failed if AM get restarted for some reason (like node failure, etc.) during its doing commit job no matter if AM attempts reach to the maximum attempts.
+In this improvement, we add a new API isCommitJobRepeatable() to OutputCommitter interface which to indicate if job's committer can do commitJob again if previous commit work is interrupted by NM/AM failures, etc. The instance of OutputCommitter, which support repeatable job commit (like FileOutputCommitter in algorithm 2), can allow AM to continue the commitJob() after AM restart as a new attempt.
+
+
+---
+
+* [HADOOP-12294](https://issues.apache.org/jira/browse/HADOOP-12294) | *Major* | **Throw an Exception when fs.permissions.umask-mode is misconfigured**
+
+The support of the deprecated dfs.umask key is removed in Hadoop 3.0.
+
+
+---
+
+* [HADOOP-10465](https://issues.apache.org/jira/browse/HADOOP-10465) | *Minor* | **Fix use of generics within SortedMapWritable**
+
+SortedMapWritable has changed to SortedMapWritable\<K extends WritableComparable\<? super K\>\>. That way user can declare the class by such as SortedMapWritable\<Text\>.
+
+
+---
+
+* [HADOOP-12313](https://issues.apache.org/jira/browse/HADOOP-12313) | *Critical* | **NPE in JvmPauseMonitor when calling stop() before start()**
+
+Allow stop() before start() completed in JvmPauseMonitor
+
+
+---
+
+* [HDFS-5165](https://issues.apache.org/jira/browse/HDFS-5165) | *Minor* | **Remove the TotalFiles metrics**
+
+Now TotalFiles metric is removed from FSNameSystem. Use FilesTotal instead.
+
+
+---
+
+* [HDFS-8335](https://issues.apache.org/jira/browse/HDFS-8335) | *Major* | **FSNamesystem should construct FSPermissionChecker only if permission is enabled**
+
+Only check permissions when permissions enabled in FSDirStatAndListingOp.getFileInfo() and getListingInt()
+
+
+---
+
+* [HDFS-8831](https://issues.apache.org/jira/browse/HDFS-8831) | *Major* | **Trash Support for deletion in HDFS encryption zone**
+
+Add Trash support for deleting files within encryption zones. Deleted files will remain encrypted and they will be moved to a \u201c.Trash\u201d subdirectory under the root of the encryption zone, prefixed by $USER/current. Checkpoint and expunge continue to work like the existing Trash.
+
+
+---
+
+* [HDFS-9214](https://issues.apache.org/jira/browse/HDFS-9214) | *Major* | **Support reconfiguring dfs.datanode.balance.max.concurrent.moves without DN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> start
+3. repeat step 2 until all DNs are reconfigured
+4. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> status
+5. to query a list reconfigurable properties on DN, run
+ hdfs dfsadmin -reconfig datanode \<dn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | *Major* | **We should have a config to indicate the Timeline Service version**
+
+Add a new configuration "yarn.timeline-service.version" to indicate what is the current version of the running timeline service. For example, if "yarn.timeline-service.version" is 1.5, and "yarn.timeline-service.enabled" is true, it means the cluster will and should bring up the timeline service v.1.5. On the client side, if the client uses the same version of timeline service, it should succeed. If the client chooses to use a smaller version in spite of this, then depending on how robust the compatibility story is between versions, the results may vary.
+
+
+---
+
+* [YARN-4207](https://issues.apache.org/jira/browse/YARN-4207) | *Major* | **Add a non-judgemental YARN app completion status**
+
+Adds the ENDED attribute to o.a.h.yarn.api.records.FinalApplicationStatus
+
+
+---
+
+* [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | *Minor* | **Add a option to skip newline on empty files with getMerge -nl**
+
+Added -skip-empty-file option to hadoop fs -getmerge command. With the option, delimiter (LF) is not printed for empty files even if -nl option is used.
+
+
+---
+
+* [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | *Major* | **Retire libwebhdfs**
+
+libwebhdfs has been retired in 2.8.0 due to the lack of maintenance.
+
+
+---
+
+* [HADOOP-11262](https://issues.apache.org/jira/browse/HADOOP-11262) | *Major* | **Enable YARN to use S3A**
+
+S3A has been made accessible through the FileContext API.
+
+
+---
+
+* [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | *Major* | **Adding Append API support for WASB**
+
+The Azure Blob Storage file system (WASB) now includes optional support for use of the append API by a single writer on a path. Please note that the implementation differs from the semantics of HDFS append. HDFS append internally guarantees that only a single writer may append to a path at a given time. WASB does not enforce this guarantee internally. Instead, the application must enforce access by a single writer, such as by running single-threaded or relying on some external locking mechanism to coordinate concurrent processes. Refer to the Azure Blob Storage documentation page for more details on enabling append in configuration.
+
+
+---
+
+* [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | *Major* | **Replace dev-support with wrappers to Yetus**
+
+<!-- markdown -->
+
+* Major portions of dev-support have been replaced with wrappers to Apache Yetus:
+ * releasedocmaker.py is now dev-support/bin/releasedocmaker
+ * shelldocs.py is now dev-support/bin/shelldocs
+ * smart-apply-patch.sh is now dev-support/bin/smart-apply-patch
+ * test-patch.sh is now dev-support/bin/test-patch
+* See the dev-support/README.md file for more details on how to control the wrappers to various degrees.
+
+
+---
+
+* [HDFS-9525](https://issues.apache.org/jira/browse/HDFS-9525) | *Blocker* | **hadoop utilities need to support provided delegation tokens**
+
+If hadoop.token.files property is defined and configured to one or more comma-delimited delegation token files, Hadoop will use those token files to connect to the services as named in the token.
+
+
+---
+
+* [HDFS-9503](https://issues.apache.org/jira/browse/HDFS-9503) | *Major* | **Replace -namenode option with -fs for NNThroughputBenchmark**
+
+The patch replaces -namenode option with -fs for specifying the remote name node against which the benchmark is running. Before this patch, if '-namenode' was not given, the benchmark would run in standalone mode, ignoring the 'fs.defaultFS' in config file even if it's remote. With this patch, the benchmark, as other tools, will rely on the 'fs.defaultFS' config, which is overridable by -fs command option, to run standalone mode or remote mode.
+
+
+---
+
+* [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | *Minor* | **Add Entry point for Kerberos health check**
+
+Hadoop now includes a shell command named KDiag that helps with diagnosis of Kerberos misconfiguration problems. Please refer to the Secure Mode documentation for full details on usage of the command.
+
+
+---
+
+* [HADOOP-12805](https://issues.apache.org/jira/browse/HADOOP-12805) | *Major* | **Annotate CanUnbuffer with @InterfaceAudience.Public**
+
+Made CanBuffer interface public for use in client applications.
+
+
+---
+
+* [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | *Major* | **Read s3a creds from a Credential Provider**
+
+The S3A Hadoop-compatible file system now support reading its S3 credentials from the Hadoop Credential Provider API in addition to XML configuration files.
+
+
+---
+
+* [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | *Major* | **Integrate CSRF prevention filter in WebHDFS.**
+
+WebHDFS now supports options to enforce cross-site request forgery (CSRF) prevention for HTTP requests to both the NameNode and the DataNode. Please refer to the updated WebHDFS documentation for a description of this feature and further details on how to configure it.
+
+
+---
+
+* [HADOOP-12794](https://issues.apache.org/jira/browse/HADOOP-12794) | *Major* | **Support additional compression levels for GzipCodec**
+
+Added New compression levels for GzipCodec that can be set in zlib.compress.level
+
+
+---
+
+* [MAPREDUCE-6613](https://issues.apache.org/jira/browse/MAPREDUCE-6613) | *Minor* | **Change mapreduce.jobhistory.jhist.format default from json to binary**
+
+Default of 'mapreduce.jobhistory.jhist.format' property changed from 'json' to 'binary'. Creates smaller, binary Avro .jhist files for faster JHS performance.
+
+
+---
+
+* [HDFS-9425](https://issues.apache.org/jira/browse/HDFS-9425) | *Major* | **Expose number of blocks per volume as a metric**
+
+Number of blocks per volume is made available as a metric.
+
+
+---
+
+* [HADOOP-12668](https://issues.apache.org/jira/browse/HADOOP-12668) | *Critical* | **Support excluding weak Ciphers in HttpServer2 through ssl-server.conf**
+
+The Code Changes include following:
+- Modified DFSUtil.java in Apache HDFS project for supplying new parameter ssl.server.exclude.cipher.list
+- Modified HttpServer2.java in Apache Hadoop-common project to work with new parameter and exclude ciphers using jetty setExcludeCihers method.
+- Modfied associated test classes to owrk with existing code and also cover the newfunctionality in junit
+
+
+---
+
+* [HADOOP-12555](https://issues.apache.org/jira/browse/HADOOP-12555) | *Minor* | **WASB to read credentials from a credential provider**
+
+The hadoop-azure file system now supports configuration of the Azure Storage account credentials using the standard Hadoop Credential Provider API. For details, please refer to the documentation on hadoop-azure and the Credential Provider API.
+
+
+---
+
+* [HDFS-9395](https://issues.apache.org/jira/browse/HDFS-9395) | *Major* | **Make HDFS audit logging consistant**
+
+<!-- markdown -->
+
+Audit logs will now only be generated in the following two cases:
+* When an operation results in an `AccessControlException`
+* When an operation is successful
+
+Notably, this means audit log events will not be generated for exceptions besides AccessControlException.
+
+
+---
+
+* [MAPREDUCE-6622](https://issues.apache.org/jira/browse/MAPREDUCE-6622) | *Critical* | **Add capability to set JHS job cache to a task-based limit**
+
+Two recommendations for the mapreduce.jobhistory.loadedtasks.cache.size property:
+1) For every 100k of cache size, set the heap size of the Job History Server to 1.2GB. For example, mapreduce.jobhistory.loadedtasks.cache.size=500000, heap size=6GB.
+2) Make sure that the cache size is larger than the number of tasks required for the largest job run on the cluster. It might be a good idea to set the value slightly higher (say, 20%) in order to allow for job size growth.
+
+
+---
+
+* [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | *Minor* | **Fix undeclared/unused dependency to httpclient**
+
+Dependency on commons-httpclient::commons-httpclient was removed from hadoop-common. Downstream projects using commons-httpclient transitively provided by hadoop-common need to add explicit dependency to their pom. Since commons-httpclient is EOL, it is recommended to migrate to org.apache.httpcomponents:httpclient which is the successor.
+
+
+---
+
+* [HADOOP-12850](https://issues.apache.org/jira/browse/HADOOP-12850) | *Major* | **pull shell code out of hadoop-dist**
+
+This change contains the content of HADOOP-10115 which is an incompatible change.
+
+
+---
+
+* [HDFS-8791](https://issues.apache.org/jira/browse/HDFS-8791) | *Blocker* | **block ID-based DN storage layout can be very slow for datanode on ext4**
+
+HDFS-8791 introduces a new datanode layout format. This layout is identical to the previous block id based layout except it has a smaller 32x32 sub-directory structure in each data storage. On startup, the datanode will automatically upgrade it's storages to this new layout. Currently, datanode layout changes support rolling upgrades, on the other hand downgrading is not supported between datanode layout changes and a rollback would be required.
+
+
+---
+
+* [HDFS-9887](https://issues.apache.org/jira/browse/HDFS-9887) | *Major* | **WebHdfs socket timeouts should be configurable**
+
+Added new configuration options: dfs.webhdfs.socket.connect-timeout and dfs.webhdfs.socket.read-timeout both defaulting to 60s.
+
+
+---
+
+* [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | *Major* | **Remove all of the CHANGES.txt files**
+
+With the introduction of the markdown-formatted and automatically built changes file, the CHANGES.txt files have been eliminated.
+
+
+---
+
+* [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | *Major* | **DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness**
+
+This release adds a new feature called the DataNode Lifeline Protocol. If configured, then DataNodes can report that they are still alive to the NameNode via a fallback protocol, separate from the existing heartbeat messages. This can prevent the NameNode from incorrectly marking DataNodes as stale or dead in highly overloaded clusters where heartbeat processing is suffering delays. For more information, please refer to the hdfs-default.xml documentation for several new configuration properties: dfs.namenode.lifeline.rpc-address, dfs.namenode.lifeline.rpc-bind-host, dfs.datanode.lifeline.interval.seconds, dfs.namenode.lifeline.handler.ratio and dfs.namenode.lifeline.handler.count.
+
+
+---
+
+* [YARN-4762](https://issues.apache.org/jira/browse/YARN-4762) | *Blocker* | **NMs failing on DelegatingLinuxContainerRuntime init with LCE on**
+
+Fixed CgroupHandler's creation and usage to avoid NodeManagers crashing when LinuxContainerExecutor is enabled.
+
+
+---
+
+* [HDFS-1477](https://issues.apache.org/jira/browse/HDFS-1477) | *Major* | **Support reconfiguring dfs.heartbeat.interval and dfs.namenode.heartbeat.recheck-interval without NN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> start
+3. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> status
+4. to query a list reconfigurable properties on NN, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | *Major* | **inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler**
+
+**WARNING: No release note provided for this change.**
+
+
+---
+
+* [YARN-4732](https://issues.apache.org/jira/browse/YARN-4732) | *Trivial* | **\*ProcessTree classes have too many whitespace issues**
+
+
+
+
+---
+
+* [HADOOP-12857](https://issues.apache.org/jira/browse/HADOOP-12857) | *Major* | **Rework hadoop-tools**
+
+<!-- markdown -->
+* Turning on optional things from the tools directory such as S3 support can now be done in hadoop-env.sh with the HADOOP\_OPTIONAL\_TOOLS environment variable without impacting the various user-facing CLASSPATH variables.
+* The tools directory is no longer pulled in blindly for any utilities that pull it in.
+* TOOL\_PATH / HADOOP\_TOOLS\_PATH has been broken apart and replaced with HADOOP\_TOOLS\_HOME, HADOOP\_TOOLS\_DIR and HADOOP\_TOOLS\_LIB\_JARS\_DIR to be consistent with the rest of Hadoop.
+
+
+---
+
+* [HDFS-9694](https://issues.apache.org/jira/browse/HDFS-9694) | *Major* | **Make existing DFSClient#getFileChecksum() work for striped blocks**
+
+Makes the getFileChecksum API works with striped layout EC files. Checksum computation done by block level in the distributed fashion. The current API does not support to compare the checksum generated with normal file and the checksum generated for the same file but in striped layout.
+
+
+---
+
+* [HDFS-9640](https://issues.apache.org/jira/browse/HDFS-9640) | *Major* | **Remove hsftp from DistCp in trunk**
+
+DistCp in Hadoop 3.0 no longer supports -mapredSSLConf option. Use global ssl-client.xml configuration file for swebhdfs file systems instead.
+
+
+---
+
+* [HDFS-9349](https://issues.apache.org/jira/browse/HDFS-9349) | *Major* | **Support reconfiguring fs.protected.directories without NN restart**
+
+Steps to reconfigure:
+1. change value of the parameter in corresponding xml configuration file
+2. to reconfigure, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> start
+3. to check status of the most recent reconfigure operation, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> status
+4. to query a list reconfigurable properties on NN, run
+ hdfs dfsadmin -reconfig namenode \<nn\_addr\>:\<ipc\_port\> properties
+
+
+---
+
+* [HADOOP-11393](https://issues.apache.org/jira/browse/HADOOP-11393) | *Major* | **Revert HADOOP\_PREFIX, go back to HADOOP\_HOME**
+
+On Unix platforms, HADOOP\_PREFIX has been deprecated in favor of returning to HADOOP\_HOME as in prior Apache Hadoop releases.
+
+
+---
+
+* [HADOOP-12967](https://issues.apache.org/jira/browse/HADOOP-12967) | *Major* | **Remove FileUtil#copyMerge**
+
+Removed FileUtil.copyMerge.
+
+
+---
+
+* [MAPREDUCE-6670](https://issues.apache.org/jira/browse/MAPREDUCE-6670) | *Minor* | **TestJobListCache#testEviction sometimes fails on Windows with timeout**
+
+Backport the fix to 2.7 and 2.8
+
+
+---
+
+* [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | *Major* | **Datanode command for evicting writers**
+
+This new dfsadmin command, evictWriters, stops active block writing activities on a data node. The affected writes will continue without the node after a write pipeline recovery. This is useful when data node decommissioning is blocked by slow writers. If issued against a non-decommissioing data node, all current writers will be stopped, but new write requests will continue to be served.
+
+
+---
+
+* [HADOOP-12963](https://issues.apache.org/jira/browse/HADOOP-12963) | *Minor* | **Allow using path style addressing for accessing the s3 endpoint**
+
+Add new flag to allow supporting path style addressing for s3a
+
+
+---
+
+* [HADOOP-12811](https://issues.apache.org/jira/browse/HADOOP-12811) | *Critical* | **Change kms server port number which conflicts with HMaster port number**
+
+The default port for KMS service is now 9600. This is to avoid conflicts on the previous port 16000, which is also used by HMaster as the default port.
+
+
+---
+
+* [YARN-4784](https://issues.apache.org/jira/browse/YARN-4784) | *Major* | **Fairscheduler: defaultQueueSchedulingPolicy should not accept FIFO**
+
+Clusters cannot use FIFO policy as the defaultQueueSchedulingPolicy. Clusters with a single level of queues will have to explicitly set the policy to FIFO if that is desired.
+
+
+---
+
+* [HDFS-9427](https://issues.apache.org/jira/browse/HDFS-9427) | *Critical* | **HDFS should not default to ephemeral ports**
+
+The patch updates the HDFS default HTTP/RPC ports to non-ephemeral ports. The changes are listed below:
+Namenode ports: 50470 --\> 9871, 50070 --\> 9870, 8020 --\> 9820
+Secondary NN ports: 50091 --\> 9869, 50090 --\> 9868
+Datanode ports: 50020 --\> 9867, 50010 --\> 9866, 50475 --\> 9865, 50075 --\> 9864
+
+
+---
+
+* [HDFS-3702](https://issues.apache.org/jira/browse/HDFS-3702) | *Minor* | **Add an option for NOT writing the blocks locally if there is a datanode on the same box as the client**
+
+This patch will attempt to allocate all replicas to remote DataNodes, by adding local DataNode to the excluded DataNodes. If no sufficient replicas can be obtained, it will fall back to default block placement policy, which writes one replica to local DataNode.
+
+
+---
+
+* [HADOOP-12563](https://issues.apache.org/jira/browse/HADOOP-12563) | *Major* | **Updated utility to create/modify token files**
+
+This feature introduces a new command called "hadoop dtutil" which lets users request and download delegation tokens with certain attributes.
+
+
+---
+
+* [HADOOP-13045](https://issues.apache.org/jira/browse/HADOOP-13045) | *Major* | **hadoop\_add\_classpath is not working in .hadooprc**
+
+<!-- markdown -->
+With this change, the `.hadooprc` file is now processed after Apache Hadoop has been fully bootstrapped. This allows for usage of the Apache Hadoop Shell API. A new file, `.hadoop-env`, now provides the ability for end users to override `hadoop-env.sh`.
+
+
+---
+
+* [MAPREDUCE-6526](https://issues.apache.org/jira/browse/MAPREDUCE-6526) | *Blocker* | **Remove usage of metrics v1 from hadoop-mapreduce**
+
+LocalJobRunnerMetrics and ShuffleClientMetrics were updated to use Hadoop Metrics V2 framework.
+
+
+---
+
+* [HDFS-9902](https://issues.apache.org/jira/browse/HDFS-9902) | *Major* | **Support different values of dfs.datanode.du.reserved per storage type**
+
+Reserved space can be configured independently for different storage types for clusters with heterogeneous storage. The 'dfs.datanode.du.reserved' property name can be suffixed with a storage types (i.e. one of ssd, disk, archival or ram\_disk). e.g. reserved space for RAM\_DISK storage can be configured using the property 'dfs.datanode.du.reserved.ram\_disk'. If specific storage type reservation is not configured then the value specified by 'dfs.datanode.du.reserved' will be used for all volumes.
+
+
+---
+
+* [HADOOP-12504](https://issues.apache.org/jira/browse/HADOOP-12504) | *Blocker* | **Remove metrics v1**
+
+<!-- markdown -->
+* org.apache.hadoop.metrics package was removed. Use org.apache.hadoop.metrics2 package instead.
+* "/metrics" endpoint was removed. Use "/jmx" instead to see the metrics.
+
+
+---
+
+* [HDFS-10324](https://issues.apache.org/jira/browse/HDFS-10324) | *Major* | **Trash directory in an encryption zone should be pre-created with correct permissions**
+
+HDFS will create a ".Trash" subdirectory when creating a new encryption zone to support soft delete for files deleted within the encryption zone. A new "crypto -provisionTrash" command has been introduced to provision trash directories for encryption zones created with Apache Hadoop minor releases prior to 2.8.0.
+
+
+---
+
+* [HDFS-10337](https://issues.apache.org/jira/browse/HDFS-10337) | *Minor* | **OfflineEditsViewer stats option should print 0 instead of null for the count of operations**
+
+The output of "hdfs oev -p stats" has changed. The option prints 0 instead of null for the count of the operations that have never been executed.
+
+
+---
+
+* [HADOOP-10694](https://issues.apache.org/jira/browse/HADOOP-10694) | *Major* | **Remove synchronized input streams from Writable deserialization**
+
+Remove invisible synchronization primitives from DataInputBuffer
+
+
+---
+
+* [HADOOP-13122](https://issues.apache.org/jira/browse/HADOOP-13122) | *Minor* | **Customize User-Agent header sent in HTTP requests by S3A.**
+
+S3A now includes the current Hadoop version in the User-Agent string passed through the AWS SDK to the S3 service. Users also may include optional additional information to identify their application. See the documentation of configuration property fs.s3a.user.agent.prefix for further details.
+
+
+---
+
+* [HADOOP-11858](https://issues.apache.org/jira/browse/HADOOP-11858) | *Blocker* | **[JDK8] Set minimum version of Hadoop 3 to JDK 8**
+
+The minimum required JDK version for Hadoop has been increased from JDK7 to JDK8.
+
+
+---
+
+* [HADOOP-12930](https://issues.apache.org/jira/browse/HADOOP-12930) | *Critical* | **[Umbrella] Dynamic subcommands for hadoop shell scripts**
+
+It is now possible to add or modify the behavior of existing subcommands in the hadoop, hdfs, mapred, and yarn scripts. See the Unix Shell Guide for more information.
+
+
+---
+
+* [HADOOP-12782](https://issues.apache.org/jira/browse/HADOOP-12782) | *Major* | **Faster LDAP group name resolution with ActiveDirectory**
+
+If the user object returned by LDAP server has the user's group object DN (supported by Active Directory), Hadoop can reduce LDAP group mapping latency by setting hadoop.security.group.mapping.ldap.search.attr.memberof to memberOf.
+
+
+---
+
+* [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | *Major* | **S3A: Add ability to plug in any AWSCredentialsProvider**
+
+Users can integrate a custom credential provider with S3A. See documentation of configuration property fs.s3a.aws.credentials.provider for further details.
+
+
+---
+
+* [MAPREDUCE-6607](https://issues.apache.org/jira/browse/MAPREDUCE-6607) | *Minor* | **Enable regex pattern matching when mapreduce.task.files.preserve.filepattern is set**
+
+Before this fix, the files in .staging directory are always preserved when mapreduce.task.files.preserve.filepattern is set. After this fix, the files in .staging directory are preserved if the name of the directory matches the regex pattern specified by mapreduce.task.files.preserve.filepattern.
+
+
+---
+
+* [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | *Major* | **FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on**
+
+Introducing a new configuration "yarn.scheduler.fair.dynamic.max.assign" to dynamically determine the resources to assign per heartbeat when assignmultiple is turned on. When turned on, the scheduler allocates roughly half of the remaining resources overriding any max.assign settings configured. This is turned ON by default.
+
+
+---
+
+* [YARN-5132](https://issues.apache.org/jira/browse/YARN-5132) | *Critical* | **Exclude generated protobuf sources from YARN Javadoc build**
+
+Exclude javadocs for proto-generated java classes.
+
+
+---
+
+* [HADOOP-13105](https://issues.apache.org/jira/browse/HADOOP-13105) | *Major* | **Support timeouts in LDAP queries in LdapGroupsMapping.**
+
+This patch adds two new config keys for supporting timeouts in LDAP query operations. The property "hadoop.security.group.mapping.ldap.connection.timeout.ms" is the connection timeout (in milliseconds), within which period if the LDAP provider doesn't establish a connection, it will abort the connect attempt. The property "hadoop.security.group.mapping.ldap.read.timeout.ms" is the read timeout (in milliseconds), within which period if the LDAP provider doesn't get a LDAP response, it will abort the read attempt.
+
+
+---
+
+* [HADOOP-13155](https://issues.apache.org/jira/browse/HADOOP-13155) | *Major* | **Implement TokenRenewer to renew and cancel delegation tokens in KMS**
+
+Enables renewal and cancellation of KMS delegation tokens. hadoop.security.key.provider.path needs to be configured to reach the key provider.
+
+
+---
+
+* [HADOOP-12807](https://issues.apache.org/jira/browse/HADOOP-12807) | *Minor* | **S3AFileSystem should read AWS credentials from environment variables**
+
+Adds support to S3AFileSystem for reading AWS credentials from environment variables.
+
+
+---
+
+* [HDFS-10375](https://issues.apache.org/jira/browse/HDFS-10375) | *Trivial* | **Remove redundant TestMiniDFSCluster.testDualClusters**
+
+Remove redundent TestMiniDFSCluster.testDualClusters to save time.
+
+
+---
+
+* [HDFS-10220](https://issues.apache.org/jira/browse/HDFS-10220) | *Major* | **A large number of expired leases can make namenode unresponsive and cause failover**
+
+Two new configuration have been added "dfs.namenode.lease-recheck-interval-ms" and "dfs.namenode.max-lock-hold-to-release-lease-ms" to fine tune the duty cycle with which the Namenode recovers old leases.
+
+
+---
+
+* [HADOOP-13237](https://issues.apache.org/jira/browse/HADOOP-13237) | *Minor* | **s3a initialization against public bucket fails if caller lacks any credentials**
+
+S3A now supports read access to a public S3 bucket even if the client does not configure any AWS credentials. See the documentation of configuration property fs.s3a.aws.credentials.provider for further details.
+
+
+---
+
+* [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | *Minor* | **S3A to support Amazon STS temporary credentials**
+
+S3A now supports use of AWS Security Token Service temporary credentials for authentication to S3. Refer to the documentation of configuration property fs.s3a.session.token for further details.
+
+
+---
+
+* [HADOOP-13175](https://issues.apache.org/jira/browse/HADOOP-13175) | *Major* | **Remove hadoop-ant from hadoop-tools**
+
+The hadoop-ant module in hadoop-tools has been removed.
+
+
+---
+
+* [HADOOP-12666](https://issues.apache.org/jira/browse/HADOOP-12666) | *Major* | **Support Microsoft Azure Data Lake - as a file system in Hadoop**
+
+Hadoop now supports integration with Azure Data Lake as an alternative Hadoop-compatible file system. Please refer to the Hadoop site documentation of Azure Data Lake for details on usage and configuration.
+
+
+---
+
+* [HDFS-9016](https://issues.apache.org/jira/browse/HDFS-9016) | *Major* | **Display upgrade domain information in fsck**
+
+New fsck option "-upgradedomains" has been added to display upgrade domains of any block.
+
+
+---
+
+* [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | *Blocker* | **fix/rewrite create-release**
+
+This rewrites the release process with a new dev-support/bin/create-release script. See http://wiki.apache.org/hadoop/HowToRelease for updated instructions on how to use it.
+
+
+---
+
+* [HADOOP-3733](https://issues.apache.org/jira/browse/HADOOP-3733) | *Minor* | **"s3:" URLs break when Secret Key contains a slash, even if encoded**
+
+Allows userinfo component of URI authority to contain a slash (escaped as %2F). Especially useful for accessing AWS S3 with distcp or hadoop fs.
+
+
+---
+
+* [HADOOP-13242](https://issues.apache.org/jira/browse/HADOOP-13242) | *Major* | **Authenticate to Azure Data Lake using client ID and keys**
+
+Adds support for Azure ActiveDirectory tokens using client ID and keys
+
+
+---
+
+* [HADOOP-9613](https://issues.apache.org/jira/browse/HADOOP-9613) | *Major* | **[JDK8] Update jersey version to latest 1.x release**
+
+Upgrading Jersey and its related libraries:
+
+1. Upgrading jersey from 1.9 to 1.19
+2. Adding jersey-servlet 1.19
+3. Upgrading grizzly-http-servlet from 2.1.2 to 2.2.21
+4. Adding grizzly-http 2.2.21
+5. Adding grizzly-http-server 2.2.21
+
+After upgrading Jersey from 1.12 to 1.13, the root element whose content is empty collection is changed from null to empty object({}).
+
+
+---
+
+* [HDFS-10328](https://issues.apache.org/jira/browse/HDFS-10328) | *Minor* | **Add per-cache-pool default replication num configuration**
+
+Add per-cache-pool default replication num configuration
+
+
+---
+
+* [HADOOP-13203](https://issues.apache.org/jira/browse/HADOOP-13203) | *Major* | **S3A: Support fadvise "random" mode for high performance readPositioned() reads**
+
+S3A has added support for configurable input policies. Similar to fadvise, this configuration provides applications with a way to specify their expected access pattern (sequential or random) while reading a file. S3A then performs optimizations tailored to that access pattern. See site documentation of the fs.s3a.experimental.input.fadvise configuration property for more details. Please be advised that this feature is experimental and subject to backward-incompatible changes in future releases.
+
+
+---
+
+* [HDFS-1312](https://issues.apache.org/jira/browse/HDFS-1312) | *Major* | **Re-balance disks within a Datanode**
+
+The Disk Balancer lets administrators rebalance data across multiple disks of a DataNode. It is useful to correct skewed data distribution often seen after adding or replacing disks. Disk Balancer can be enabled by setting dfs.disk.balancer.enabled to true in hdfs-site.xml. It can be invoked by running "hdfs diskbalancer". See the "HDFS Diskbalancer" section in the HDFS Commands guide for detailed usage.
+
+
+---
+
+* [HADOOP-13263](https://issues.apache.org/jira/browse/HADOOP-13263) | *Major* | **Reload cached groups in background after expiry**
+
+hadoop.security.groups.cache.background.reload can be set to true to enable background reload of expired groups cache entries. This setting can improve the performance of services that use Groups.java (e.g. the NameNode) when group lookups are slow. The setting is disabled by default.
+
+
+---
+
+* [HDFS-10440](https://issues.apache.org/jira/browse/HDFS-10440) | *Major* | **Improve DataNode web UI**
+
+DataNode Web UI has been improved with new HTML5 page, showing useful information.
+
+
+---
+
+* [HADOOP-13209](https://issues.apache.org/jira/browse/HADOOP-13209) | *Major* | **replace slaves with workers**
+
+The 'slaves' file has been deprecated in favor of the 'workers' file and, other than the deprecation warnings, all references to slavery have been removed from the source tree.
+
+
+---
+
+* [HDFS-6434](https://issues.apache.org/jira/browse/HDFS-6434) | *Minor* | **Default permission for creating file should be 644 for WebHdfs/HttpFS**
+
+The default permissions of files and directories created via WebHDFS and HttpFS are now 644 and 755 respectively. See HDFS-10488 for related discussion.
+
+
+---
+
+* [HADOOP-12864](https://issues.apache.org/jira/browse/HADOOP-12864) | *Blocker* | **Remove bin/rcc script**
+
+The rcc command has been removed. See HADOOP-12485 where unused Hadoop Streaming classes were removed.
+
+
+---
+
+* [HADOOP-12709](https://issues.apache.org/jira/browse/HADOOP-12709) | *Major* | **Cut s3:// from trunk**
+
+The s3 file system has been removed. The s3a file system should be used instead.
+
+
+---
+
+* [HADOOP-12064](https://issues.apache.org/jira/browse/HADOOP-12064) | *Blocker* | **[JDK8] Update guice version to 4.0**
+
+Upgrading following dependences:
+\* Guice from 3.0 to 4.0
+\* cglib from 2.2 to 3.2.0
+\* asm from 3.2 to 5.0.4
+
+
+---
+
+* [HDFS-10548](https://issues.apache.org/jira/browse/HDFS-10548) | *Major* | **Remove the long deprecated BlockReaderRemote**
+
+This removes the configuration property {{dfs.client.use.legacy.blockreader}}, since the legacy remote block reader class has been removed from the codebase.
+
+
+---
+
+* [YARN-2928](https://issues.apache.org/jira/browse/YARN-2928) | *Critical* | **YARN Timeline Service v.2: alpha 1**
+
+We are introducing an early preview (alpha 1) of a major revision of YARN Timeline Service: v.2. YARN Timeline Service v.2 addresses two major challenges: improving scalability and reliability of Timeline Service, and enhancing usability by introducing flows and aggregation.
+
+YARN Timeline Service v.2 alpha 1 is provided so that users and developers can test it and provide feedback and suggestions for making it a ready replacement for Timeline Service v.1.x. It should be used only in a test capacity. Most importantly, security is not enabled. Do not set up or use Timeline Service v.2 until security is implemented if security is a critical requirement.
+
+More details are available in the [YARN Timeline Service v.2](./hadoop-yarn/hadoop-yarn-site/TimelineServiceV2.html) documentation.
+
+
+---
+
+* [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | *Major* | **remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects**
+
+Dependencies on commons-httpclien
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/48] hadoop git commit: HDFS-10820. Reuse closeResponder to reset
the response variable in DataStreamer#run. Contributed by Yiqun Lin.
Posted by aw...@apache.org.
HDFS-10820. Reuse closeResponder to reset the response variable in DataStreamer#run. Contributed by Yiqun Lin.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0690f096
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0690f096
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0690f096
Branch: refs/heads/HADOOP-13341
Commit: 0690f0969efef201b074a8b26400b2e5ba9b97ad
Parents: 74f4bae
Author: Xiao Chen <xi...@apache.org>
Authored: Thu Sep 1 15:34:47 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Thu Sep 1 15:35:25 2016 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/hdfs/DataStreamer.java | 10 ++--------
1 file changed, 2 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0690f096/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index ee673ed..062a416 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -536,14 +536,8 @@ class DataStreamer extends Daemon {
TraceScope scope = null;
while (!streamerClosed && dfsClient.clientRunning) {
// if the Responder encountered an error, shutdown Responder
- if (errorState.hasError() && response != null) {
- try {
- response.close();
- response.join();
- response = null;
- } catch (InterruptedException e) {
- LOG.warn("Caught exception", e);
- }
+ if (errorState.hasError()) {
+ closeResponder();
}
DFSPacket one;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/48] hadoop git commit: Revert "HADOOP-13218. Migrate other Hadoop
side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou
and Kai Zheng"
Posted by aw...@apache.org.
Revert "HADOOP-13218. Migrate other Hadoop side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and Kai Zheng"
This reverts commit 62a9667136ebd8a048f556b534fcff4fdaf8e2ec
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d355573f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d355573f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d355573f
Branch: refs/heads/HADOOP-13341
Commit: d355573f5681f43e760a1bc23ebed553bd35fca5
Parents: f414d5e
Author: Kai Zheng <ka...@intel.com>
Authored: Thu Sep 8 05:50:17 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Thu Sep 8 05:50:17 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 ++-
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 ++++++++++++++-
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 +
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 +++++++++++++--
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 +--
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +---
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 ++---
.../hadoop/security/TestDoAsEffectiveUser.java | 291 ++++++++++++-------
.../security/TestUserGroupInformation.java | 28 +-
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 +
.../TestClientProtocolWithDelegationToken.java | 119 ++++++++
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 +
18 files changed, 876 insertions(+), 294 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index e68bfd4..83e4b9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -60,7 +60,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private static final ThreadLocal<AsyncGet<Message, Exception>>
ASYNC_RETURN_MESSAGE = new ThreadLocal<>();
- static { // Register the rpcRequest deserializer for ProtobufRpcEngine
+ static { // Register the rpcRequest deserializer for WritableRpcEngine
org.apache.hadoop.ipc.Server.registerProtocolEngine(
RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcProtobufRequest.class,
new Server.ProtoBufRpcInvoker());
@@ -194,8 +194,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (args.length != 2) { // RpcController + Message
- throw new ServiceException(
- "Too many or few parameters for request. Method: ["
+ throw new ServiceException("Too many parameters for request. Method: ["
+ method.getName() + "]" + ", Expected: 2, Actual: "
+ args.length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 12a07a5..3f68d63 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.io.InterruptedIOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
@@ -28,6 +26,7 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
+import java.io.*;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -38,12 +37,11 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.apache.commons.logging.*;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.Writable;
+import org.apache.hadoop.io.*;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolInfoService;
@@ -56,6 +54,7 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
@@ -88,7 +87,7 @@ public class RPC {
RPC_WRITABLE ((short) 2), // Use WritableRpcEngine
RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
- private final short value;
+ public final short value; //TODO make it private
RpcKind(short val) {
this.value = val;
@@ -208,7 +207,7 @@ public class RPC {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
- ProtobufRpcEngine.class);
+ WritableRpcEngine.class);
engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
PROTOCOL_ENGINES.put(protocol, engine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 531d574..f20ba94 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -237,14 +237,14 @@ public abstract class Server {
static class RpcKindMapValue {
final Class<? extends Writable> rpcRequestWrapperClass;
final RpcInvoker rpcInvoker;
-
RpcKindMapValue (Class<? extends Writable> rpcRequestWrapperClass,
RpcInvoker rpcInvoker) {
this.rpcInvoker = rpcInvoker;
this.rpcRequestWrapperClass = rpcRequestWrapperClass;
}
}
- static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new HashMap<>(4);
+ static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new
+ HashMap<RPC.RpcKind, RpcKindMapValue>(4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index ed3a9d0..0ad9abc 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -730,7 +730,7 @@ public class UserGroupInformation {
*
* @param user The principal name to load from the ticket
* cache
- * @param ticketCache the path to the ticket cache file
+ * @param ticketCachePath the path to the ticket cache file
*
* @throws IOException if the kerberos login fails
*/
@@ -790,7 +790,7 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation from a Subject with Kerberos principal.
*
- * @param subject The KerberosPrincipal to use in UGI
+ * @param user The KerberosPrincipal to use in UGI
*
* @throws IOException if the kerberos login fails
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index 9356dab..eb7b949 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -17,8 +17,13 @@
*/
package org.apache.hadoop.ipc;
-import com.google.common.base.Joiner;
-import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicLong;
+
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -29,6 +34,7 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RPC.Server;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
@@ -39,12 +45,8 @@ import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.base.Joiner;
+import com.google.protobuf.BlockingService;
/**
* Benchmark for protobuf RPC.
@@ -66,7 +68,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
public int secondsToRun = 15;
private int msgSize = 1024;
public Class<? extends RpcEngine> rpcEngine =
- ProtobufRpcEngine.class;
+ WritableRpcEngine.class;
private MyOptions(String args[]) {
try {
@@ -133,7 +135,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
opts.addOption(
OptionBuilder.withLongOpt("engine").hasArg(true)
- .withArgName("protobuf")
+ .withArgName("writable|protobuf")
.withDescription("engine to use")
.create('e'));
@@ -182,6 +184,8 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
String eng = line.getOptionValue('e');
if ("protobuf".equals(eng)) {
rpcEngine = ProtobufRpcEngine.class;
+ } else if ("writable".equals(eng)) {
+ rpcEngine = WritableRpcEngine.class;
} else {
throw new ParseException("invalid engine: " + eng);
}
@@ -233,6 +237,11 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
.setInstance(service).setBindAddress(opts.host).setPort(opts.getPort())
.setNumHandlers(opts.serverThreads).setVerbose(false).build();
+ } else if (opts.rpcEngine == WritableRpcEngine.class) {
+ server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host)
+ .setPort(opts.getPort()).setNumHandlers(opts.serverThreads)
+ .setVerbose(false).build();
} else {
throw new RuntimeException("Bad engine: " + opts.rpcEngine);
}
@@ -390,6 +399,15 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
return responseProto.getMessage();
}
};
+ } else if (opts.rpcEngine == WritableRpcEngine.class) {
+ final TestProtocol proxy = RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID, addr, conf);
+ return new RpcServiceWrapper() {
+ @Override
+ public String doEcho(String msg) throws Exception {
+ return proxy.echo(msg);
+ }
+ };
} else {
throw new RuntimeException("unsupported engine: " + opts.rpcEngine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
index 10e23ba..8b419e3 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
@@ -17,28 +17,252 @@
*/
package org.apache.hadoop.ipc;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+
import org.apache.hadoop.conf.Configuration;
-import org.junit.After;
+import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
+import org.apache.hadoop.net.NetUtils;
import org.junit.Before;
+import org.junit.After;
import org.junit.Test;
+import com.google.protobuf.BlockingService;
public class TestMultipleProtocolServer extends TestRpcBase {
-
+ private static InetSocketAddress addr;
private static RPC.Server server;
- @Before
- public void setUp() throws Exception {
- super.setupConf();
+ private static Configuration conf = new Configuration();
+
+
+ @ProtocolInfo(protocolName="Foo")
+ interface Foo0 extends VersionedProtocol {
+ public static final long versionID = 0L;
+ String ping() throws IOException;
+
+ }
+
+ @ProtocolInfo(protocolName="Foo")
+ interface Foo1 extends VersionedProtocol {
+ public static final long versionID = 1L;
+ String ping() throws IOException;
+ String ping2() throws IOException;
+ }
+
+ @ProtocolInfo(protocolName="Foo")
+ interface FooUnimplemented extends VersionedProtocol {
+ public static final long versionID = 2L;
+ String ping() throws IOException;
+ }
+
+ interface Mixin extends VersionedProtocol{
+ public static final long versionID = 0L;
+ void hello() throws IOException;
+ }
+
+ interface Bar extends Mixin {
+ public static final long versionID = 0L;
+ int echo(int i) throws IOException;
+ }
+
+ class Foo0Impl implements Foo0 {
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return Foo0.versionID;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class<? extends VersionedProtocol> inter;
+ try {
+ inter = (Class<? extends VersionedProtocol>)getClass().
+ getGenericInterfaces()[0];
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ getProtocolVersion(protocol, clientVersion), inter);
+ }
- server = setupTestServer(conf, 2);
+ @Override
+ public String ping() {
+ return "Foo0";
+ }
+
}
+
+ class Foo1Impl implements Foo1 {
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return Foo1.versionID;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class<? extends VersionedProtocol> inter;
+ try {
+ inter = (Class<? extends VersionedProtocol>)getClass().
+ getGenericInterfaces()[0];
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ getProtocolVersion(protocol, clientVersion), inter);
+ }
+
+ @Override
+ public String ping() {
+ return "Foo1";
+ }
+
+ @Override
+ public String ping2() {
+ return "Foo1";
+
+ }
+
+ }
+
+
+ class BarImpl implements Bar {
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return Bar.versionID;
+ }
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ Class<? extends VersionedProtocol> inter;
+ try {
+ inter = (Class<? extends VersionedProtocol>)getClass().
+ getGenericInterfaces()[0];
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
+ return ProtocolSignature.getProtocolSignature(clientMethodsHash,
+ getProtocolVersion(protocol, clientVersion), inter);
+ }
+
+ @Override
+ public int echo(int i) {
+ return i;
+ }
+
+ @Override
+ public void hello() {
+
+
+ }
+ }
+ @Before
+ public void setUp() throws Exception {
+ // create a server with two handlers
+ server = new RPC.Builder(conf).setProtocol(Foo0.class)
+ .setInstance(new Foo0Impl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Foo1.class, new Foo1Impl());
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Bar.class, new BarImpl());
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Mixin.class, new BarImpl());
+
+
+ // Add Protobuf server
+ // Create server side implementation
+ PBServerImpl pbServerImpl = new PBServerImpl();
+ BlockingService service = TestProtobufRpcProto
+ .newReflectiveBlockingService(pbServerImpl);
+ server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService.class,
+ service);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+ }
+
@After
public void tearDown() throws Exception {
server.stop();
}
+ @Test
+ public void test1() throws IOException {
+ ProtocolProxy<?> proxy;
+ proxy = RPC.getProtocolProxy(Foo0.class, Foo0.versionID, addr, conf);
+ Foo0 foo0 = (Foo0)proxy.getProxy();
+ Assert.assertEquals("Foo0", foo0.ping());
+
+
+ proxy = RPC.getProtocolProxy(Foo1.class, Foo1.versionID, addr, conf);
+
+
+ Foo1 foo1 = (Foo1)proxy.getProxy();
+ Assert.assertEquals("Foo1", foo1.ping());
+ Assert.assertEquals("Foo1", foo1.ping());
+
+
+ proxy = RPC.getProtocolProxy(Bar.class, Foo1.versionID, addr, conf);
+
+
+ Bar bar = (Bar)proxy.getProxy();
+ Assert.assertEquals(99, bar.echo(99));
+
+ // Now test Mixin class method
+
+ Mixin mixin = bar;
+ mixin.hello();
+ }
+
+
+ // Server does not implement the FooUnimplemented version of protocol Foo.
+ // See that calls to it fail.
+ @Test(expected=IOException.class)
+ public void testNonExistingProtocol() throws IOException {
+ ProtocolProxy<?> proxy;
+ proxy = RPC.getProtocolProxy(FooUnimplemented.class,
+ FooUnimplemented.versionID, addr, conf);
+
+ FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
+ foo.ping();
+ }
+
+ /**
+ * getProtocolVersion of an unimplemented version should return highest version
+ * Similarly getProtocolSignature should work.
+ * @throws IOException
+ */
+ @Test
+ public void testNonExistingProtocol2() throws IOException {
+ ProtocolProxy<?> proxy;
+ proxy = RPC.getProtocolProxy(FooUnimplemented.class,
+ FooUnimplemented.versionID, addr, conf);
+
+ FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
+ Assert.assertEquals(Foo1.versionID,
+ foo.getProtocolVersion(RPC.getProtocolName(FooUnimplemented.class),
+ FooUnimplemented.versionID));
+ foo.getProtocolSignature(RPC.getProtocolName(FooUnimplemented.class),
+ FooUnimplemented.versionID, 0);
+ }
+
+ @Test(expected=IOException.class)
+ public void testIncorrectServerCreation() throws IOException {
+ new RPC.Builder(conf).setProtocol(Foo1.class).setInstance(new Foo0Impl())
+ .setBindAddress(ADDRESS).setPort(0).setNumHandlers(2).setVerbose(false)
+ .build();
+ }
+
// Now test a PB service - a server hosts both PB and Writable Rpcs.
@Test
public void testPBService() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
index 6d83d7d..969f728 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
@@ -26,6 +26,19 @@ import org.junit.Test;
public class TestRPCCallBenchmark {
@Test(timeout=20000)
+ public void testBenchmarkWithWritable() throws Exception {
+ int rc = ToolRunner.run(new RPCCallBenchmark(),
+ new String[] {
+ "--clientThreads", "30",
+ "--serverThreads", "30",
+ "--time", "5",
+ "--serverReaderThreads", "4",
+ "--messageSize", "1024",
+ "--engine", "writable"});
+ assertEquals(0, rc);
+ }
+
+ @Test(timeout=20000)
public void testBenchmarkWithProto() throws Exception {
int rc = ToolRunner.run(new RPCCallBenchmark(),
new String[] {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
index a06d9fd..2ac2be9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
@@ -18,20 +18,28 @@
package org.apache.hadoop.ipc;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+
+import org.junit.Assert;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
+import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
+import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
+import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
+import org.apache.hadoop.net.NetUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-
/** Unit test for supporting method-name based compatible RPCs. */
public class TestRPCCompatibility {
private static final String ADDRESS = "0.0.0.0";
@@ -41,7 +49,7 @@ public class TestRPCCompatibility {
public static final Log LOG =
LogFactory.getLog(TestRPCCompatibility.class);
-
+
private static Configuration conf = new Configuration();
public interface TestProtocol0 extends VersionedProtocol {
@@ -112,21 +120,6 @@ public class TestRPCCompatibility {
@Before
public void setUp() {
ProtocolSignature.resetCache();
-
- RPC.setProtocolEngine(conf,
- TestProtocol0.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol1.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol2.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol3.class, ProtobufRpcEngine.class);
-
- RPC.setProtocolEngine(conf,
- TestProtocol4.class, ProtobufRpcEngine.class);
}
@After
@@ -140,7 +133,117 @@ public class TestRPCCompatibility {
server = null;
}
}
+
+ @Test // old client vs new server
+ public void testVersion0ClientVersion1Server() throws Exception {
+ // create a server with two handlers
+ TestImpl1 impl = new TestImpl1();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ proxy = RPC.getProtocolProxy(
+ TestProtocol0.class, TestProtocol0.versionID, addr, conf);
+
+ TestProtocol0 proxy0 = (TestProtocol0)proxy.getProxy();
+ proxy0.ping();
+ }
+
+ @Test // old client vs new server
+ public void testVersion1ClientVersion0Server() throws Exception {
+ // create a server with two handlers
+ server = new RPC.Builder(conf).setProtocol(TestProtocol0.class)
+ .setInstance(new TestImpl0()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ proxy = RPC.getProtocolProxy(
+ TestProtocol1.class, TestProtocol1.versionID, addr, conf);
+
+ TestProtocol1 proxy1 = (TestProtocol1)proxy.getProxy();
+ proxy1.ping();
+ try {
+ proxy1.echo("hello");
+ fail("Echo should fail");
+ } catch(IOException e) {
+ }
+ }
+
+ private class Version2Client {
+ private TestProtocol2 proxy2;
+ private ProtocolProxy<TestProtocol2> serverInfo;
+
+ private Version2Client() throws IOException {
+ serverInfo = RPC.getProtocolProxy(
+ TestProtocol2.class, TestProtocol2.versionID, addr, conf);
+ proxy2 = serverInfo.getProxy();
+ }
+
+ public int echo(int value) throws IOException, NumberFormatException {
+ if (serverInfo.isMethodSupported("echo", int.class)) {
+System.out.println("echo int is supported");
+ return -value; // use version 3 echo long
+ } else { // server is version 2
+System.out.println("echo int is NOT supported");
+ return Integer.parseInt(proxy2.echo(String.valueOf(value)));
+ }
+ }
+
+ public String echo(String value) throws IOException {
+ return proxy2.echo(value);
+ }
+
+ public void ping() throws IOException {
+ proxy2.ping();
+ }
+ }
+
+ @Test // Compatible new client & old server
+ public void testVersion2ClientVersion1Server() throws Exception {
+ // create a server with two handlers
+ TestImpl1 impl = new TestImpl1();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+
+ Version2Client client = new Version2Client();
+ client.ping();
+ assertEquals("hello", client.echo("hello"));
+
+ // echo(int) is not supported by server, so returning 3
+ // This verifies that echo(int) and echo(String)'s hash codes are different
+ assertEquals(3, client.echo(3));
+ }
+
+ @Test // equal version client and server
+ public void testVersion2ClientVersion2Server() throws Exception {
+ // create a server with two handlers
+ TestImpl2 impl = new TestImpl2();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ Version2Client client = new Version2Client();
+
+ client.ping();
+ assertEquals("hello", client.echo("hello"));
+
+ // now that echo(int) is supported by the server, echo(int) should return -3
+ assertEquals(-3, client.echo(3));
+ }
+
public interface TestProtocol3 {
int echo(String value);
int echo(int value);
@@ -194,4 +297,97 @@ public class TestRPCCompatibility {
@Override
int echo(int value) throws IOException;
}
+
+ @Test
+ public void testVersionMismatch() throws IOException {
+ server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
+ .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ TestProtocol4 proxy = RPC.getProxy(TestProtocol4.class,
+ TestProtocol4.versionID, addr, conf);
+ try {
+ proxy.echo(21);
+ fail("The call must throw VersionMismatch exception");
+ } catch (RemoteException ex) {
+ Assert.assertEquals(RPC.VersionMismatch.class.getName(),
+ ex.getClassName());
+ Assert.assertTrue(ex.getErrorCode().equals(
+ RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH));
+ } catch (IOException ex) {
+ fail("Expected version mismatch but got " + ex);
+ }
+ }
+
+ @Test
+ public void testIsMethodSupported() throws IOException {
+ server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
+ .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
+ server.start();
+ addr = NetUtils.getConnectAddress(server);
+
+ TestProtocol2 proxy = RPC.getProxy(TestProtocol2.class,
+ TestProtocol2.versionID, addr, conf);
+ boolean supported = RpcClientUtil.isMethodSupported(proxy,
+ TestProtocol2.class, RPC.RpcKind.RPC_WRITABLE,
+ RPC.getProtocolVersion(TestProtocol2.class), "echo");
+ Assert.assertTrue(supported);
+ supported = RpcClientUtil.isMethodSupported(proxy,
+ TestProtocol2.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ RPC.getProtocolVersion(TestProtocol2.class), "echo");
+ Assert.assertFalse(supported);
+ }
+
+ /**
+ * Verify that ProtocolMetaInfoServerSideTranslatorPB correctly looks up
+ * the server registry to extract protocol signatures and versions.
+ */
+ @Test
+ public void testProtocolMetaInfoSSTranslatorPB() throws Exception {
+ TestImpl1 impl = new TestImpl1();
+ server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
+ .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
+ .setVerbose(false).build();
+ server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
+ server.start();
+
+ ProtocolMetaInfoServerSideTranslatorPB xlator =
+ new ProtocolMetaInfoServerSideTranslatorPB(server);
+
+ GetProtocolSignatureResponseProto resp = xlator.getProtocolSignature(
+ null,
+ createGetProtocolSigRequestProto(TestProtocol1.class,
+ RPC.RpcKind.RPC_PROTOCOL_BUFFER));
+ //No signatures should be found
+ Assert.assertEquals(0, resp.getProtocolSignatureCount());
+ resp = xlator.getProtocolSignature(
+ null,
+ createGetProtocolSigRequestProto(TestProtocol1.class,
+ RPC.RpcKind.RPC_WRITABLE));
+ Assert.assertEquals(1, resp.getProtocolSignatureCount());
+ ProtocolSignatureProto sig = resp.getProtocolSignatureList().get(0);
+ Assert.assertEquals(TestProtocol1.versionID, sig.getVersion());
+ boolean found = false;
+ int expected = ProtocolSignature.getFingerprint(TestProtocol1.class
+ .getMethod("echo", String.class));
+ for (int m : sig.getMethodsList()) {
+ if (expected == m) {
+ found = true;
+ break;
+ }
+ }
+ Assert.assertTrue(found);
+ }
+
+ private GetProtocolSignatureRequestProto createGetProtocolSigRequestProto(
+ Class<?> protocol, RPC.RpcKind rpcKind) {
+ GetProtocolSignatureRequestProto.Builder builder =
+ GetProtocolSignatureRequestProto.newBuilder();
+ builder.setProtocol(protocol.getName());
+ builder.setRpcKind(rpcKind.toString());
+ return builder.build();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
index b22f91b..5807998 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ipc;
import org.apache.hadoop.conf.Configuration;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
+import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -28,13 +30,11 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.nio.channels.ClosedByInterruptException;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
-
/**
* tests that the proxy can be interrupted
*/
-public class TestRPCWaitForProxy extends TestRpcBase {
+public class TestRPCWaitForProxy extends Assert {
+ private static final String ADDRESS = "0.0.0.0";
private static final Logger
LOG = LoggerFactory.getLogger(TestRPCWaitForProxy.class);
@@ -46,15 +46,14 @@ public class TestRPCWaitForProxy extends TestRpcBase {
*
* @throws Throwable any exception other than that which was expected
*/
- @Test(timeout = 50000)
+ @Test(timeout = 10000)
public void testWaitForProxy() throws Throwable {
RpcThread worker = new RpcThread(0);
worker.start();
worker.join();
Throwable caught = worker.getCaught();
- Throwable cause = caught.getCause();
- Assert.assertNotNull("No exception was raised", cause);
- if (!(cause instanceof ConnectException)) {
+ assertNotNull("No exception was raised", caught);
+ if (!(caught instanceof ConnectException)) {
throw caught;
}
}
@@ -70,11 +69,11 @@ public class TestRPCWaitForProxy extends TestRpcBase {
RpcThread worker = new RpcThread(100);
worker.start();
Thread.sleep(1000);
- Assert.assertTrue("worker hasn't started", worker.waitStarted);
+ assertTrue("worker hasn't started", worker.waitStarted);
worker.interrupt();
worker.join();
Throwable caught = worker.getCaught();
- Assert.assertNotNull("No exception was raised", caught);
+ assertNotNull("No exception was raised", caught);
// looking for the root cause here, which can be wrapped
// as part of the NetUtils work. Having this test look
// a the type of exception there would be brittle to improvements
@@ -83,8 +82,6 @@ public class TestRPCWaitForProxy extends TestRpcBase {
if (cause == null) {
// no inner cause, use outer exception as root cause.
cause = caught;
- } else if (cause.getCause() != null) {
- cause = cause.getCause();
}
if (!(cause instanceof InterruptedIOException)
&& !(cause instanceof ClosedByInterruptException)) {
@@ -115,16 +112,12 @@ public class TestRPCWaitForProxy extends TestRpcBase {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
connectRetries);
waitStarted = true;
-
- short invalidPort = 20;
- InetSocketAddress invalidAddress = new InetSocketAddress(ADDRESS,
- invalidPort);
- TestRpcBase.TestRpcService proxy = RPC.getProxy(
- TestRpcBase.TestRpcService.class,
- 1L, invalidAddress, conf);
- // Test echo method
- proxy.echo(null, newEchoRequest("hello"));
-
+ TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
+ TestProtocol.versionID,
+ new InetSocketAddress(ADDRESS, 20),
+ config,
+ 15000L);
+ proxy.echo("");
} catch (Throwable throwable) {
caught = throwable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index 5a8f8d0..bc604a4 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -112,8 +112,7 @@ public class TestRpcBase {
return setupTestServer(builder);
}
- protected static RPC.Server setupTestServer(
- RPC.Builder builder) throws IOException {
+ protected static RPC.Server setupTestServer(RPC.Builder builder) throws IOException {
RPC.Server server = builder.build();
server.start();
@@ -176,21 +175,17 @@ public class TestRpcBase {
public TestTokenIdentifier() {
this(new Text(), new Text());
}
-
public TestTokenIdentifier(Text tokenid) {
this(tokenid, new Text());
}
-
public TestTokenIdentifier(Text tokenid, Text realUser) {
this.tokenid = tokenid == null ? new Text() : tokenid;
this.realUser = realUser == null ? new Text() : realUser;
}
-
@Override
public Text getKind() {
return KIND_NAME;
}
-
@Override
public UserGroupInformation getUser() {
if (realUser.toString().isEmpty()) {
@@ -208,7 +203,6 @@ public class TestRpcBase {
tokenid.readFields(in);
realUser.readFields(in);
}
-
@Override
public void write(DataOutput out) throws IOException {
tokenid.write(out);
@@ -240,7 +234,7 @@ public class TestRpcBase {
@SuppressWarnings("unchecked")
@Override
public Token<TestTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
+ Collection<Token<? extends TokenIdentifier>> tokens) {
if (service == null) {
return null;
}
@@ -394,17 +388,19 @@ public class TestRpcBase {
}
@Override
- public TestProtos.UserResponseProto getAuthUser(
+ public TestProtos.AuthUserResponseProto getAuthUser(
RpcController controller, TestProtos.EmptyRequestProto request)
throws ServiceException {
- UserGroupInformation authUser;
+ UserGroupInformation authUser = null;
try {
authUser = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new ServiceException(e);
}
- return newUserResponse(authUser.getUserName());
+ return TestProtos.AuthUserResponseProto.newBuilder()
+ .setAuthUser(authUser.getUserName())
+ .build();
}
@Override
@@ -436,34 +432,6 @@ public class TestRpcBase {
return TestProtos.EmptyResponseProto.newBuilder().build();
}
-
- @Override
- public TestProtos.UserResponseProto getCurrentUser(
- RpcController controller,
- TestProtos.EmptyRequestProto request) throws ServiceException {
- String user;
- try {
- user = UserGroupInformation.getCurrentUser().toString();
- } catch (IOException e) {
- throw new ServiceException("Failed to get current user", e);
- }
-
- return newUserResponse(user);
- }
-
- @Override
- public TestProtos.UserResponseProto getServerRemoteUser(
- RpcController controller,
- TestProtos.EmptyRequestProto request) throws ServiceException {
- String serverRemoteUser = Server.getRemoteUser().toString();
- return newUserResponse(serverRemoteUser);
- }
-
- private TestProtos.UserResponseProto newUserResponse(String user) {
- return TestProtos.UserResponseProto.newBuilder()
- .setUser(user)
- .build();
- }
}
protected static TestProtos.EmptyRequestProto newEmptyRequest() {
@@ -510,4 +478,8 @@ public class TestRpcBase {
}
return null;
}
+
+ protected static String convert(TestProtos.AuthUserResponseProto response) {
+ return response.getAuthUser();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index c48ff2e..72371a7 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -45,55 +45,30 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import javax.security.auth.callback.Callback;
-import javax.security.auth.callback.CallbackHandler;
-import javax.security.auth.callback.NameCallback;
-import javax.security.auth.callback.PasswordCallback;
-import javax.security.auth.callback.UnsupportedCallbackException;
-import javax.security.sasl.AuthorizeCallback;
-import javax.security.sasl.Sasl;
-import javax.security.sasl.SaslClient;
-import javax.security.sasl.SaslException;
-import javax.security.sasl.SaslServer;
+import javax.security.auth.callback.*;
+import javax.security.sasl.*;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.security.Security;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.TimeoutException;
+import java.util.*;
+import java.util.concurrent.*;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.KERBEROS;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.SIMPLE;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.TOKEN;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.*;
+import static org.junit.Assert.*;
/** Unit tests for using Sasl over RPC. */
@RunWith(Parameterized.class)
public class TestSaslRPC extends TestRpcBase {
@Parameters
public static Collection<Object[]> data() {
- Collection<Object[]> params = new ArrayList<>();
+ Collection<Object[]> params = new ArrayList<Object[]>();
for (QualityOfProtection qop : QualityOfProtection.values()) {
params.add(new Object[]{ new QualityOfProtection[]{qop},qop, null });
}
@@ -139,7 +114,7 @@ public class TestSaslRPC extends TestRpcBase {
NONE(),
VALID(),
INVALID(),
- OTHER()
+ OTHER();
}
@BeforeClass
@@ -255,7 +230,7 @@ public class TestSaslRPC extends TestRpcBase {
final Server server = setupTestServer(conf, 5, sm);
doDigestRpc(server, sm);
} finally {
- SecurityUtil.setSecurityInfoProviders();
+ SecurityUtil.setSecurityInfoProviders(new SecurityInfo[0]);
}
}
@@ -284,7 +259,7 @@ public class TestSaslRPC extends TestRpcBase {
addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()));
- Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
@@ -321,8 +296,8 @@ public class TestSaslRPC extends TestRpcBase {
// set doPing to true
newConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
- ConnectionId remoteId = ConnectionId.getConnectionId(
- new InetSocketAddress(0), TestRpcService.class, null, 0, null, newConf);
+ ConnectionId remoteId = ConnectionId.getConnectionId(new InetSocketAddress(0),
+ TestRpcService.class, null, 0, null, newConf);
assertEquals(CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT,
remoteId.getPingInterval());
// set doPing to false
@@ -831,13 +806,13 @@ public class TestSaslRPC extends TestRpcBase {
final TestTokenSecretManager sm = new TestTokenSecretManager();
boolean useSecretManager = (serverAuth != SIMPLE);
if (enableSecretManager != null) {
- useSecretManager &= enableSecretManager;
+ useSecretManager &= enableSecretManager.booleanValue();
}
if (forceSecretManager != null) {
- useSecretManager |= forceSecretManager;
+ useSecretManager |= forceSecretManager.booleanValue();
}
final SecretManager<?> serverSm = useSecretManager ? sm : null;
-
+
Server server = serverUgi.doAs(new PrivilegedExceptionAction<Server>() {
@Override
public Server run() throws IOException {
@@ -892,13 +867,13 @@ public class TestSaslRPC extends TestRpcBase {
proxy.ping(null, newEmptyRequest());
// make sure the other side thinks we are who we said we are!!!
assertEquals(clientUgi.getUserName(),
- proxy.getAuthUser(null, newEmptyRequest()).getUser());
+ convert(proxy.getAuthUser(null, newEmptyRequest())));
AuthMethod authMethod =
convert(proxy.getAuthMethod(null, newEmptyRequest()));
// verify sasl completed with correct QOP
assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
- RPC.getConnectionIdForProxy(proxy).getSaslQop());
- return authMethod != null ? authMethod.toString() : null;
+ RPC.getConnectionIdForProxy(proxy).getSaslQop());
+ return authMethod.toString();
} catch (ServiceException se) {
if (se.getCause() instanceof RemoteException) {
throw (RemoteException) se.getCause();
@@ -923,18 +898,21 @@ public class TestSaslRPC extends TestRpcBase {
String actual) {
assertEquals(expect.toString(), actual);
}
-
- private static void assertAuthEquals(Pattern expect, String actual) {
+
+ private static void assertAuthEquals(Pattern expect,
+ String actual) {
// this allows us to see the regexp and the value it didn't match
if (!expect.matcher(actual).matches()) {
- fail(); // it failed
+ assertEquals(expect, actual); // it failed
+ } else {
+ assertTrue(true); // it matched
}
}
/*
* Class used to test overriding QOP values using SaslPropertiesResolver
*/
- static class AuthSaslPropertiesResolver extends SaslPropertiesResolver {
+ static class AuthSaslPropertiesResolver extends SaslPropertiesResolver{
@Override
public Map<String, String> getServerProperties(InetAddress address) {
@@ -943,7 +921,7 @@ public class TestSaslRPC extends TestRpcBase {
return newPropertes;
}
}
-
+
public static void main(String[] args) throws Exception {
System.out.println("Testing Kerberos authentication over RPC");
if (args.length != 2) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
index c4dbcac..50d389c 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
@@ -17,35 +17,40 @@
*/
package org.apache.hadoop.security;
-import com.google.protobuf.ServiceException;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.InetSocketAddress;
+import java.net.NetworkInterface;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Enumeration;
+
+import org.junit.Assert;
+
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtobufRpcEngine;
+import org.apache.hadoop.ipc.ProtocolSignature;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.TestRpcBase;
+import org.apache.hadoop.ipc.VersionedProtocol;
+import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
-import org.junit.Assert;
+import org.apache.hadoop.security.token.TokenInfo;
import org.junit.Before;
import org.junit.Test;
-
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.NetworkInterface;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Enumeration;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenSecretManager;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenSelector;
+import org.apache.commons.logging.*;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
/**
- * Test do as effective user.
+ *
*/
-public class TestDoAsEffectiveUser extends TestRpcBase {
+public class TestDoAsEffectiveUser {
final private static String REAL_USER_NAME = "realUser1@HADOOP.APACHE.ORG";
final private static String REAL_USER_SHORT_NAME = "realUser1";
final private static String PROXY_USER_NAME = "proxyUser";
@@ -53,8 +58,8 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
final private static String GROUP2_NAME = "group2";
final private static String[] GROUP_NAMES = new String[] { GROUP1_NAME,
GROUP2_NAME };
-
- private TestRpcService client;
+ private static final String ADDRESS = "0.0.0.0";
+ private TestProtocol proxy;
private static final Configuration masterConf = new Configuration();
@@ -77,7 +82,7 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
private void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
- ArrayList<String> ipList = new ArrayList<>();
+ ArrayList<String> ipList = new ArrayList<String>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
.getNetworkInterfaces();
while (netInterfaceList.hasMoreElements()) {
@@ -125,19 +130,50 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
curUGI.toString());
}
- private void checkRemoteUgi(final UserGroupInformation ugi,
- final Configuration conf) throws Exception {
+ @TokenInfo(TestTokenSelector.class)
+ public interface TestProtocol extends VersionedProtocol {
+ public static final long versionID = 1L;
+
+ String aMethod() throws IOException;
+ String getServerRemoteUser() throws IOException;
+ }
+
+ public class TestImpl implements TestProtocol {
+
+ @Override
+ public String aMethod() throws IOException {
+ return UserGroupInformation.getCurrentUser().toString();
+ }
+
+ @Override
+ public String getServerRemoteUser() throws IOException {
+ return Server.getRemoteUser().toString();
+ }
+
+ @Override
+ public long getProtocolVersion(String protocol, long clientVersion)
+ throws IOException {
+ return TestProtocol.versionID;
+ }
+
+ @Override
+ public ProtocolSignature getProtocolSignature(String protocol,
+ long clientVersion, int clientMethodsHash) throws IOException {
+ return new ProtocolSignature(TestProtocol.versionID, null);
+ }
+ }
+
+ private void checkRemoteUgi(final Server server,
+ final UserGroupInformation ugi, final Configuration conf)
+ throws Exception {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws ServiceException {
- client = getClient(addr, conf);
- String currentUser = client.getCurrentUser(null,
- newEmptyRequest()).getUser();
- String serverRemoteUser = client.getServerRemoteUser(null,
- newEmptyRequest()).getUser();
-
- Assert.assertEquals(ugi.toString(), currentUser);
- Assert.assertEquals(ugi.toString(), serverRemoteUser);
+ public Void run() throws IOException {
+ proxy = RPC.getProxy(
+ TestProtocol.class, TestProtocol.versionID,
+ NetUtils.getConnectAddress(server), conf);
+ Assert.assertEquals(ugi.toString(), proxy.aMethod());
+ Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser());
return null;
}
});
@@ -149,27 +185,29 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- // Set RPC engine to protobuf RPC engine
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(5).setVerbose(true).build();
refreshConf(conf);
try {
+ server.start();
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(realUserUgi, conf);
+ checkRemoteUgi(server, realUserUgi, conf);
- UserGroupInformation proxyUserUgi =
- UserGroupInformation.createProxyUserForTesting(
+ UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(proxyUserUgi, conf);
+ checkRemoteUgi(server, proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -180,25 +218,29 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(realUserUgi, conf);
+ checkRemoteUgi(server, realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation
.createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(proxyUserUgi, conf);
+ checkRemoteUgi(server, proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -214,14 +256,17 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -230,10 +275,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -241,7 +287,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -250,14 +299,17 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
final Configuration conf = new Configuration();
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 2);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -266,10 +318,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -277,7 +330,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -285,12 +341,15 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
public void testRealUserGroupNotSpecified() throws IOException {
final Configuration conf = new Configuration();
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 2);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -299,10 +358,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -310,7 +370,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -321,14 +384,17 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group3");
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
- UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 2);
+ Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(2).setVerbose(false).build();
refreshConf(conf);
try {
+ server.start();
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -337,10 +403,11 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws ServiceException {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ public String run() throws IOException {
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
}
});
@@ -348,7 +415,10 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
} catch (Exception e) {
e.printStackTrace();
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
@@ -362,17 +432,20 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
final Configuration conf = new Configuration(masterConf);
TestTokenSecretManager sm = new TestTokenSecretManager();
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
- RPC.setProtocolEngine(conf, TestRpcService.class,
- ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(conf);
- final Server server = setupTestServer(conf, 5, sm);
+ final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
+ .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
+ .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
+
+ server.start();
final UserGroupInformation current = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
-
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<>(tokenId,
+ Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
sm);
SecurityUtil.setTokenService(token, addr);
UserGroupInformation proxyUserUgi = UserGroupInformation
@@ -380,19 +453,23 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
proxyUserUgi.addToken(token);
refreshConf(conf);
-
+
String retVal = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- client = getClient(addr, conf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, conf);
+ String ret = proxy.aMethod();
+ return ret;
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
});
@@ -409,34 +486,42 @@ public class TestDoAsEffectiveUser extends TestRpcBase {
TestTokenSecretManager sm = new TestTokenSecretManager();
final Configuration newConf = new Configuration(masterConf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, newConf);
- // Set RPC engine to protobuf RPC engine
- RPC.setProtocolEngine(newConf, TestRpcService.class,
- ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(newConf);
- final Server server = setupTestServer(newConf, 5, sm);
+ final Server server = new RPC.Builder(newConf)
+ .setProtocol(TestProtocol.class).setInstance(new TestImpl())
+ .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
+ .setSecretManager(sm).build();
+
+ server.start();
final UserGroupInformation current = UserGroupInformation
.createUserForTesting(REAL_USER_NAME, GROUP_NAMES);
refreshConf(newConf);
-
+
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
+ sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
String retVal = current.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- client = getClient(addr, newConf);
- return client.getCurrentUser(null,
- newEmptyRequest()).getUser();
+ proxy = RPC.getProxy(TestProtocol.class,
+ TestProtocol.versionID, addr, newConf);
+ String ret = proxy.aMethod();
+ return ret;
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- stop(server, client);
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index e45d70d..b3ea5f2 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -20,7 +20,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -29,11 +28,7 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Before;
-import org.junit.BeforeClass;
-import org.junit.Test;
+import org.junit.*;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -55,22 +50,9 @@ import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
-import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
-import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt;
-import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
-import static org.apache.hadoop.test.MetricsAsserts.getDoubleGauge;
-import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
-import static org.junit.Assert.assertArrayEquals;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.assertNotEquals;
-import static org.junit.Assert.assertNotNull;
-import static org.junit.Assert.assertNotSame;
-import static org.junit.Assert.assertNull;
-import static org.junit.Assert.assertSame;
-import static org.junit.Assert.assertTrue;
-import static org.junit.Assert.fail;
+import static org.apache.hadoop.ipc.TestSaslRPC.*;
+import static org.apache.hadoop.test.MetricsAsserts.*;
+import static org.junit.Assert.*;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -127,7 +109,7 @@ public class TestUserGroupInformation {
UserGroupInformation.setLoginUser(null);
}
- @Test(timeout = 30000)
+ @Test (timeout = 30000)
public void testSimpleLogin() throws IOException {
tryLoginAuthenticationMethod(AuthenticationMethod.SIMPLE, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 6411f97..99cd93d 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -88,6 +88,6 @@ message AuthMethodResponseProto {
required string mechanismName = 2;
}
-message UserResponseProto {
- required string user = 1;
+message AuthUserResponseProto {
+ required string authUser = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 06f6c4f..3292115 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -40,11 +40,9 @@ service TestProtobufRpcProto {
rpc exchange(ExchangeRequestProto) returns (ExchangeResponseProto);
rpc sleep(SleepRequestProto) returns (EmptyResponseProto);
rpc getAuthMethod(EmptyRequestProto) returns (AuthMethodResponseProto);
- rpc getAuthUser(EmptyRequestProto) returns (UserResponseProto);
+ rpc getAuthUser(EmptyRequestProto) returns (AuthUserResponseProto);
rpc echoPostponed(EchoRequestProto) returns (EchoResponseProto);
rpc sendPostponed(EmptyRequestProto) returns (EmptyResponseProto);
- rpc getCurrentUser(EmptyRequestProto) returns (UserResponseProto);
- rpc getServerRemoteUser(EmptyRequestProto) returns (UserResponseProto);
}
service TestProtobufRpc2Proto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 57f7cb1..6b52949 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -168,6 +168,7 @@ import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
+import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
import org.apache.hadoop.net.Node;
@@ -316,6 +317,8 @@ public class NameNodeRpcServer implements NamenodeProtocols {
new TraceAdminProtocolServerSideTranslatorPB(this);
BlockingService traceAdminService = TraceAdminService
.newReflectiveBlockingService(traceAdminXlator);
+
+ WritableRpcEngine.ensureInitialized();
InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
if (serviceRpcAddr != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
new file mode 100644
index 0000000..0b7ee33
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
@@ -0,0 +1,119 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.security;
+
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
+import static org.mockito.Mockito.mock;
+
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.protocol.ClientProtocol;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.Client;
+import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.Server;
+import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.SaslInputStream;
+import org.apache.hadoop.security.SaslRpcClient;
+import org.apache.hadoop.security.SaslRpcServer;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.log4j.Level;
+import org.junit.Test;
+
+/** Unit tests for using Delegation Token over RPC. */
+public class TestClientProtocolWithDelegationToken {
+ private static final String ADDRESS = "0.0.0.0";
+
+ public static final Log LOG = LogFactory
+ .getLog(TestClientProtocolWithDelegationToken.class);
+
+ private static final Configuration conf;
+ static {
+ conf = new Configuration();
+ conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ UserGroupInformation.setConfiguration(conf);
+ }
+
+ static {
+ GenericTestUtils.setLogLevel(Client.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(Server.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(SaslRpcClient.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(SaslRpcServer.LOG, Level.ALL);
+ GenericTestUtils.setLogLevel(SaslInputStream.LOG, Level.ALL);
+ }
+
+ @Test
+ public void testDelegationTokenRpc() throws Exception {
+ ClientProtocol mockNN = mock(ClientProtocol.class);
+ FSNamesystem mockNameSys = mock(FSNamesystem.class);
+
+ DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
+ DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
+ 3600000, mockNameSys);
+ sm.startThreads();
+ final Server server = new RPC.Builder(conf)
+ .setProtocol(ClientProtocol.class).setInstance(mockNN)
+ .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
+ .setSecretManager(sm).build();
+
+ server.start();
+
+ final UserGroupInformation current = UserGroupInformation.getCurrentUser();
+ final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+ String user = current.getUserName();
+ Text owner = new Text(user);
+ DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null);
+ Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
+ dtId, sm);
+ SecurityUtil.setTokenService(token, addr);
+ LOG.info("Service for token is " + token.getService());
+ current.addToken(token);
+ current.doAs(new PrivilegedExceptionAction<Object>() {
+ @Override
+ public Object run() throws Exception {
+ ClientProtocol proxy = null;
+ try {
+ proxy = RPC.getProxy(ClientProtocol.class,
+ ClientProtocol.versionID, addr, conf);
+ proxy.getServerDefaults();
+ } finally {
+ server.stop();
+ if (proxy != null) {
+ RPC.stopProxy(proxy);
+ }
+ }
+ return null;
+ }
+ });
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d355573f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 729af0a..3fef5e2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
+import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -97,6 +98,8 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
BlockingService refreshHSAdminProtocolService = HSAdminRefreshProtocolService
.newReflectiveBlockingService(refreshHSAdminProtocolXlator);
+ WritableRpcEngine.ensureInitialized();
+
clientRpcAddress = conf.getSocketAddr(
JHAdminConfig.MR_HISTORY_BIND_HOST,
JHAdminConfig.JHS_ADMIN_ADDRESS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/48] hadoop git commit: HDFS-9038. DFS reserved space is
erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Posted by aw...@apache.org.
HDFS-9038. DFS reserved space is erroneously counted towards non-DFS used. (Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5f23abfa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5f23abfa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5f23abfa
Branch: refs/heads/HADOOP-13341
Commit: 5f23abfa30ea29a5474513c463b4d462c0e824ee
Parents: f6c0b75
Author: Arpit Agarwal <ar...@apache.org>
Authored: Tue Sep 6 13:37:21 2016 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Tue Sep 6 13:37:21 2016 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/protocol/DatanodeInfo.java | 33 +++++++++++++---
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 41 +++++++++++++-------
.../hdfs/server/protocol/StorageReport.java | 10 ++++-
.../src/main/proto/hdfs.proto | 2 +
.../blockmanagement/DatanodeDescriptor.java | 3 ++
.../server/blockmanagement/DatanodeStats.java | 6 ++-
.../blockmanagement/DatanodeStorageInfo.java | 8 +++-
.../datanode/fsdataset/impl/FsDatasetImpl.java | 3 +-
.../datanode/fsdataset/impl/FsVolumeImpl.java | 37 ++++++++++++++++--
.../hadoop/hdfs/protocolPB/TestPBHelper.java | 24 ++++++++++++
.../blockmanagement/BlockManagerTestUtil.java | 2 +-
.../blockmanagement/TestBlockManager.java | 2 +-
.../server/datanode/SimulatedFSDataset.java | 2 +-
.../extdataset/ExternalDatasetImpl.java | 2 +-
.../fsdataset/impl/TestFsVolumeList.java | 40 +++++++++++++++++++
.../server/namenode/NNThroughputBenchmark.java | 4 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 2 +-
.../namenode/TestNamenodeCapacityReport.java | 36 ++++++++++++++---
.../namenode/metrics/TestNameNodeMetrics.java | 4 +-
19 files changed, 219 insertions(+), 42 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
index cd32a53..e9ee8b9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/DatanodeInfo.java
@@ -43,6 +43,7 @@ import static org.apache.hadoop.hdfs.DFSUtilClient.percent2String;
public class DatanodeInfo extends DatanodeID implements Node {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private long remaining;
private long blockPoolUsed;
private long cacheCapacity;
@@ -89,6 +90,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(from);
this.capacity = from.getCapacity();
this.dfsUsed = from.getDfsUsed();
+ this.nonDfsUsed = from.getNonDfsUsed();
this.remaining = from.getRemaining();
this.blockPoolUsed = from.getBlockPoolUsed();
this.cacheCapacity = from.getCacheCapacity();
@@ -105,6 +107,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
super(nodeID);
this.capacity = 0L;
this.dfsUsed = 0L;
+ this.nonDfsUsed = 0L;
this.remaining = 0L;
this.blockPoolUsed = 0L;
this.cacheCapacity = 0L;
@@ -158,10 +161,26 @@ public class DatanodeInfo extends DatanodeID implements Node {
final int xceiverCount, final String networkLocation,
final AdminStates adminState,
final String upgradeDomain) {
- super(ipAddr, hostName, datanodeUuid, xferPort, infoPort,
- infoSecurePort, ipcPort);
+ this(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort, capacity, dfsUsed, 0L, remaining, blockPoolUsed,
+ cacheCapacity, cacheUsed, lastUpdate, lastUpdateMonotonic,
+ xceiverCount, networkLocation, adminState, upgradeDomain);
+ }
+
+ /** Constructor. */
+ public DatanodeInfo(final String ipAddr, final String hostName,
+ final String datanodeUuid, final int xferPort, final int infoPort,
+ final int infoSecurePort, final int ipcPort, final long capacity,
+ final long dfsUsed, final long nonDfsUsed, final long remaining,
+ final long blockPoolUsed, final long cacheCapacity, final long cacheUsed,
+ final long lastUpdate, final long lastUpdateMonotonic,
+ final int xceiverCount, final String networkLocation,
+ final AdminStates adminState, final String upgradeDomain) {
+ super(ipAddr, hostName, datanodeUuid, xferPort, infoPort, infoSecurePort,
+ ipcPort);
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = blockPoolUsed;
this.cacheCapacity = cacheCapacity;
@@ -174,7 +193,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.upgradeDomain = upgradeDomain;
}
- /** Network location name */
+ /** Network location name. */
@Override
public String getName() {
return getXferAddr();
@@ -191,8 +210,7 @@ public class DatanodeInfo extends DatanodeID implements Node {
/** The used space by the data node. */
public long getNonDfsUsed() {
- long nonDFSUsed = capacity - dfsUsed - remaining;
- return nonDFSUsed < 0 ? 0 : nonDFSUsed;
+ return nonDfsUsed;
}
/** The used space by the data node as percentage of present capacity */
@@ -282,6 +300,11 @@ public class DatanodeInfo extends DatanodeID implements Node {
this.dfsUsed = dfsUsed;
}
+ /** Sets the nondfs-used space for the datanode. */
+ public void setNonDfsUsed(long nonDfsUsed) {
+ this.nonDfsUsed = nonDfsUsed;
+ }
+
/** Sets raw free space. */
public void setRemaining(long remaining) {
this.remaining = remaining;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 7df68e7..246b5a5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -289,6 +289,7 @@ public class PBHelperClient {
.setId(convert((DatanodeID) info))
.setCapacity(info.getCapacity())
.setDfsUsed(info.getDfsUsed())
+ .setNonDfsUsed(info.getNonDfsUsed())
.setRemaining(info.getRemaining())
.setBlockPoolUsed(info.getBlockPoolUsed())
.setCacheCapacity(info.getCacheCapacity())
@@ -581,15 +582,24 @@ public class PBHelperClient {
}
static public DatanodeInfo convert(DatanodeInfoProto di) {
- if (di == null) return null;
- return new DatanodeInfo(
- convert(di.getId()),
- di.hasLocation() ? di.getLocation() : null,
- di.getCapacity(), di.getDfsUsed(), di.getRemaining(),
- di.getBlockPoolUsed(), di.getCacheCapacity(), di.getCacheUsed(),
- di.getLastUpdate(), di.getLastUpdateMonotonic(),
- di.getXceiverCount(), convert(di.getAdminState()),
+ if (di == null) {
+ return null;
+ }
+ DatanodeInfo dinfo = new DatanodeInfo(convert(di.getId()),
+ di.hasLocation() ? di.getLocation() : null, di.getCapacity(),
+ di.getDfsUsed(), di.getRemaining(), di.getBlockPoolUsed(),
+ di.getCacheCapacity(), di.getCacheUsed(), di.getLastUpdate(),
+ di.getLastUpdateMonotonic(), di.getXceiverCount(),
+ convert(di.getAdminState()),
di.hasUpgradeDomain() ? di.getUpgradeDomain() : null);
+ if (di.hasNonDfsUsed()) {
+ dinfo.setNonDfsUsed(di.getNonDfsUsed());
+ } else {
+ // use the legacy way for older datanodes
+ long nonDFSUsed = di.getCapacity() - di.getDfsUsed() - di.getRemaining();
+ dinfo.setNonDfsUsed(nonDFSUsed < 0 ? 0 : nonDFSUsed);
+ }
+ return dinfo;
}
public static StorageType[] convertStorageTypes(
@@ -1556,12 +1566,12 @@ public class PBHelperClient {
}
public static StorageReport convert(StorageReportProto p) {
- return new StorageReport(
- p.hasStorage() ?
- convert(p.getStorage()) :
- new DatanodeStorage(p.getStorageUuid()),
- p.getFailed(), p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
- p.getBlockPoolUsed());
+ long nonDfsUsed = p.hasNonDfsUsed() ? p.getNonDfsUsed() : p.getCapacity()
+ - p.getDfsUsed() - p.getRemaining();
+ return new StorageReport(p.hasStorage() ? convert(p.getStorage())
+ : new DatanodeStorage(p.getStorageUuid()), p.getFailed(),
+ p.getCapacity(), p.getDfsUsed(), p.getRemaining(),
+ p.getBlockPoolUsed(), nonDfsUsed);
}
public static DatanodeStorage convert(DatanodeStorageProto s) {
@@ -2136,7 +2146,8 @@ public class PBHelperClient {
.setBlockPoolUsed(r.getBlockPoolUsed()).setCapacity(r.getCapacity())
.setDfsUsed(r.getDfsUsed()).setRemaining(r.getRemaining())
.setStorageUuid(r.getStorage().getStorageID())
- .setStorage(convert(r.getStorage()));
+ .setStorage(convert(r.getStorage()))
+ .setNonDfsUsed(r.getNonDfsUsed());
return builder.build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
index 3fbc424..7042b58 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/server/protocol/StorageReport.java
@@ -25,17 +25,19 @@ public class StorageReport {
private final boolean failed;
private final long capacity;
private final long dfsUsed;
+ private final long nonDfsUsed;
private final long remaining;
private final long blockPoolUsed;
public static final StorageReport[] EMPTY_ARRAY = {};
- public StorageReport(DatanodeStorage storage, boolean failed,
- long capacity, long dfsUsed, long remaining, long bpUsed) {
+ public StorageReport(DatanodeStorage storage, boolean failed, long capacity,
+ long dfsUsed, long remaining, long bpUsed, long nonDfsUsed) {
this.storage = storage;
this.failed = failed;
this.capacity = capacity;
this.dfsUsed = dfsUsed;
+ this.nonDfsUsed = nonDfsUsed;
this.remaining = remaining;
this.blockPoolUsed = bpUsed;
}
@@ -56,6 +58,10 @@ public class StorageReport {
return dfsUsed;
}
+ public long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
public long getRemaining() {
return remaining;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7de8550..1141fcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -101,6 +101,7 @@ message DatanodeInfoProto {
optional uint64 cacheUsed = 12 [default = 0];
optional uint64 lastUpdateMonotonic = 13 [default = 0];
optional string upgradeDomain = 14;
+ optional uint64 nonDfsUsed = 15;
}
/**
@@ -125,6 +126,7 @@ message StorageReportProto {
optional uint64 remaining = 5 [ default = 0 ];
optional uint64 blockPoolUsed = 6 [ default = 0 ];
optional DatanodeStorageProto storage = 7; // supersedes StorageUuid
+ optional uint64 nonDfsUsed = 8;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 1646129..c74d7c5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -417,6 +417,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
long totalRemaining = 0;
long totalBlockPoolUsed = 0;
long totalDfsUsed = 0;
+ long totalNonDfsUsed = 0;
Set<DatanodeStorageInfo> failedStorageInfos = null;
// Decide if we should check for any missing StorageReport and mark it as
@@ -477,6 +478,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
totalRemaining += report.getRemaining();
totalBlockPoolUsed += report.getBlockPoolUsed();
totalDfsUsed += report.getDfsUsed();
+ totalNonDfsUsed += report.getNonDfsUsed();
}
rollBlocksScheduled(getLastUpdateMonotonic());
@@ -485,6 +487,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
setRemaining(totalRemaining);
setBlockPoolUsed(totalBlockPoolUsed);
setDfsUsed(totalDfsUsed);
+ setNonDfsUsed(totalNonDfsUsed);
if (checkFailedStorages) {
updateFailedStorage(failedStorageInfos);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
index 0d4e235..8386b27 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStats.java
@@ -35,6 +35,7 @@ class DatanodeStats {
private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
private long capacityTotal = 0L;
private long capacityUsed = 0L;
+ private long capacityUsedNonDfs = 0L;
private long capacityRemaining = 0L;
private long blockPoolUsed = 0L;
private int xceiverCount = 0;
@@ -49,6 +50,7 @@ class DatanodeStats {
xceiverCount += node.getXceiverCount();
if (node.isInService()) {
capacityUsed += node.getDfsUsed();
+ capacityUsedNonDfs += node.getNonDfsUsed();
blockPoolUsed += node.getBlockPoolUsed();
nodesInService++;
nodesInServiceXceiverCount += node.getXceiverCount();
@@ -77,6 +79,7 @@ class DatanodeStats {
xceiverCount -= node.getXceiverCount();
if (node.isInService()) {
capacityUsed -= node.getDfsUsed();
+ capacityUsedNonDfs -= node.getNonDfsUsed();
blockPoolUsed -= node.getBlockPoolUsed();
nodesInService--;
nodesInServiceXceiverCount -= node.getXceiverCount();
@@ -159,8 +162,7 @@ class DatanodeStats {
}
synchronized long getCapacityUsedNonDFS() {
- final long nonDFSUsed = capacityTotal - capacityRemaining - capacityUsed;
- return nonDFSUsed < 0L? 0L : nonDFSUsed;
+ return capacityUsedNonDfs;
}
synchronized float getCapacityUsedPercent() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 843a8d5..d98a2c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -93,6 +93,7 @@ public class DatanodeStorageInfo {
private long capacity;
private long dfsUsed;
+ private long nonDfsUsed;
private volatile long remaining;
private long blockPoolUsed;
@@ -202,6 +203,10 @@ public class DatanodeStorageInfo {
return dfsUsed;
}
+ long getNonDfsUsed() {
+ return nonDfsUsed;
+ }
+
long getRemaining() {
return remaining;
}
@@ -283,6 +288,7 @@ public class DatanodeStorageInfo {
void updateState(StorageReport r) {
capacity = r.getCapacity();
dfsUsed = r.getDfsUsed();
+ nonDfsUsed = r.getNonDfsUsed();
remaining = r.getRemaining();
blockPoolUsed = r.getBlockPoolUsed();
}
@@ -332,7 +338,7 @@ public class DatanodeStorageInfo {
StorageReport toStorageReport() {
return new StorageReport(
new DatanodeStorage(storageID, state, storageType),
- false, capacity, dfsUsed, remaining, blockPoolUsed);
+ false, capacity, dfsUsed, remaining, blockPoolUsed, nonDfsUsed);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index e0d2baf..ffd2f8a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -164,7 +164,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
volume.getCapacity(),
volume.getDfsUsed(),
volume.getAvailable(),
- volume.getBlockPoolUsed(bpid));
+ volume.getBlockPoolUsed(bpid),
+ volume.getNonDfsUsed());
reports.add(sr);
} catch (ClosedChannelException e) {
continue;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
index 59c9ed4..afcc5dd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsVolumeImpl.java
@@ -385,15 +385,46 @@ public class FsVolumeImpl implements FsVolumeSpi {
*/
@Override
public long getAvailable() throws IOException {
- long remaining = getCapacity() - getDfsUsed() - reservedForReplicas.get();
- long available = usage.getAvailable() - reserved
- - reservedForReplicas.get();
+ long remaining = getCapacity() - getDfsUsed() - getReservedForReplicas();
+ long available = usage.getAvailable() - getRemainingReserved()
+ - getReservedForReplicas();
if (remaining > available) {
remaining = available;
}
return (remaining > 0) ? remaining : 0;
}
+ long getActualNonDfsUsed() throws IOException {
+ return usage.getUsed() - getDfsUsed();
+ }
+
+ private long getRemainingReserved() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return reserved - actualNonDfsUsed;
+ }
+ return 0L;
+ }
+
+ /**
+ * Unplanned Non-DFS usage, i.e. Extra usage beyond reserved.
+ *
+ * @return
+ * @throws IOException
+ */
+ public long getNonDfsUsed() throws IOException {
+ long actualNonDfsUsed = getActualNonDfsUsed();
+ if (actualNonDfsUsed < reserved) {
+ return 0L;
+ }
+ return actualNonDfsUsed - reserved;
+ }
+
+ @VisibleForTesting
+ long getDfAvailable() {
+ return usage.getAvailable();
+ }
+
@VisibleForTesting
public long getReservedForReplicas() {
return reservedForReplicas.get();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
index 45cd4ea..af17756 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/protocolPB/TestPBHelper.java
@@ -720,6 +720,28 @@ public class TestPBHelper {
assertBlockECRecoveryInfoEquals(blkECRecoveryInfo1, iterator.next());
}
+ @Test
+ public void testDataNodeInfoPBHelper() {
+ DatanodeID id = DFSTestUtil.getLocalDatanodeID();
+ DatanodeInfo dnInfos0 = new DatanodeInfo(id);
+ dnInfos0.setCapacity(3500L);
+ dnInfos0.setDfsUsed(1000L);
+ dnInfos0.setNonDfsUsed(2000L);
+ dnInfos0.setRemaining(500L);
+ HdfsProtos.DatanodeInfoProto dnproto = PBHelperClient.convert(dnInfos0);
+ DatanodeInfo dnInfos1 = PBHelperClient.convert(dnproto);
+ compare(dnInfos0, dnInfos1);
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos1.getNonDfsUsed());
+
+ //Testing without nonDfs field
+ HdfsProtos.DatanodeInfoProto.Builder b =
+ HdfsProtos.DatanodeInfoProto.newBuilder();
+ b.setId(PBHelperClient.convert(id)).setCapacity(3500L).setDfsUsed(1000L)
+ .setRemaining(500L);
+ DatanodeInfo dnInfos3 = PBHelperClient.convert(b.build());
+ assertEquals(dnInfos0.getNonDfsUsed(), dnInfos3.getNonDfsUsed());
+ }
+
private void assertBlockECRecoveryInfoEquals(
BlockECReconstructionInfo blkECRecoveryInfo1,
BlockECReconstructionInfo blkECRecoveryInfo2) {
@@ -768,4 +790,6 @@ public class TestPBHelper {
compare(dnInfos1[i], dnInfos2[i]);
}
}
+
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
index 1b565ee..5fb3b25 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManagerTestUtil.java
@@ -294,7 +294,7 @@ public class BlockManagerTestUtil {
StorageReport report = new StorageReport(
dns ,false, storage.getCapacity(),
storage.getDfsUsed(), storage.getRemaining(),
- storage.getBlockPoolUsed());
+ storage.getBlockPoolUsed(), 0);
reports.add(report);
}
return reports.toArray(StorageReport.EMPTY_ARRAY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
index 1f58f99..942569a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockManager.java
@@ -1200,7 +1200,7 @@ public class TestBlockManager {
StorageReport report = new StorageReport(
dns, true, storageInfo.getCapacity(),
storageInfo.getDfsUsed(), storageInfo.getRemaining(),
- storageInfo.getBlockPoolUsed());
+ storageInfo.getBlockPoolUsed(), 0L);
reports.add(report);
break;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
index c0c76ad..a082fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/SimulatedFSDataset.java
@@ -441,7 +441,7 @@ public class SimulatedFSDataset implements FsDatasetSpi<FsVolumeSpi> {
synchronized StorageReport getStorageReport(String bpid) {
return new StorageReport(dnStorage,
false, getCapacity(), getUsed(), getFree(),
- map.get(bpid).getUsed());
+ map.get(bpid).getUsed(), 0L);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
index b8564bb..fcd960a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/extdataset/ExternalDatasetImpl.java
@@ -72,7 +72,7 @@ public class ExternalDatasetImpl implements FsDatasetSpi<ExternalVolumeImpl> {
@Override
public StorageReport[] getStorageReports(String bpid) throws IOException {
StorageReport[] result = new StorageReport[1];
- result[0] = new StorageReport(storage, false, 0, 0, 0, 0);
+ result[0] = new StorageReport(storage, false, 0, 0, 0, 0, 0);
return result;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
index 796d249..3d4c38c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsVolumeList.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.StringUtils;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import java.io.File;
import java.io.IOException;
@@ -177,4 +178,43 @@ public class TestFsVolumeList {
conf, StorageType.DEFAULT);
assertEquals("", 100L, volume4.getReserved());
}
+
+ @Test
+ public void testNonDfsUsedMetricForVolume() throws Exception {
+ File volDir = new File(baseDir, "volume-0");
+ volDir.mkdirs();
+ /*
+ * Lets have the example.
+ * Capacity - 1000
+ * Reserved - 100
+ * DfsUsed - 200
+ * Actual Non-DfsUsed - 300 -->(expected)
+ * ReservedForReplicas - 50
+ */
+ long diskCapacity = 1000L;
+ long duReserved = 100L;
+ long dfsUsage = 200L;
+ long actualNonDfsUsage = 300L;
+ long reservedForReplicas = 50L;
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_DU_RESERVED_KEY, duReserved);
+ FsVolumeImpl volume = new FsVolumeImpl(dataset, "storage-id", volDir, conf,
+ StorageType.DEFAULT);
+ FsVolumeImpl spyVolume = Mockito.spy(volume);
+ // Set Capacity for testing
+ long testCapacity = diskCapacity - duReserved;
+ spyVolume.setCapacityForTesting(testCapacity);
+ // Mock volume.getDfAvailable()
+ long dfAvailable = diskCapacity - dfsUsage - actualNonDfsUsage;
+ Mockito.doReturn(dfAvailable).when(spyVolume).getDfAvailable();
+ // Mock dfsUsage
+ Mockito.doReturn(dfsUsage).when(spyVolume).getDfsUsed();
+ // Mock reservedForReplcas
+ Mockito.doReturn(reservedForReplicas).when(spyVolume)
+ .getReservedForReplicas();
+ Mockito.doReturn(actualNonDfsUsage).when(spyVolume)
+ .getActualNonDfsUsed();
+ long expectedNonDfsUsage =
+ actualNonDfsUsage - duReserved;
+ assertEquals(expectedNonDfsUsage, spyVolume.getNonDfsUsed());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index be2a678..ad336ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -949,7 +949,7 @@ public class NNThroughputBenchmark implements Tool {
// register datanode
// TODO:FEDERATION currently a single block pool is supported
StorageReport[] rep = { new StorageReport(storage, false,
- DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0L) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration, rep,
0L, 0L, 0, 0, 0, null, true).getCommands();
if(cmds != null) {
@@ -998,7 +998,7 @@ public class NNThroughputBenchmark implements Tool {
int replicateBlocks() throws IOException {
// register datanode
StorageReport[] rep = { new StorageReport(storage,
- false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED) };
+ false, DF_CAPACITY, DF_USED, DF_CAPACITY - DF_USED, DF_USED, 0) };
DatanodeCommand[] cmds = dataNodeProto.sendHeartbeat(dnRegistration,
rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
if (cmds != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index 341a149..14b14b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -128,7 +128,7 @@ public class TestDeadDatanode {
// that asks datanode to register again
StorageReport[] rep = { new StorageReport(
new DatanodeStorage(reg.getDatanodeUuid()),
- false, 0, 0, 0, 0) };
+ false, 0, 0, 0, 0, 0) };
DatanodeCommand[] cmd =
dnp.sendHeartbeat(reg, rep, 0L, 0L, 0, 0, 0, null, true).getCommands();
assertEquals(1, cmd.length);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
index 99323f5..38ec9f8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestNamenodeCapacityReport.java
@@ -71,7 +71,6 @@ public class TestNamenodeCapacityReport {
try {
cluster = new MiniDFSCluster.Builder(conf).build();
cluster.waitActive();
-
final FSNamesystem namesystem = cluster.getNamesystem();
final DatanodeManager dm = cluster.getNamesystem().getBlockManager(
).getDatanodeManager();
@@ -100,8 +99,9 @@ public class TestNamenodeCapacityReport {
+ " used " + used + " non DFS used " + nonDFSUsed
+ " remaining " + remaining + " perentUsed " + percentUsed
+ " percentRemaining " + percentRemaining);
-
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
configCapacity));
assertTrue(percentRemaining == DFSUtilClient.getPercentRemaining(
@@ -148,7 +148,9 @@ public class TestNamenodeCapacityReport {
assertTrue(configCapacity == diskCapacity - reserved);
// Ensure new total capacity reported excludes the reserved space
- assertTrue(configCapacity == (used + remaining + nonDFSUsed));
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assertTrue(configCapacity >= (used + remaining + nonDFSUsed));
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentUsed == DFSUtilClient.getPercentUsed(used,
@@ -160,9 +162,33 @@ public class TestNamenodeCapacityReport {
// Ensure percent used is calculated based on used and present capacity
assertTrue(percentRemaining == ((float)remaining * 100.0f)/(float)configCapacity);
+
+ //Adding testcase for non-dfs used where we need to consider
+ // reserved replica also.
+ final int fileCount = 5;
+ final DistributedFileSystem fs = cluster.getFileSystem();
+ // create streams and hsync to force datastreamers to start
+ DFSOutputStream[] streams = new DFSOutputStream[fileCount];
+ for (int i=0; i < fileCount; i++) {
+ streams[i] = (DFSOutputStream)fs.create(new Path("/f"+i))
+ .getWrappedStream();
+ streams[i].write("1".getBytes());
+ streams[i].hsync();
+ }
+ triggerHeartbeats(cluster.getDataNodes());
+ assertTrue(configCapacity > (namesystem.getCapacityUsed() + namesystem
+ .getCapacityRemaining() + namesystem.getNonDfsUsedSpace()));
+ // There is a chance that nonDFS usage might have slightly due to
+ // testlogs, So assume 1MB other files used within this gap
+ assertTrue(
+ (namesystem.getCapacityUsed() + namesystem.getCapacityRemaining()
+ + namesystem.getNonDfsUsedSpace() + fileCount * fs
+ .getDefaultBlockSize()) - configCapacity < 1 * 1024);
}
finally {
- if (cluster != null) {cluster.shutdown();}
+ if (cluster != null) {
+ cluster.shutdown();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5f23abfa/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
index c4a9837..9e95650 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/metrics/TestNameNodeMetrics.java
@@ -166,7 +166,9 @@ public class TestNameNodeMetrics {
MetricsAsserts.getLongGauge("CapacityRemaining", rb);
long capacityUsedNonDFS =
MetricsAsserts.getLongGauge("CapacityUsedNonDFS", rb);
- assert(capacityUsed + capacityRemaining + capacityUsedNonDFS ==
+ // There will be 5% space reserved in ext filesystem which is not
+ // considered.
+ assert (capacityUsed + capacityRemaining + capacityUsedNonDFS <=
capacityTotal);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/48] hadoop git commit: HADOOP-13564. modify mapred to use
hadoop_subcommand_opts
Posted by aw...@apache.org.
HADOOP-13564. modify mapred to use hadoop_subcommand_opts
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/930e03b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/930e03b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/930e03b7
Branch: refs/heads/HADOOP-13341
Commit: 930e03b71e779772bb75e15ed5503c7392a68fdd
Parents: 59b59ca
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 15:11:24 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/bin/mapred | 11 +++++++----
hadoop-mapreduce-project/bin/mapred-config.sh | 10 ++++++----
hadoop-mapreduce-project/conf/mapred-env.sh | 4 ++--
3 files changed, 15 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/930e03b7/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index c566d5a..fe7c56a 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -69,8 +69,6 @@ function mapredcmd_case
historyserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.JobHistoryServer
- hadoop_debug "Appending HADOOP_JOB_HISTORYSERVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOB_HISTORYSERVER_OPTS}"
if [ -n "${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}" ]; then
# shellcheck disable=SC2034
HADOOP_HEAPSIZE_MAX="${HADOOP_JOB_HISTORYSERVER_HEAPSIZE}"
@@ -144,13 +142,18 @@ fi
hadoop_add_client_opts
-if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
- hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
+if [[ ${HADOOP_WORKER_MODE} = true ]]; then
+ hadoop_common_worker_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
exit $?
fi
+hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+
+ hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
hadoop_verify_secure_prereq
hadoop_setup_secure_service
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/930e03b7/hadoop-mapreduce-project/bin/mapred-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred-config.sh b/hadoop-mapreduce-project/bin/mapred-config.sh
old mode 100644
new mode 100755
index a989792..68d3463d
--- a/hadoop-mapreduce-project/bin/mapred-config.sh
+++ b/hadoop-mapreduce-project/bin/mapred-config.sh
@@ -26,7 +26,7 @@ function hadoop_subproject_init
export HADOOP_MAPRED_ENV_PROCESSED=true
fi
fi
-
+
# at some point in time, someone thought it would be a good idea to
# create separate vars for every subproject. *sigh*
# let's perform some overrides and setup some defaults for bw compat
@@ -38,15 +38,17 @@ function hadoop_subproject_init
hadoop_deprecate_envvar HADOOP_MAPRED_LOG_DIR HADOOP_LOG_DIR
hadoop_deprecate_envvar HADOOP_MAPRED_LOGFILE HADOOP_LOGFILE
-
+
hadoop_deprecate_envvar HADOOP_MAPRED_NICENESS HADOOP_NICENESS
-
+
hadoop_deprecate_envvar HADOOP_MAPRED_STOP_TIMEOUT HADOOP_STOP_TIMEOUT
-
+
hadoop_deprecate_envvar HADOOP_MAPRED_PID_DIR HADOOP_PID_DIR
hadoop_deprecate_envvar HADOOP_MAPRED_ROOT_LOGGER HADOOP_ROOT_LOGGER
+ hadoop_deprecate_envvar HADOOP_JOB_HISTORY_OPTS MAPRED_HISTORYSERVER_OPTS
+
HADOOP_MAPRED_HOME="${HADOOP_MAPRED_HOME:-$HADOOP_HOME}"
hadoop_deprecate_envvar HADOOP_MAPRED_IDENT_STRING HADOOP_IDENT_STRING
http://git-wip-us.apache.org/repos/asf/hadoop/blob/930e03b7/hadoop-mapreduce-project/conf/mapred-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/conf/mapred-env.sh b/hadoop-mapreduce-project/conf/mapred-env.sh
index bbe4a49..53bc296 100644
--- a/hadoop-mapreduce-project/conf/mapred-env.sh
+++ b/hadoop-mapreduce-project/conf/mapred-env.sh
@@ -31,14 +31,14 @@
# Specify the max heapsize for the JobHistoryServer. If no units are
# given, it will be assumed to be in MB.
# This value will be overridden by an Xmx setting specified in HADOOP_OPTS,
-# and/or HADOOP_JOB_HISTORYSERVER_OPTS.
+# and/or MAPRED_HISTORYSERVER_OPTS.
# Default is the same as HADOOP_HEAPSIZE_MAX.
#export HADOOP_JOB_HISTORYSERVER_HEAPSIZE=
# Specify the JVM options to be used when starting the HistoryServer.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
-#export HADOOP_JOB_HISTORYSERVER_OPTS=
+#export MAPRED_HISTORYSERVER_OPTS=
# Specify the log4j settings for the JobHistoryServer
# Java property: hadoop.root.logger
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/48] hadoop git commit: HADOOP-13357. Modify common to use
hadoop_subcommand_opts
Posted by aw...@apache.org.
HADOOP-13357. Modify common to use hadoop_subcommand_opts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1a3e102c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1a3e102c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1a3e102c
Branch: refs/heads/HADOOP-13341
Commit: 1a3e102cdf5be3b88c9101bfafe0189b9ed98165
Parents: ff98786
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 12:42:32 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/src/main/bin/hadoop | 5 +++++
1 file changed, 5 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1a3e102c/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 4aae621..9b682e6 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -208,8 +208,13 @@ if [[ ${HADOOP_WORKER_MODE} = true ]]; then
exit $?
fi
+hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+
+ hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
hadoop_verify_secure_prereq
hadoop_setup_secure_service
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/48] hadoop git commit: HADOOP-13541 explicitly declare the Joda
time version S3A depends on. Contributed by Stevel Loughran
Posted by aw...@apache.org.
HADOOP-13541 explicitly declare the Joda time version S3A depends on. Contributed by Stevel Loughran
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7fdfcd8a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7fdfcd8a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7fdfcd8a
Branch: refs/heads/HADOOP-13341
Commit: 7fdfcd8a6c9e2dd9b0fb6d4196bc371f6f9a676c
Parents: c0e492e
Author: Steve Loughran <st...@apache.org>
Authored: Wed Sep 7 12:19:05 2016 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Wed Sep 7 12:25:23 2016 +0100
----------------------------------------------------------------------
hadoop-project/pom.xml | 7 +++++++
hadoop-tools/hadoop-aws/pom.xml | 4 ++++
2 files changed, 11 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fdfcd8a/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 65e9672..d9a01a0 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -88,6 +88,7 @@
<tomcat.version>6.0.44</tomcat.version>
<guice.version>4.0</guice.version>
+ <joda-time.version>2.9.4</joda-time.version>
<!-- define the Java language version used by the compiler -->
<javac.version>1.8</javac.version>
@@ -1037,6 +1038,12 @@
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ <version>${joda-time.version}</version>
+ </dependency>
+
<dependency>
<groupId>com.nimbusds</groupId>
<artifactId>nimbus-jose-jwt</artifactId>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7fdfcd8a/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 13dcdf1..49b0379 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -286,6 +286,10 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>joda-time</groupId>
+ <artifactId>joda-time</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk-sts</artifactId>
<scope>test</scope>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/48] hadoop git commit: HADOOP-13388. Clean up
TestLocalFileSystemPermission. Contributed by Andras Bokor.
Posted by aw...@apache.org.
HADOOP-13388. Clean up TestLocalFileSystemPermission. Contributed by Andras Bokor.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f414d5e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f414d5e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f414d5e1
Branch: refs/heads/HADOOP-13341
Commit: f414d5e118940cb98015c0b66e11102a9704a505
Parents: 7fdfcd8
Author: Anu Engineer <ae...@apache.org>
Authored: Wed Sep 7 11:09:41 2016 -0700
Committer: Anu Engineer <ae...@apache.org>
Committed: Wed Sep 7 11:09:41 2016 -0700
----------------------------------------------------------------------
.../fs/TestLocalFileSystemPermission.java | 149 ++++++++-----------
1 file changed, 60 insertions(+), 89 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f414d5e1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
index 817285c..1478111 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalFileSystemPermission.java
@@ -18,24 +18,23 @@
package org.apache.hadoop.fs;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.permission.*;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
import org.apache.hadoop.util.Shell;
-
-import java.io.*;
-import java.util.*;
-
+import org.apache.log4j.Level;
import org.junit.Test;
-import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.StringTokenizer;
+
+import static org.apache.hadoop.test.PlatformAssumptions.assumeNotWindows;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.CoreMatchers.not;
-import static org.junit.Assert.assertThat;
+import static org.junit.Assert.*;
/**
* This class tests the local file system via the FileSystem abstraction.
@@ -60,18 +59,22 @@ public class TestLocalFileSystemPermission {
return f;
}
- private Path writeFile(FileSystem fs, String name, FsPermission perm) throws IOException {
+ private Path writeFile(FileSystem fs, String name, FsPermission perm)
+ throws IOException {
Path f = new Path(TEST_PATH_PREFIX + name);
- FSDataOutputStream stm = fs.create(f, perm, true, 2048, (short)1, 32 * 1024 * 1024, null);
+ FSDataOutputStream stm = fs.create(f, perm, true, 2048, (short)1,
+ 32 * 1024 * 1024, null);
stm.writeBytes("42\n");
stm.close();
return f;
}
private void cleanup(FileSystem fs, Path name) throws IOException {
- assertTrue(fs.exists(name));
- fs.delete(name, true);
- assertTrue(!fs.exists(name));
+ if (name!=null) {
+ assertTrue(fs.exists(name));
+ fs.delete(name, true);
+ assertFalse(fs.exists(name));
+ }
}
@Test
@@ -82,39 +85,33 @@ public class TestLocalFileSystemPermission {
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
Path dir = new Path(TEST_PATH_PREFIX + "dir");
localfs.mkdirs(dir);
+ Path dir1 = new Path(TEST_PATH_PREFIX + "dir1");
+ Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
+
try {
FsPermission initialPermission = getPermission(localfs, dir);
assertEquals(
- FsPermission.getDirDefault().applyUMask(FsPermission.getUMask(conf)),
+ FsPermission.getDirDefault()
+ .applyUMask(FsPermission.getUMask(conf)),
initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- FsPermission perm = new FsPermission((short)0755);
- Path dir1 = new Path(TEST_PATH_PREFIX + "dir1");
- localfs.mkdirs(dir1, perm);
- try {
- FsPermission initialPermission = getPermission(localfs, dir1);
- assertEquals(perm.applyUMask(FsPermission.getUMask(conf)), initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
+ FsPermission perm = new FsPermission((short)0755);
- Path dir2 = new Path(TEST_PATH_PREFIX + "dir2");
- localfs.mkdirs(dir2);
- try {
- FsPermission initialPermission = getPermission(localfs, dir2);
+ localfs.mkdirs(dir1, perm);
+
+ initialPermission = getPermission(localfs, dir1);
+ assertEquals(perm.applyUMask(FsPermission.getUMask(conf)),
+ initialPermission);
+
+ localfs.mkdirs(dir2);
+
+ initialPermission = getPermission(localfs, dir2);
Path copyPath = new Path(TEST_PATH_PREFIX + "dir_copy");
localfs.rename(dir2, copyPath);
FsPermission copyPermission = getPermission(localfs, copyPath);
- assertEquals(copyPermission, initialPermission);
+ assertEquals(initialPermission, copyPermission);
dir2 = copyPath;
- } catch (Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
+
} finally {
cleanup(localfs, dir);
cleanup(localfs, dir1);
@@ -124,52 +121,42 @@ public class TestLocalFileSystemPermission {
}
}
- /** Test LocalFileSystem.setPermission */
+ /** Test LocalFileSystem.setPermission. */
@Test
public void testLocalFSsetPermission() throws IOException {
assumeNotWindows();
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
LocalFileSystem localfs = FileSystem.getLocal(conf);
+ Path f = null;
+ Path f1 = null;
+ Path f2 = null;
String filename = "foo";
- Path f = writeFile(localfs, filename);
+ String filename1 = "foo1";
+ String filename2 = "foo2";
+ FsPermission perm = new FsPermission((short)0755);
+
try {
+ f = writeFile(localfs, filename);
+ f1 = writeFile(localfs, filename1, perm);
+ f2 = writeFile(localfs, filename2);
+
FsPermission initialPermission = getPermission(localfs, f);
assertEquals(
FsPermission.getFileDefault().applyUMask(FsPermission.getUMask(conf)),
initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- String filename1 = "foo1";
- FsPermission perm = new FsPermission((short)0755);
- Path f1 = writeFile(localfs, filename1, perm);
- try {
- FsPermission initialPermission = getPermission(localfs, f1);
+ initialPermission = getPermission(localfs, f1);
assertEquals(
perm.applyUMask(FsPermission.getUMask(conf)), initialPermission);
- } catch(Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- String filename2 = "foo2";
- Path f2 = writeFile(localfs, filename2);
- try {
- FsPermission initialPermission = getPermission(localfs, f2);
+ initialPermission = getPermission(localfs, f2);
Path copyPath = new Path(TEST_PATH_PREFIX + "/foo_copy");
localfs.rename(f2, copyPath);
FsPermission copyPermission = getPermission(localfs, copyPath);
assertEquals(copyPermission, initialPermission);
f2 = copyPath;
- } catch (Exception e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- try {
// create files and manipulate them.
FsPermission all = new FsPermission((short)0777);
FsPermission none = new FsPermission((short)0);
@@ -179,8 +166,7 @@ public class TestLocalFileSystemPermission {
localfs.setPermission(f, all);
assertEquals(all, getPermission(localfs, f));
- }
- finally {
+ } finally {
cleanup(localfs, f);
cleanup(localfs, f1);
if (localfs.exists(f2)) {
@@ -196,33 +182,19 @@ public class TestLocalFileSystemPermission {
/** Test LocalFileSystem.setOwner. */
@Test
public void testLocalFSsetOwner() throws IOException {
- if (Path.WINDOWS) {
- LOGGER.info("Cannot run test for Windows");
- return;
- }
+ assumeNotWindows();
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "044");
LocalFileSystem localfs = FileSystem.getLocal(conf);
String filename = "bar";
Path f = writeFile(localfs, filename);
- List<String> groups = null;
+ List<String> groups;
try {
groups = getGroups();
LOGGER.info("{}: {}", filename, getPermission(localfs, f));
- }
- catch(IOException e) {
- LOGGER.error("Cannot run test", e);
- return;
- }
- if (groups == null || groups.size() < 1) {
- LOGGER.error("Cannot run test: need at least one group. groups={}",
- groups);
- return;
- }
- // create files and manipulate them.
- try {
+ // create files and manipulate them.
String g0 = groups.get(0);
localfs.setOwner(f, null, g0);
assertEquals(g0, getGroup(localfs, f));
@@ -235,8 +207,9 @@ public class TestLocalFileSystemPermission {
LOGGER.info("Not testing changing the group since user " +
"belongs to only one group.");
}
- }
- finally {cleanup(localfs, f);}
+ } finally {
+ cleanup(localfs, f);
+ }
}
/**
@@ -250,10 +223,7 @@ public class TestLocalFileSystemPermission {
*/
@Test
public void testSetUmaskInRealTime() throws Exception {
- if (Path.WINDOWS) {
- LOGGER.info("Cannot run test for Windows");
- return;
- }
+ assumeNotWindows();
LocalFileSystem localfs = FileSystem.getLocal(new Configuration());
Configuration conf = localfs.getConf();
@@ -289,9 +259,10 @@ public class TestLocalFileSystemPermission {
}
static List<String> getGroups() throws IOException {
- List<String> a = new ArrayList<String>();
+ List<String> a = new ArrayList<>();
String s = Shell.execCommand(Shell.getGroupsCommand());
- for(StringTokenizer t = new StringTokenizer(s); t.hasMoreTokens(); ) {
+ StringTokenizer t = new StringTokenizer(s);
+ while (t.hasMoreTokens()) {
a.add(t.nextToken());
}
return a;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/48] hadoop git commit: HADOOP-13554. Add an equivalent of
hadoop_subcmd_opts for secure opts (aw)
Posted by aw...@apache.org.
HADOOP-13554. Add an equivalent of hadoop_subcmd_opts for secure opts (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6bc45fa9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6bc45fa9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6bc45fa9
Branch: refs/heads/HADOOP-13341
Commit: 6bc45fa90f509cae020eb70a4d540ae215893dc6
Parents: 0e011b7
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 08:16:23 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../src/main/bin/hadoop-functions.sh | 45 +++++++++++++++++
.../scripts/hadoop_subcommand_secure_opts.bats | 52 ++++++++++++++++++++
2 files changed, 97 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bc45fa9/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 9003913..dd82347 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -2057,6 +2057,51 @@ function hadoop_subcommand_opts
fi
}
+## @description Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS.
+## @description This *does not* handle the pre-3.x deprecated cases
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param program
+## @param subcommand
+## @return will exit on failure conditions
+function hadoop_subcommand_secure_opts
+{
+ declare program=$1
+ declare command=$2
+ declare uvar
+ declare uprogram
+ declare ucommand
+
+ if [[ -z "${program}" || -z "${command}" ]]; then
+ return 1
+ fi
+
+ # bash 4 and up have built-in ways to upper and lower
+ # case the contents of vars. This is faster than
+ # calling tr.
+
+ if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
+ ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
+ else
+ uprogram=${program^^}
+ ucommand=${command^^}
+ fi
+
+ # HDFS_DATANODE_SECURE_EXTRA_OPTS
+ # HDFS_NFS3_SECURE_EXTRA_OPTS
+ # ...
+ uvar="${uprogram}_${ucommand}_SECURE_EXTRA_OPTS"
+
+ if [[ -n ${!uvar} ]]; then
+ hadoop_debug "Appending ${!uvar} onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
+ return 0
+ fi
+}
+
## @description Perform the 'hadoop classpath', etc subcommand with the given
## @description parameters
## @audience private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6bc45fa9/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_secure_opts.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_secure_opts.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_secure_opts.bats
new file mode 100644
index 0000000..1b3506c
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_secure_opts.bats
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_subcommand_secure_opts (missing param)" {
+ HADOOP_OPTS="x"
+ run hadoop_subcommand_secure_opts testvar
+ [ "${status}" = "1" ]
+}
+
+@test "hadoop_subcommand_secure_opts (simple not exist)" {
+ HADOOP_OPTS="x"
+ hadoop_subcommand_secure_opts hadoop subcommand
+ [ "${HADOOP_OPTS}" = "x" ]
+}
+
+@test "hadoop_subcommand_secure_opts (hadoop simple exist)" {
+ HADOOP_OPTS="x"
+ HADOOP_TEST_SECURE_EXTRA_OPTS="y"
+ hadoop_subcommand_secure_opts hadoop test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
+
+@test "hadoop_subcommand_secure_opts (hadoop complex exist)" {
+ HADOOP_OPTS="x"
+ HADOOP_TEST_SECURE_EXTRA_OPTS="y z"
+ hadoop_subcommand_secure_opts hadoop test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y z" ]
+}
+
+@test "hadoop_subcommand_secure_opts (hdfs simple exist)" {
+ HADOOP_OPTS="x"
+ HDFS_TEST_SECURE_EXTRA_OPTS="y"
+ hadoop_subcommand_secure_opts hdfs test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/48] hadoop git commit: HADOOP-13360. Documentation for
HADOOP_subcommand_OPTS
Posted by aw...@apache.org.
HADOOP-13360. Documentation for HADOOP_subcommand_OPTS
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5b7a3df7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5b7a3df7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5b7a3df7
Branch: refs/heads/HADOOP-13341
Commit: 5b7a3df75c6fc78793bf8638f3ddaa11f7e0658e
Parents: dc6d490
Author: Allen Wittenauer <aw...@apache.org>
Authored: Wed Aug 31 07:39:34 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../src/site/markdown/ClusterSetup.md | 19 ++++-------
.../src/site/markdown/UnixShellGuide.md | 34 +++++++++++++++++---
.../src/site/markdown/HdfsNfsGateway.md | 2 +-
3 files changed, 37 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7a3df7/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
index 0d551b1..f222769 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/ClusterSetup.md
@@ -64,17 +64,17 @@ Administrators can configure individual daemons using the configuration options
| Daemon | Environment Variable |
|:---- |:---- |
-| NameNode | HADOOP\_NAMENODE\_OPTS |
-| DataNode | HADOOP\_DATANODE\_OPTS |
-| Secondary NameNode | HADOOP\_SECONDARYNAMENODE\_OPTS |
+| NameNode | HDFS\_NAMENODE\_OPTS |
+| DataNode | HDFS\_DATANODE\_OPTS |
+| Secondary NameNode | HDFS\_SECONDARYNAMENODE\_OPTS |
| ResourceManager | YARN\_RESOURCEMANAGER\_OPTS |
| NodeManager | YARN\_NODEMANAGER\_OPTS |
| WebAppProxy | YARN\_PROXYSERVER\_OPTS |
-| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_OPTS |
+| Map Reduce Job History Server | MAPRED\_HISTORYSERVER\_OPTS |
-For example, To configure Namenode to use parallelGC, the following statement should be added in hadoop-env.sh :
+For example, To configure Namenode to use parallelGC and a 4GB Java Heap, the following statement should be added in hadoop-env.sh :
- export HADOOP_NAMENODE_OPTS="-XX:+UseParallelGC"
+ export HDFS_NAMENODE_OPTS="-XX:+UseParallelGC -Xmx4g"
See `etc/hadoop/hadoop-env.sh` for other examples.
@@ -91,13 +91,6 @@ It is also traditional to configure `HADOOP_HOME` in the system-wide shell envir
HADOOP_HOME=/path/to/hadoop
export HADOOP_HOME
-| Daemon | Environment Variable |
-|:---- |:---- |
-| ResourceManager | YARN\_RESOURCEMANAGER\_HEAPSIZE |
-| NodeManager | YARN\_NODEMANAGER\_HEAPSIZE |
-| WebAppProxy | YARN\_PROXYSERVER\_HEAPSIZE |
-| Map Reduce Job History Server | HADOOP\_JOB\_HISTORYSERVER\_HEAPSIZE |
-
### Configuring the Hadoop Daemons
This section deals with important parameters to be specified in the given configuration files:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7a3df7/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
index 940627d..b130f0f 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/UnixShellGuide.md
@@ -24,7 +24,7 @@ Apache Hadoop has many environment variables that control various aspects of the
### `HADOOP_CLIENT_OPTS`
-This environment variable is used for almost all end-user operations. It can be used to set any Java options as well as any Apache Hadoop options via a system property definition. For example:
+This environment variable is used for all end-user, non-daemon operations. It can be used to set any Java options as well as any Apache Hadoop options via a system property definition. For example:
```bash
HADOOP_CLIENT_OPTS="-Xmx1g -Dhadoop.socks.server=localhost:4000" hadoop fs -ls /tmp
@@ -32,6 +32,18 @@ HADOOP_CLIENT_OPTS="-Xmx1g -Dhadoop.socks.server=localhost:4000" hadoop fs -ls /
will increase the memory and send this command via a SOCKS proxy server.
+### `(command)_(subcommand)_OPTS`
+
+It is also possible to set options on a per subcommand basis. This allows for one to create special options for particular cases. The first part of the pattern is the command being used, but all uppercase. The second part of the command is the subcommand being used. Then finally followed by the string `_OPT`.
+
+For example, to configure `mapred distcp` to use a 2GB heap, one would use:
+
+```bash
+MAPRED_DISTCP_OPTS="-Xmx2g"
+```
+
+These options will appear *after* `HADOOP_CLIENT_OPTS` during execution and will generally take precedence.
+
### `HADOOP_CLASSPATH`
NOTE: Site-wide settings should be configured via a shellprofile entry and permanent user-wide settings should be configured via ${HOME}/.hadooprc using the `hadoop_add_classpath` function. See below for more information.
@@ -56,6 +68,8 @@ For example:
#
HADOOP_CLIENT_OPTS="-Xmx1g"
+MAPRED_DISTCP_OPTS="-Xmx2g"
+HADOOP_DISTCP_OPTS="-Xmx2g"
```
The `.hadoop-env` file can also be used to extend functionality and teach Apache Hadoop new tricks. For example, to run hadoop commands accessing the server referenced in the environment variable `${HADOOP_SERVER}`, the following in the `.hadoop-env` will do just that:
@@ -71,11 +85,23 @@ One word of warning: not all of Unix Shell API routines are available or work c
## Administrator Environment
-There are many environment variables that impact how the system operates. By far, the most important are the series of `_OPTS` variables that control how daemons work. These variables should contain all of the relevant settings for those daemons.
+In addition to the various XML files, there are two key capabilities for administrators to configure Apache Hadoop when using the Unix Shell:
+
+ * Many environment variables that impact how the system operates. This guide will only highlight some key ones. There is generally more information in the various `*-env.sh` files.
+
+ * Supplement or do some platform-specific changes to the existing scripts. Apache Hadoop provides the capabilities to do function overrides so that the existing code base may be changed in place without all of that work. Replacing functions is covered later under the Shell API documentation.
+
+### `(command)_(subcommand)_OPTS`
+
+By far, the most important are the series of `_OPTS` variables that control how daemons work. These variables should contain all of the relevant settings for those daemons.
+
+Similar to the user commands above, all daemons will honor the `(command)_(subcommand)_OPTS` pattern. It is generally recommended that these be set in `hadoop-env.sh` to guarantee that the system will know which settings it should use on restart. Unlike user-facing subcommands, daemons will *NOT* honor `HADOOP_CLIENT_OPTS`.
+
+In addition, daemons that run in an extra security mode also support `(command)_(subcommand)_SECURE_EXTRA_OPTS`. These options are *supplemental* to the generic `*_OPTS` and will appear after, therefore generally taking precedence.
-More, detailed information is contained in `hadoop-env.sh` and the other env.sh files.
+### `(command)_(subcommand)_USER`
-Advanced administrators may wish to supplement or do some platform-specific fixes to the existing scripts. In some systems, this means copying the errant script or creating a custom build with these changes. Apache Hadoop provides the capabilities to do function overrides so that the existing code base may be changed in place without all of that work. Replacing functions is covered later under the Shell API documentation.
+Apache Hadoop provides a way to do a user check per-subcommand. While this method is easily circumvented and should not be considered a security-feature, it does provide a mechanism by which to prevent accidents. For example, setting `HDFS_NAMENODE_USER=hdfs` will make the `hdfs namenode` and `hdfs --daemon start namenode` commands verify that the user running the commands are the hdfs user by checking the `USER` environment variable. This also works for non-daemons. Setting `HADOOP_DISTCP_USER=jane` will verify that `USER` is set to `jane` before being allowed to execute the `hadoop distcp` command.
## Developer and Advanced Administrator Environment
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5b7a3df7/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
index 6731189..4742637 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsNfsGateway.md
@@ -183,7 +183,7 @@ It's strongly recommended for the users to update a few configuration properties
</property>
* JVM and log settings. You can export JVM settings (e.g., heap size and GC log) in
- HADOOP\_NFS3\_OPTS. More NFS related settings can be found in hadoop-env.sh.
+ HDFS\_NFS3\_OPTS. More NFS related settings can be found in hadoop-env.sh.
To get NFS debug trace, you can edit the log4j.property file
to add the following. Note, debug trace, especially for ONCRPC, can be very verbose.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/48] hadoop git commit: YARN-5264. Store all queue-specific
information in FSQueue. (Yufei Gu via kasha)
Posted by aw...@apache.org.
YARN-5264. Store all queue-specific information in FSQueue. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6ea9be5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6ea9be5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6ea9be5
Branch: refs/heads/HADOOP-13341
Commit: f6ea9be5473ab66798b0536317d2f32c5348eb57
Parents: cbd909c
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Fri Sep 2 14:56:29 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Fri Sep 2 14:56:29 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/AllocationConfiguration.java | 83 ++++++++++++--------
.../scheduler/fair/FSLeafQueue.java | 17 ++--
.../scheduler/fair/FSParentQueue.java | 8 +-
.../resourcemanager/scheduler/fair/FSQueue.java | 71 ++++++++++++++---
.../scheduler/fair/FairScheduler.java | 7 +-
.../scheduler/fair/MaxRunningAppsEnforcer.java | 11 +--
.../scheduler/fair/QueueManager.java | 45 ++---------
.../webapp/dao/FairSchedulerQueueInfo.java | 2 +-
.../scheduler/fair/TestAppRunnability.java | 6 +-
.../scheduler/fair/TestFSLeafQueue.java | 10 +--
.../scheduler/fair/TestFairScheduler.java | 11 ++-
.../fair/TestMaxRunningAppsEnforcer.java | 20 ++---
.../scheduler/fair/TestQueueManager.java | 17 ++--
13 files changed, 160 insertions(+), 148 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
index 5cf110f..c771887 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/AllocationConfiguration.java
@@ -22,6 +22,8 @@ import java.util.HashSet;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
@@ -37,6 +39,7 @@ import org.apache.hadoop.yarn.util.resource.Resources;
import com.google.common.annotations.VisibleForTesting;
public class AllocationConfiguration extends ReservationSchedulerConfiguration {
+ private static final Log LOG = LogFactory.getLog(FSQueue.class.getName());
private static final AccessControlList EVERYBODY_ACL = new AccessControlList("*");
private static final AccessControlList NOBODY_ACL = new AccessControlList(" ");
private static final ResourceCalculator RESOURCE_CALCULATOR =
@@ -242,26 +245,24 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return !nonPreemptableQueues.contains(queueName);
}
- public ResourceWeights getQueueWeight(String queue) {
+ private ResourceWeights getQueueWeight(String queue) {
ResourceWeights weight = queueWeights.get(queue);
return (weight == null) ? ResourceWeights.NEUTRAL : weight;
}
- public void setQueueWeight(String queue, ResourceWeights weight) {
- queueWeights.put(queue, weight);
- }
-
public int getUserMaxApps(String user) {
Integer maxApps = userMaxApps.get(user);
return (maxApps == null) ? userMaxAppsDefault : maxApps;
}
- public int getQueueMaxApps(String queue) {
+ @VisibleForTesting
+ int getQueueMaxApps(String queue) {
Integer maxApps = queueMaxApps.get(queue);
return (maxApps == null) ? queueMaxAppsDefault : maxApps;
}
-
- public float getQueueMaxAMShare(String queue) {
+
+ @VisibleForTesting
+ float getQueueMaxAMShare(String queue) {
Float maxAMShare = queueMaxAMShares.get(queue);
return (maxAMShare == null) ? queueMaxAMShareDefault : maxAMShare;
}
@@ -273,29 +274,21 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
* @return the min allocation on this queue or {@link Resources#none}
* if not set
*/
- public Resource getMinResources(String queue) {
+ @VisibleForTesting
+ Resource getMinResources(String queue) {
Resource minQueueResource = minQueueResources.get(queue);
return (minQueueResource == null) ? Resources.none() : minQueueResource;
}
/**
- * Set the maximum resource allocation for the given queue.
- *
- * @param queue the target queue
- * @param maxResource the maximum resource allocation
- */
- void setMaxResources(String queue, Resource maxResource) {
- maxQueueResources.put(queue, maxResource);
- }
-
- /**
* Get the maximum resource allocation for the given queue. If the max in not
* set, return the larger of the min and the default max.
*
* @param queue the target queue's name
* @return the max allocation on this queue
*/
- public Resource getMaxResources(String queue) {
+ @VisibleForTesting
+ Resource getMaxResources(String queue) {
Resource maxQueueResource = maxQueueResources.get(queue);
if (maxQueueResource == null) {
Resource minQueueResource = minQueueResources.get(queue);
@@ -317,21 +310,11 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
* @param queue the target queue's name
* @return the max allocation on this queue or null if not set
*/
- public Resource getMaxChildResources(String queue) {
+ @VisibleForTesting
+ Resource getMaxChildResources(String queue) {
return maxChildQueueResources.get(queue);
}
- /**
- * Set the maximum resource allocation for the children of the given queue.
- * Use of this method is primarily intended for testing purposes.
- *
- * @param queue the target queue
- * @param maxResource the maximum resource allocation
- */
- void setMaxChildResources(String queue, Resource maxResource) {
- maxChildQueueResources.put(queue, maxResource);
- }
-
public boolean hasAccess(String queueName, QueueACL acl,
UserGroupInformation user) {
int lastPeriodIndex = queueName.length();
@@ -346,8 +329,9 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
return false;
}
-
- public SchedulingPolicy getSchedulingPolicy(String queueName) {
+
+ @VisibleForTesting
+ SchedulingPolicy getSchedulingPolicy(String queueName) {
SchedulingPolicy policy = schedulingPolicies.get(queueName);
return (policy == null) ? defaultSchedulingPolicy : policy;
}
@@ -423,4 +407,35 @@ public class AllocationConfiguration extends ReservationSchedulerConfiguration {
public void setAverageCapacity(int avgCapacity) {
globalReservationQueueConfig.setAverageCapacity(avgCapacity);
}
+
+ /**
+ * Initialize a {@link FSQueue} with queue-specific properties and its
+ * metrics.
+ * @param queue the FSQueue needed to be initialized
+ * @param scheduler the scheduler which the queue belonged to
+ */
+ public void initFSQueue(FSQueue queue, FairScheduler scheduler){
+ // Set queue-specific properties.
+ String name = queue.getName();
+ queue.setWeights(getQueueWeight(name));
+ queue.setMinShare(getMinResources(name));
+ queue.setMaxShare(getMaxResources(name));
+ queue.setMaxRunningApps(getQueueMaxApps(name));
+ queue.setMaxAMShare(getQueueMaxAMShare(name));
+ queue.setMaxChildQueueResource(getMaxChildResources(name));
+ try {
+ SchedulingPolicy policy = getSchedulingPolicy(name);
+ policy.initialize(scheduler.getClusterResource());
+ queue.setPolicy(policy);
+ } catch (AllocationConfigurationException ex) {
+ LOG.warn("Failed to set the scheduling policy "
+ + getDefaultSchedulingPolicy(), ex);
+ }
+
+ // Set queue metrics.
+ queue.getMetrics().setMinShare(getMinResources(name));
+ queue.getMetrics().setMaxShare(getMaxResources(name));
+ queue.getMetrics().setMaxApps(getQueueMaxApps(name));
+ queue.getMetrics().setSchedulingPolicy(getSchedulingPolicy(name).getName());
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
index 7c9b11e..a6adb47 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
@@ -264,29 +264,27 @@ public class FSLeafQueue extends FSQueue {
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
- Resource maxRes = scheduler.getAllocationConfiguration()
- .getMaxResources(getName());
demand = Resources.createResource(0);
readLock.lock();
try {
for (FSAppAttempt sched : runnableApps) {
- if (Resources.equals(demand, maxRes)) {
+ if (Resources.equals(demand, maxShare)) {
break;
}
- updateDemandForApp(sched, maxRes);
+ updateDemandForApp(sched, maxShare);
}
for (FSAppAttempt sched : nonRunnableApps) {
- if (Resources.equals(demand, maxRes)) {
+ if (Resources.equals(demand, maxShare)) {
break;
}
- updateDemandForApp(sched, maxRes);
+ updateDemandForApp(sched, maxShare);
}
} finally {
readLock.unlock();
}
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand
- + "; the max is " + maxRes);
+ + "; the max is " + maxShare);
LOG.debug("The updated fairshare for " + getName() + " is "
+ getFairShare());
}
@@ -486,8 +484,6 @@ public class FSLeafQueue extends FSQueue {
* @return true if this queue can run
*/
public boolean canRunAppAM(Resource amResource) {
- float maxAMShare =
- scheduler.getAllocationConfiguration().getQueueMaxAMShare(getName());
if (Math.abs(maxAMShare - -1.0f) < 0.0001) {
return true;
}
@@ -543,8 +539,7 @@ public class FSLeafQueue extends FSQueue {
* @param weight queue weight
*/
public void setWeights(float weight) {
- scheduler.getAllocationConfiguration().setQueueWeight(getName(),
- new ResourceWeights(weight));
+ this.weights = new ResourceWeights(weight);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
index 035c60c..e58c3f1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
@@ -152,8 +152,6 @@ public class FSParentQueue extends FSQueue {
public void updateDemand() {
// Compute demand by iterating through apps in the queue
// Limit demand to maxResources
- Resource maxRes = scheduler.getAllocationConfiguration()
- .getMaxResources(getName());
writeLock.lock();
try {
demand = Resources.createResource(0);
@@ -166,8 +164,8 @@ public class FSParentQueue extends FSQueue {
" now " + demand);
}
demand = Resources.add(demand, toAdd);
- demand = Resources.componentwiseMin(demand, maxRes);
- if (Resources.equals(demand, maxRes)) {
+ demand = Resources.componentwiseMin(demand, maxShare);
+ if (Resources.equals(demand, maxShare)) {
break;
}
}
@@ -176,7 +174,7 @@ public class FSParentQueue extends FSQueue {
}
if (LOG.isDebugEnabled()) {
LOG.debug("The updated demand for " + getName() + " is " + demand +
- "; the max is " + maxRes);
+ "; the max is " + maxShare);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
index 25554dd..5fa2ee1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
@@ -59,6 +59,15 @@ public abstract class FSQueue implements Queue, Schedulable {
protected SchedulingPolicy policy = SchedulingPolicy.DEFAULT_POLICY;
+ protected ResourceWeights weights;
+ protected Resource minShare;
+ protected Resource maxShare;
+ protected int maxRunningApps;
+ protected Resource maxChildQueueResource;
+
+ // maxAMShare is a value between 0 and 1.
+ protected float maxAMShare;
+
private long fairSharePreemptionTimeout = Long.MAX_VALUE;
private long minSharePreemptionTimeout = Long.MAX_VALUE;
private float fairSharePreemptionThreshold = 0.5f;
@@ -68,15 +77,20 @@ public abstract class FSQueue implements Queue, Schedulable {
this.name = name;
this.scheduler = scheduler;
this.metrics = FSQueueMetrics.forQueue(getName(), parent, true, scheduler.getConf());
- metrics.setMinShare(getMinShare());
- metrics.setMaxShare(getMaxShare());
+ this.parent = parent;
+ }
+ /**
+ * Initialize a queue by setting its queue-specific properties and its
+ * metrics.
+ * This function is invoked when a new queue is created or reloading the
+ * allocation configuration.
+ */
+ public void init() {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
- metrics.setMaxApps(allocConf.getQueueMaxApps(name));
- metrics.setSchedulingPolicy(allocConf.getSchedulingPolicy(name).getName());
- this.parent = parent;
+ allocConf.initFSQueue(this, scheduler);
}
-
+
public String getName() {
return name;
}
@@ -103,19 +117,51 @@ public abstract class FSQueue implements Queue, Schedulable {
public abstract void setPolicy(SchedulingPolicy policy)
throws AllocationConfigurationException;
+ public void setWeights(ResourceWeights weights){
+ this.weights = weights;
+ }
+
@Override
public ResourceWeights getWeights() {
- return scheduler.getAllocationConfiguration().getQueueWeight(getName());
+ return weights;
}
-
+
+ public void setMinShare(Resource minShare){
+ this.minShare = minShare;
+ }
+
@Override
public Resource getMinShare() {
- return scheduler.getAllocationConfiguration().getMinResources(getName());
+ return minShare;
}
-
+
+ public void setMaxShare(Resource maxShare){
+ this.maxShare = maxShare;
+ }
+
@Override
public Resource getMaxShare() {
- return scheduler.getAllocationConfiguration().getMaxResources(getName());
+ return maxShare;
+ }
+
+ public void setMaxChildQueueResource(Resource maxChildShare){
+ this.maxChildQueueResource = maxChildShare;
+ }
+
+ public Resource getMaxChildQueueResource() {
+ return maxChildQueueResource;
+ }
+
+ public void setMaxRunningApps(int maxRunningApps){
+ this.maxRunningApps = maxRunningApps;
+ }
+
+ public int getMaxRunningApps() {
+ return maxRunningApps;
+ }
+
+ public void setMaxAMShare(float maxAMShare){
+ this.maxAMShare = maxAMShare;
}
@Override
@@ -302,8 +348,7 @@ public abstract class FSQueue implements Queue, Schedulable {
* @return true if check passes (can assign) or false otherwise
*/
protected boolean assignContainerPreCheck(FSSchedulerNode node) {
- if (!Resources.fitsIn(getResourceUsage(),
- scheduler.getAllocationConfiguration().getMaxResources(getName()))
+ if (!Resources.fitsIn(getResourceUsage(), maxShare)
|| node.getReservedContainer() != null) {
return false;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 140b4f7..ac8dbe1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -1599,11 +1599,10 @@ public class FairScheduler extends
if ((queue.getParent() != null) &&
!configuredLeafQueues.contains(queue.getName()) &&
!configuredParentQueues.contains(queue.getName())) {
- Resource max =
- allocConf.getMaxChildResources(queue.getParent().getName());
+ Resource max = queue.getParent().getMaxChildQueueResource();
if (max != null) {
- allocConf.setMaxResources(queue.getName(), max);
+ queue.setMaxShare(max);
}
}
}
@@ -1665,7 +1664,7 @@ public class FairScheduler extends
FSQueue cur = targetQueue;
while (cur != lowestCommonAncestor) {
// maxRunningApps
- if (cur.getNumRunnableApps() == allocConf.getQueueMaxApps(cur.getQueueName())) {
+ if (cur.getNumRunnableApps() == cur.getMaxRunningApps()) {
throw new YarnException("Moving app attempt " + appAttId + " to queue "
+ queueName + " would violate queue maxRunningApps constraints on"
+ " queue " + cur.getQueueName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
index f750438..8592fa6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/MaxRunningAppsEnforcer.java
@@ -66,8 +66,7 @@ public class MaxRunningAppsEnforcer {
}
// Check queue and all parent queues
while (queue != null) {
- int queueMaxApps = allocConf.getQueueMaxApps(queue.getName());
- if (queue.getNumRunnableApps() >= queueMaxApps) {
+ if (queue.getNumRunnableApps() >= queue.getMaxRunningApps()) {
return false;
}
queue = queue.getParent();
@@ -143,11 +142,10 @@ public class MaxRunningAppsEnforcer {
// Thus we find the ancestor queue highest in the tree for which the app
// that was at its maxRunningApps before the removal.
FSQueue highestQueueWithAppsNowRunnable = (queue.getNumRunnableApps() ==
- allocConf.getQueueMaxApps(queue.getName()) - 1) ? queue : null;
+ queue.getMaxRunningApps() - 1) ? queue : null;
FSParentQueue parent = queue.getParent();
while (parent != null) {
- if (parent.getNumRunnableApps() == allocConf.getQueueMaxApps(parent
- .getName()) - 1) {
+ if (parent.getNumRunnableApps() == parent.getMaxRunningApps() - 1) {
highestQueueWithAppsNowRunnable = parent;
}
parent = parent.getParent();
@@ -265,8 +263,7 @@ public class MaxRunningAppsEnforcer {
*/
private void gatherPossiblyRunnableAppLists(FSQueue queue,
List<List<FSAppAttempt>> appLists) {
- if (queue.getNumRunnableApps() < scheduler.getAllocationConfiguration()
- .getQueueMaxApps(queue.getName())) {
+ if (queue.getNumRunnableApps() < queue.getMaxRunningApps()) {
if (queue instanceof FSLeafQueue) {
appLists.add(
((FSLeafQueue)queue).getCopyOfNonRunnableAppSchedulables());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
index 964c72b..bca0ea4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/QueueManager.java
@@ -73,8 +73,9 @@ public class QueueManager {
public void initialize(Configuration conf) throws IOException,
SAXException, AllocationConfigurationException, ParserConfigurationException {
rootQueue = new FSParentQueue("root", scheduler, null);
+ rootQueue.init();
queues.put(rootQueue.getName(), rootQueue);
-
+
// Create the default queue
getLeafQueue(YarnConfiguration.DEFAULT_QUEUE_NAME, true);
}
@@ -174,7 +175,7 @@ public class QueueManager {
}
return queue;
}
-
+
/**
* Create a leaf or parent queue based on what is specified in
* {@code queueType} and place it in the tree. Create any parents that don't
@@ -273,31 +274,14 @@ public class QueueManager {
// Only create a leaf queue at the very end
if (!i.hasNext() && (queueType != FSQueueType.PARENT)) {
FSLeafQueue leafQueue = new FSLeafQueue(queueName, scheduler, parent);
-
- try {
- leafQueue.setPolicy(queueConf.getDefaultSchedulingPolicy());
- } catch (AllocationConfigurationException ex) {
- LOG.warn("Failed to set default scheduling policy "
- + queueConf.getDefaultSchedulingPolicy()
- + " on new leaf queue.", ex);
- }
-
leafQueues.add(leafQueue);
queue = leafQueue;
} else {
newParent = new FSParentQueue(queueName, scheduler, parent);
-
- try {
- newParent.setPolicy(queueConf.getDefaultSchedulingPolicy());
- } catch (AllocationConfigurationException ex) {
- LOG.warn("Failed to set default scheduling policy "
- + queueConf.getDefaultSchedulingPolicy()
- + " on new parent queue.", ex);
- }
-
queue = newParent;
}
+ queue.init();
parent.addChildQueue(queue);
setChildResourceLimits(parent, queue, queueConf);
queues.put(queue.getName(), queue);
@@ -331,10 +315,10 @@ public class QueueManager {
!configuredQueues.get(FSQueueType.PARENT).contains(child.getName())) {
// For ad hoc queues, set their max reource allocations based on
// their parents' default child settings.
- Resource maxChild = queueConf.getMaxChildResources(parent.getName());
+ Resource maxChild = parent.getMaxChildQueueResource();
if (maxChild != null) {
- queueConf.setMaxResources(child.getName(), maxChild);
+ child.setMaxShare(maxChild);
}
}
}
@@ -515,22 +499,7 @@ public class QueueManager {
rootQueue.recomputeSteadyShares();
for (FSQueue queue : queues.values()) {
- // Update queue metrics
- FSQueueMetrics queueMetrics = queue.getMetrics();
- queueMetrics.setMinShare(queue.getMinShare());
- queueMetrics.setMaxShare(queue.getMaxShare());
- // Set scheduling policies and update queue metrics
- try {
- SchedulingPolicy policy = queueConf.getSchedulingPolicy(queue.getName());
- policy.initialize(scheduler.getClusterResource());
- queue.setPolicy(policy);
-
- queueMetrics.setMaxApps(queueConf.getQueueMaxApps(queue.getName()));
- queueMetrics.setSchedulingPolicy(policy.getName());
- } catch (AllocationConfigurationException ex) {
- LOG.warn("Cannot apply configured scheduling policy to queue "
- + queue.getName(), ex);
- }
+ queue.init();
}
// Update steady fair shares for all queues
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index a09b92f..64e573b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -101,7 +101,7 @@ public class FairSchedulerQueueInfo {
fractionMemMinShare = (float)minResources.getMemorySize() / clusterResources.getMemorySize();
fractionMemMaxShare = (float)maxResources.getMemorySize() / clusterResources.getMemorySize();
- maxApps = allocConf.getQueueMaxApps(queueName);
+ maxApps = queue.getMaxRunningApps();
pendingContainers = queue.getMetrics().getPendingContainers();
allocatedContainers = queue.getMetrics().getAllocatedContainers();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
index 7ca9606..f581935 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestAppRunnability.java
@@ -247,8 +247,8 @@ public class TestAppRunnability extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
+ oldQueue.setMaxRunningApps(0);
+ targetQueue.setMaxRunningApps(0);
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
@@ -265,7 +265,7 @@ public class TestAppRunnability extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
FSLeafQueue targetQueue = queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue1", 0);
+ oldQueue.setMaxRunningApps(0);
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
index ad4e2e4..0a2ce81 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFSLeafQueue.java
@@ -22,7 +22,6 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
import java.io.File;
import java.io.FileWriter;
@@ -80,16 +79,11 @@ public class TestFSLeafQueue extends FairSchedulerTestBase {
resourceManager = new MockRM(conf);
resourceManager.start();
scheduler = (FairScheduler) resourceManager.getResourceScheduler();
- scheduler.allocConf = mock(AllocationConfiguration.class);
String queueName = "root.queue1";
- when(scheduler.allocConf.getMaxResources(queueName)).thenReturn(maxResource);
- when(scheduler.allocConf.getMinResources(queueName)).thenReturn(Resources.none());
- when(scheduler.allocConf.getQueueMaxApps(queueName)).
- thenReturn(Integer.MAX_VALUE);
- when(scheduler.allocConf.getSchedulingPolicy(queueName))
- .thenReturn(SchedulingPolicy.DEFAULT_POLICY);
FSLeafQueue schedulable = new FSLeafQueue(queueName, scheduler, null);
+ schedulable.init();
+ schedulable.setMaxShare(maxResource);
assertEquals(schedulable.getMetrics().getMaxApps(), Integer.MAX_VALUE);
assertEquals(schedulable.getMetrics().getSchedulingPolicy(),
SchedulingPolicy.DEFAULT_POLICY.getName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
index c348572..98af8b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
@@ -4348,9 +4348,9 @@ public class TestFairScheduler extends FairSchedulerTestBase {
scheduler.reinitialize(conf, resourceManager.getRMContext());
QueueManager queueMgr = scheduler.getQueueManager();
- queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().queueMaxApps.put("root.queue2", 0);
-
+ FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
+ queue2.setMaxRunningApps(0);
+
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
@@ -4365,9 +4365,8 @@ public class TestFairScheduler extends FairSchedulerTestBase {
QueueManager queueMgr = scheduler.getQueueManager();
FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
- queueMgr.getLeafQueue("queue2", true);
- scheduler.getAllocationConfiguration().maxQueueResources.put("root.queue2",
- Resource.newInstance(1024, 1));
+ FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
+ queue2.setMaxShare(Resource.newInstance(1024, 1));
ApplicationAttemptId appAttId =
createSchedulingRequest(1024, 1, "queue1", "user1", 3);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
index 6cca19a..3f17081 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestMaxRunningAppsEnforcer.java
@@ -37,7 +37,6 @@ import org.junit.Test;
public class TestMaxRunningAppsEnforcer {
private QueueManager queueManager;
- private Map<String, Integer> queueMaxApps;
private Map<String, Integer> userMaxApps;
private MaxRunningAppsEnforcer maxAppsEnforcer;
private int appNum;
@@ -59,7 +58,6 @@ public class TestMaxRunningAppsEnforcer {
queueManager = new QueueManager(scheduler);
queueManager.initialize(conf);
- queueMaxApps = allocConf.queueMaxApps;
userMaxApps = allocConf.userMaxApps;
maxAppsEnforcer = new MaxRunningAppsEnforcer(scheduler);
appNum = 0;
@@ -90,11 +88,12 @@ public class TestMaxRunningAppsEnforcer {
@Test
public void testRemoveDoesNotEnableAnyApp() {
+ FSParentQueue root = queueManager.getRootQueue();
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue2", true);
- queueMaxApps.put("root", 2);
- queueMaxApps.put("root.queue1", 1);
- queueMaxApps.put("root.queue2", 1);
+ root.setMaxRunningApps(2);
+ leaf1.setMaxRunningApps(1);
+ leaf2.setMaxRunningApps(1);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -111,7 +110,8 @@ public class TestMaxRunningAppsEnforcer {
public void testRemoveEnablesAppOnCousinQueue() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
- queueMaxApps.put("root.queue1", 2);
+ FSParentQueue queue1 = queueManager.getParentQueue("root.queue1", true);
+ queue1.setMaxRunningApps(2);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -128,7 +128,7 @@ public class TestMaxRunningAppsEnforcer {
public void testRemoveEnablesOneByQueueOneByUser() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.leaf2", true);
- queueMaxApps.put("root.queue1.leaf1", 2);
+ leaf1.setMaxRunningApps(2);
userMaxApps.put("user1", 1);
FSAppAttempt app1 = addApp(leaf1, "user1");
addApp(leaf1, "user2");
@@ -148,7 +148,8 @@ public class TestMaxRunningAppsEnforcer {
public void testRemoveEnablingOrderedByStartTime() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
- queueMaxApps.put("root.queue1", 2);
+ FSParentQueue queue1 = queueManager.getParentQueue("root.queue1", true);
+ queue1.setMaxRunningApps(2);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
@@ -168,7 +169,8 @@ public class TestMaxRunningAppsEnforcer {
public void testMultipleAppsWaitingOnCousinQueue() {
FSLeafQueue leaf1 = queueManager.getLeafQueue("root.queue1.subqueue1.leaf1", true);
FSLeafQueue leaf2 = queueManager.getLeafQueue("root.queue1.subqueue2.leaf2", true);
- queueMaxApps.put("root.queue1", 2);
+ FSParentQueue queue1 = queueManager.getParentQueue("root.queue1", true);
+ queue1.setMaxRunningApps(2);
FSAppAttempt app1 = addApp(leaf1, "user");
addApp(leaf2, "user");
addApp(leaf2, "user");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6ea9be5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
index a9c3788..05ab11c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestQueueManager.java
@@ -45,8 +45,6 @@ public class TestQueueManager {
// Set up some queues to test default child max resource inheritance
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test");
- allocConf.setMaxChildResources("root.test",
- Resources.createResource(8192, 256));
allocConf.configuredQueues.get(FSQueueType.LEAF).add("root.test.childA");
allocConf.configuredQueues.get(FSQueueType.PARENT).add("root.test.childB");
@@ -207,16 +205,17 @@ public class TestQueueManager {
AllocationConfiguration allocConf = scheduler.getAllocationConfiguration();
queueManager.updateAllocationConfiguration(allocConf);
+ queueManager.getQueue("root.test").setMaxChildQueueResource(
+ Resources.createResource(8192, 256));
FSQueue q1 = queueManager.createQueue("root.test.childC", FSQueueType.LEAF);
-
assertNotNull("Leaf queue root.test.childC was not created",
queueManager.getLeafQueue("root.test.childC", false));
assertEquals("createQueue() returned wrong queue",
"root.test.childC", q1.getName());
assertEquals("Max resources for root.queue1 were not inherited from "
+ "parent's max child resources", Resources.createResource(8192, 256),
- allocConf.getMaxResources("root.test.childC"));
+ q1.getMaxShare());
FSQueue q2 = queueManager.createQueue("root.test.childD",
FSQueueType.PARENT);
@@ -228,7 +227,7 @@ public class TestQueueManager {
assertEquals("Max resources for root.test.childD were not inherited "
+ "from parent's max child resources",
Resources.createResource(8192, 256),
- allocConf.getMaxResources("root.test.childD"));
+ q2.getMaxShare());
// Check that the childA and childB queues weren't impacted
// by the child defaults
@@ -236,12 +235,12 @@ public class TestQueueManager {
queueManager.getLeafQueue("root.test.childA", false));
assertEquals("Max resources for root.test.childA were inherited from "
+ "parent's max child resources", Resources.unbounded(),
- allocConf.getMaxResources("root.test.childA"));
+ queueManager.getLeafQueue("root.test.childA", false).getMaxShare());
assertNotNull("Leaf queue root.test.childB was not created during setup",
queueManager.getParentQueue("root.test.childB", false));
assertEquals("Max resources for root.test.childB were inherited from "
+ "parent's max child resources", Resources.unbounded(),
- allocConf.getMaxResources("root.test.childB"));
+ queueManager.getParentQueue("root.test.childB", false).getMaxShare());
}
/**
@@ -260,11 +259,11 @@ public class TestQueueManager {
// Min default is 0,0
assertEquals("Min resources were not set to default",
Resources.createResource(0, 0),
- allocConf.getMinResources("root.queue1"));
+ q1.getMinShare());
// Max default is unbounded
assertEquals("Max resources were not set to default", Resources.unbounded(),
- allocConf.getMaxResources("root.queue1"));
+ q1.getMaxShare());
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/48] hadoop git commit: HDFS-6962. ACL inheritance conflicts with
umaskmode. Contributed by Chris Nauroth.
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
new file mode 100644
index 0000000..7e9ace1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testAclCLIWithPosixAclInheritance.xml
@@ -0,0 +1,1075 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="testConf.xsl"?>
+
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<configuration>
+ <!-- Normal mode is test. To run just the commands and dump the output
+ to the log, set it to nocompare -->
+ <mode>test</mode>
+
+ <!-- Comparator types:
+ ExactComparator
+ SubstringComparator
+ RegexpComparator
+ TokenComparator
+ -->
+ <tests>
+ <!-- Tests for setfacl and getfacl-->
+ <test>
+ <description>getfacl: basic permissions</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: basic permissions for directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add an ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r-- /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add multiple ACLs at once</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove an ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,user:charlie:r-x /file1</command>
+ <command>-fs NAMENODE -setfacl -x user:bob /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!bob)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Add minimal default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user::rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default\:mask)*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : try adding default ACL to file</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>setfacl: Invalid ACL: only directories may have a default ACL</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove one default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -x default:user:charlie /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default:user:charlie).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:bob:r--,group:users:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -k /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:bob:r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:users:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all but base ACLs for a directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -b /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!charlie).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!admin).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : Remove all but base ACLs for a file</description>
+ <test-commands>
+ <command>-fs NAMENODE -touchz /file1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /file1</command>
+ <command>-fs NAMENODE -setfacl -b /file1</command>
+ <command>-fs NAMENODE -getfacl /file1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /file1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /file1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!charlie).*</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!admin).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : check inherit default ACL to file</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file</command>
+ <command>-fs NAMENODE -getfacl /dir1/file</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1/file</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r--</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl : check inherit default ACL to dir</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:r-x,default:group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -getfacl /dir1/dir2</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1/dir2</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rwx$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group:admin:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl -R : recursive</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -m user:user1:r-x,group:users:rwx /dir1/dir2</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:user1:r-x#LF#group::r-x#LF#group:users:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl -R : recursive</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -mkdir /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:r-x#LF#group::r-x#LF#group:admin:rwx#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl --set : Set full set of ACLs</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl --set user::rw-,group::r--,other::r--,user:user1:r-x,group:users:rw- /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:user1:r-x#LF#group::r--#LF#group:users:rw-#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl -x mask : remove mask entry along with other ACL entries</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:r-x,group:admin:rwx /dir1</command>
+ <command>-fs NAMENODE -setfacl -x mask::,user:charlie,group:admin /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#group::r-x#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: only default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:rwx /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user:charlie:rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:group::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>getfacl: effective permissions</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:rwx,group::-wx,group:sales:rwx,mask::r-x,default:user:charlie:rwx,default:group::r-x,default:group:sales:rwx,default:mask::rw- /dir1</command>
+ <command>-fs NAMENODE -getfacl /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># file: /dir1</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># owner: USERNAME</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output># group: supergroup</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user:charlie:rwx\t#effective:r-x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group::-wx\t#effective:--x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group:sales:rwx\t#effective:r-x$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>mask::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>other::r-x</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:user::rwx</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:user:charlie:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:group::r-x\t#effective:r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^default:group:sales:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:mask::rw-</expected-output>
+ </comparator>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>default:other::r-x</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>ls: display extended acl marker</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -setfacl -m user:charlie:rwx,group::-wx,group:sales:rwx,mask::r-x,default:user:charlie:rwx,default:group::r-x,default:group:sales:rwx,default:mask::rw- /dir1/dir2</command>
+ <command>-fs NAMENODE -ls /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^drwxr-xr-x\+( )*-( )*USERNAME( )*supergroup( )*0( )*[0-9]{4,}-[0-9]{2,}-[0-9]{2,} [0-9]{2,}:[0-9]{2,}( )*/dir1/dir2</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive modify entries with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:charlie:rwx,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive remove entries with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R -m user:bob:rwx,user:charlie:rwx,default:user:bob:rwx,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -setfacl -R -x user:bob,default:user:bob /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rw-#LF#user:charlie:rwx#LF#group::r--#LF#mask::rwx#LF#other::r--#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>setfacl: recursive set with mix of files and directories</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir -p /dir1</command>
+ <command>-fs NAMENODE -touchz /dir1/file1</command>
+ <command>-fs NAMENODE -mkdir -p /dir1/dir2</command>
+ <command>-fs NAMENODE -touchz /dir1/dir2/file2</command>
+ <command>-fs NAMENODE -setfacl -R --set user::rwx,user:charlie:rwx,group::r-x,other::r-x,default:user:charlie:r-x /dir1</command>
+ <command>-fs NAMENODE -getfacl -R /dir1</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>ExactComparator</type>
+ <expected-output># file: /dir1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF#default:user::rwx#LF#default:user:charlie:r-x#LF#default:group::r-x#LF#default:mask::r-x#LF#default:other::r-x#LF##LF## file: /dir1/dir2/file2#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF##LF## file: /dir1/file1#LF## owner: USERNAME#LF## group: supergroup#LF#user::rwx#LF#user:charlie:rwx#LF#group::r-x#LF#mask::rwx#LF#other::r-x#LF##LF#</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ <test>
+ <description>copyFromLocal: copying file into a directory with a default ACL</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /dir1</command>
+ <command>-fs NAMENODE -setfacl -m default:user:charlie:rwx /dir1</command>
+ <command>-fs NAMENODE -copyFromLocal CLITEST_DATA/data1k /dir1/data1k</command>
+ <command>-fs NAMENODE -getfacl /dir1/data1k</command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -R /dir1</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# file: /dir1/data1k$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# owner: USERNAME$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^# group: supergroup$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^user:charlie:rwx\t#effective:rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^group::r-x\t#effective:r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^mask::rw-$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpComparator</type>
+ <expected-output>^other::r--$</expected-output>
+ </comparator>
+ <comparator>
+ <type>RegexpAcrossOutputComparator</type>
+ <expected-output>.*(?!default).*</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+ </tests>
+</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/48] hadoop git commit: HADOOP-13218. Migrate other Hadoop side
tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and
Kai Zheng
Posted by aw...@apache.org.
HADOOP-13218. Migrate other Hadoop side tests to prepare for removing WritableRPCEngine. Contributed by Wei Zhou and Kai Zheng
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62a96671
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62a96671
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62a96671
Branch: refs/heads/HADOOP-13341
Commit: 62a9667136ebd8a048f556b534fcff4fdaf8e2ec
Parents: f955712
Author: Kai Zheng <ka...@intel.com>
Authored: Wed Sep 7 17:05:33 2016 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Wed Sep 7 17:05:33 2016 +0800
----------------------------------------------------------------------
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 5 +-
.../main/java/org/apache/hadoop/ipc/RPC.java | 15 +-
.../main/java/org/apache/hadoop/ipc/Server.java | 4 +-
.../hadoop/security/UserGroupInformation.java | 4 +-
.../org/apache/hadoop/ipc/RPCCallBenchmark.java | 38 +--
.../hadoop/ipc/TestMultipleProtocolServer.java | 236 +--------------
.../apache/hadoop/ipc/TestRPCCallBenchmark.java | 13 -
.../apache/hadoop/ipc/TestRPCCompatibility.java | 242 ++-------------
.../apache/hadoop/ipc/TestRPCWaitForProxy.java | 37 ++-
.../java/org/apache/hadoop/ipc/TestRpcBase.java | 50 +++-
.../java/org/apache/hadoop/ipc/TestSaslRPC.java | 74 +++--
.../hadoop/security/TestDoAsEffectiveUser.java | 291 +++++++------------
.../security/TestUserGroupInformation.java | 28 +-
.../hadoop-common/src/test/proto/test.proto | 4 +-
.../src/test/proto/test_rpc_service.proto | 4 +-
.../hdfs/server/namenode/NameNodeRpcServer.java | 3 -
.../TestClientProtocolWithDelegationToken.java | 119 --------
.../mapreduce/v2/hs/server/HSAdminServer.java | 3 -
18 files changed, 294 insertions(+), 876 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index 83e4b9e..e68bfd4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -60,7 +60,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private static final ThreadLocal<AsyncGet<Message, Exception>>
ASYNC_RETURN_MESSAGE = new ThreadLocal<>();
- static { // Register the rpcRequest deserializer for WritableRpcEngine
+ static { // Register the rpcRequest deserializer for ProtobufRpcEngine
org.apache.hadoop.ipc.Server.registerProtocolEngine(
RPC.RpcKind.RPC_PROTOCOL_BUFFER, RpcProtobufRequest.class,
new Server.ProtoBufRpcInvoker());
@@ -194,7 +194,8 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (args.length != 2) { // RpcController + Message
- throw new ServiceException("Too many parameters for request. Method: ["
+ throw new ServiceException(
+ "Too many or few parameters for request. Method: ["
+ method.getName() + "]" + ", Expected: 2, Actual: "
+ args.length);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
index 3f68d63..12a07a5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RPC.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ipc;
+import java.io.IOException;
+import java.io.InterruptedIOException;
import java.lang.reflect.Field;
import java.lang.reflect.InvocationHandler;
import java.lang.reflect.Proxy;
@@ -26,7 +28,6 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.net.NoRouteToHostException;
import java.net.SocketTimeoutException;
-import java.io.*;
import java.io.Closeable;
import java.util.ArrayList;
import java.util.Arrays;
@@ -37,11 +38,12 @@ import java.util.concurrent.atomic.AtomicBoolean;
import javax.net.SocketFactory;
-import org.apache.commons.logging.*;
-
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
-import org.apache.hadoop.io.*;
+import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolInfoService;
@@ -54,7 +56,6 @@ import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.*;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.Time;
@@ -87,7 +88,7 @@ public class RPC {
RPC_WRITABLE ((short) 2), // Use WritableRpcEngine
RPC_PROTOCOL_BUFFER ((short) 3); // Use ProtobufRpcEngine
final static short MAX_INDEX = RPC_PROTOCOL_BUFFER.value; // used for array size
- public final short value; //TODO make it private
+ private final short value;
RpcKind(short val) {
this.value = val;
@@ -207,7 +208,7 @@ public class RPC {
RpcEngine engine = PROTOCOL_ENGINES.get(protocol);
if (engine == null) {
Class<?> impl = conf.getClass(ENGINE_PROP+"."+protocol.getName(),
- WritableRpcEngine.class);
+ ProtobufRpcEngine.class);
engine = (RpcEngine)ReflectionUtils.newInstance(impl, conf);
PROTOCOL_ENGINES.put(protocol, engine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 80b3303..ea4f533 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -236,14 +236,14 @@ public abstract class Server {
static class RpcKindMapValue {
final Class<? extends Writable> rpcRequestWrapperClass;
final RpcInvoker rpcInvoker;
+
RpcKindMapValue (Class<? extends Writable> rpcRequestWrapperClass,
RpcInvoker rpcInvoker) {
this.rpcInvoker = rpcInvoker;
this.rpcRequestWrapperClass = rpcRequestWrapperClass;
}
}
- static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new
- HashMap<RPC.RpcKind, RpcKindMapValue>(4);
+ static Map<RPC.RpcKind, RpcKindMapValue> rpcKindMap = new HashMap<>(4);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index 24c75be..fe6fbe4 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -715,7 +715,7 @@ public class UserGroupInformation {
*
* @param user The principal name to load from the ticket
* cache
- * @param ticketCachePath the path to the ticket cache file
+ * @param ticketCache the path to the ticket cache file
*
* @throws IOException if the kerberos login fails
*/
@@ -775,7 +775,7 @@ public class UserGroupInformation {
/**
* Create a UserGroupInformation from a Subject with Kerberos principal.
*
- * @param user The KerberosPrincipal to use in UGI
+ * @param subject The KerberosPrincipal to use in UGI
*
* @throws IOException if the kerberos login fails
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
index eb7b949..9356dab 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/RPCCallBenchmark.java
@@ -17,13 +17,8 @@
*/
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.lang.management.ManagementFactory;
-import java.lang.management.ThreadMXBean;
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-import java.util.concurrent.atomic.AtomicLong;
-
+import com.google.common.base.Joiner;
+import com.google.protobuf.BlockingService;
import org.apache.commons.cli.CommandLine;
import org.apache.commons.cli.CommandLineParser;
import org.apache.commons.cli.GnuParser;
@@ -34,7 +29,6 @@ import org.apache.commons.cli.ParseException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.ipc.RPC.Server;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoRequestProto;
import org.apache.hadoop.ipc.protobuf.TestProtos.EchoResponseProto;
import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
@@ -45,8 +39,12 @@ import org.apache.hadoop.test.MultithreadedTestUtil.TestContext;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
-import com.google.common.base.Joiner;
-import com.google.protobuf.BlockingService;
+import java.io.IOException;
+import java.lang.management.ManagementFactory;
+import java.lang.management.ThreadMXBean;
+import java.net.InetSocketAddress;
+import java.security.PrivilegedExceptionAction;
+import java.util.concurrent.atomic.AtomicLong;
/**
* Benchmark for protobuf RPC.
@@ -68,7 +66,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
public int secondsToRun = 15;
private int msgSize = 1024;
public Class<? extends RpcEngine> rpcEngine =
- WritableRpcEngine.class;
+ ProtobufRpcEngine.class;
private MyOptions(String args[]) {
try {
@@ -135,7 +133,7 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
opts.addOption(
OptionBuilder.withLongOpt("engine").hasArg(true)
- .withArgName("writable|protobuf")
+ .withArgName("protobuf")
.withDescription("engine to use")
.create('e'));
@@ -184,8 +182,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
String eng = line.getOptionValue('e');
if ("protobuf".equals(eng)) {
rpcEngine = ProtobufRpcEngine.class;
- } else if ("writable".equals(eng)) {
- rpcEngine = WritableRpcEngine.class;
} else {
throw new ParseException("invalid engine: " + eng);
}
@@ -237,11 +233,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
server = new RPC.Builder(conf).setProtocol(TestRpcService.class)
.setInstance(service).setBindAddress(opts.host).setPort(opts.getPort())
.setNumHandlers(opts.serverThreads).setVerbose(false).build();
- } else if (opts.rpcEngine == WritableRpcEngine.class) {
- server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestRPC.TestImpl()).setBindAddress(opts.host)
- .setPort(opts.getPort()).setNumHandlers(opts.serverThreads)
- .setVerbose(false).build();
} else {
throw new RuntimeException("Bad engine: " + opts.rpcEngine);
}
@@ -399,15 +390,6 @@ public class RPCCallBenchmark extends TestRpcBase implements Tool {
return responseProto.getMessage();
}
};
- } else if (opts.rpcEngine == WritableRpcEngine.class) {
- final TestProtocol proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID, addr, conf);
- return new RpcServiceWrapper() {
- @Override
- public String doEcho(String msg) throws Exception {
- return proxy.echo(msg);
- }
- };
} else {
throw new RuntimeException("unsupported engine: " + opts.rpcEngine);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
index 8b419e3..10e23ba 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestMultipleProtocolServer.java
@@ -17,252 +17,28 @@
*/
package org.apache.hadoop.ipc;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-
-import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.protobuf.TestRpcServiceProtos.TestProtobufRpcProto;
-import org.apache.hadoop.net.NetUtils;
-import org.junit.Before;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
-import com.google.protobuf.BlockingService;
public class TestMultipleProtocolServer extends TestRpcBase {
- private static InetSocketAddress addr;
- private static RPC.Server server;
-
- private static Configuration conf = new Configuration();
-
-
- @ProtocolInfo(protocolName="Foo")
- interface Foo0 extends VersionedProtocol {
- public static final long versionID = 0L;
- String ping() throws IOException;
-
- }
-
- @ProtocolInfo(protocolName="Foo")
- interface Foo1 extends VersionedProtocol {
- public static final long versionID = 1L;
- String ping() throws IOException;
- String ping2() throws IOException;
- }
-
- @ProtocolInfo(protocolName="Foo")
- interface FooUnimplemented extends VersionedProtocol {
- public static final long versionID = 2L;
- String ping() throws IOException;
- }
-
- interface Mixin extends VersionedProtocol{
- public static final long versionID = 0L;
- void hello() throws IOException;
- }
-
- interface Bar extends Mixin {
- public static final long versionID = 0L;
- int echo(int i) throws IOException;
- }
-
- class Foo0Impl implements Foo0 {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Foo0.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public String ping() {
- return "Foo0";
- }
-
- }
-
- class Foo1Impl implements Foo1 {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Foo1.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public String ping() {
- return "Foo1";
- }
- @Override
- public String ping2() {
- return "Foo1";
-
- }
-
- }
-
-
- class BarImpl implements Bar {
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return Bar.versionID;
- }
-
- @SuppressWarnings("unchecked")
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- Class<? extends VersionedProtocol> inter;
- try {
- inter = (Class<? extends VersionedProtocol>)getClass().
- getGenericInterfaces()[0];
- } catch (Exception e) {
- throw new IOException(e);
- }
- return ProtocolSignature.getProtocolSignature(clientMethodsHash,
- getProtocolVersion(protocol, clientVersion), inter);
- }
-
- @Override
- public int echo(int i) {
- return i;
- }
-
- @Override
- public void hello() {
+ private static RPC.Server server;
-
- }
- }
@Before
public void setUp() throws Exception {
- // create a server with two handlers
- server = new RPC.Builder(conf).setProtocol(Foo0.class)
- .setInstance(new Foo0Impl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Foo1.class, new Foo1Impl());
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Bar.class, new BarImpl());
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, Mixin.class, new BarImpl());
-
-
- // Add Protobuf server
- // Create server side implementation
- PBServerImpl pbServerImpl = new PBServerImpl();
- BlockingService service = TestProtobufRpcProto
- .newReflectiveBlockingService(pbServerImpl);
- server.addProtocol(RPC.RpcKind.RPC_PROTOCOL_BUFFER, TestRpcService.class,
- service);
- server.start();
- addr = NetUtils.getConnectAddress(server);
+ super.setupConf();
+
+ server = setupTestServer(conf, 2);
}
-
+
@After
public void tearDown() throws Exception {
server.stop();
}
- @Test
- public void test1() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(Foo0.class, Foo0.versionID, addr, conf);
- Foo0 foo0 = (Foo0)proxy.getProxy();
- Assert.assertEquals("Foo0", foo0.ping());
-
-
- proxy = RPC.getProtocolProxy(Foo1.class, Foo1.versionID, addr, conf);
-
-
- Foo1 foo1 = (Foo1)proxy.getProxy();
- Assert.assertEquals("Foo1", foo1.ping());
- Assert.assertEquals("Foo1", foo1.ping());
-
-
- proxy = RPC.getProtocolProxy(Bar.class, Foo1.versionID, addr, conf);
-
-
- Bar bar = (Bar)proxy.getProxy();
- Assert.assertEquals(99, bar.echo(99));
-
- // Now test Mixin class method
-
- Mixin mixin = bar;
- mixin.hello();
- }
-
-
- // Server does not implement the FooUnimplemented version of protocol Foo.
- // See that calls to it fail.
- @Test(expected=IOException.class)
- public void testNonExistingProtocol() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(FooUnimplemented.class,
- FooUnimplemented.versionID, addr, conf);
-
- FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
- foo.ping();
- }
-
- /**
- * getProtocolVersion of an unimplemented version should return highest version
- * Similarly getProtocolSignature should work.
- * @throws IOException
- */
- @Test
- public void testNonExistingProtocol2() throws IOException {
- ProtocolProxy<?> proxy;
- proxy = RPC.getProtocolProxy(FooUnimplemented.class,
- FooUnimplemented.versionID, addr, conf);
-
- FooUnimplemented foo = (FooUnimplemented)proxy.getProxy();
- Assert.assertEquals(Foo1.versionID,
- foo.getProtocolVersion(RPC.getProtocolName(FooUnimplemented.class),
- FooUnimplemented.versionID));
- foo.getProtocolSignature(RPC.getProtocolName(FooUnimplemented.class),
- FooUnimplemented.versionID, 0);
- }
-
- @Test(expected=IOException.class)
- public void testIncorrectServerCreation() throws IOException {
- new RPC.Builder(conf).setProtocol(Foo1.class).setInstance(new Foo0Impl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(2).setVerbose(false)
- .build();
- }
-
// Now test a PB service - a server hosts both PB and Writable Rpcs.
@Test
public void testPBService() throws Exception {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
index 969f728..6d83d7d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCallBenchmark.java
@@ -26,19 +26,6 @@ import org.junit.Test;
public class TestRPCCallBenchmark {
@Test(timeout=20000)
- public void testBenchmarkWithWritable() throws Exception {
- int rc = ToolRunner.run(new RPCCallBenchmark(),
- new String[] {
- "--clientThreads", "30",
- "--serverThreads", "30",
- "--time", "5",
- "--serverReaderThreads", "4",
- "--messageSize", "1024",
- "--engine", "writable"});
- assertEquals(0, rc);
- }
-
- @Test(timeout=20000)
public void testBenchmarkWithProto() throws Exception {
int rc = ToolRunner.run(new RPCCallBenchmark(),
new String[] {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
index 2ac2be9..a06d9fd 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
@@ -18,28 +18,20 @@
package org.apache.hadoop.ipc;
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertFalse;
-import static org.junit.Assert.fail;
-
-import java.io.IOException;
-import java.lang.reflect.Method;
-import java.net.InetSocketAddress;
-
-import org.junit.Assert;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureRequestProto;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.GetProtocolSignatureResponseProto;
-import org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto.RpcErrorCodeProto;
-import org.apache.hadoop.net.NetUtils;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
+import java.io.IOException;
+import java.lang.reflect.Method;
+import java.net.InetSocketAddress;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+
/** Unit test for supporting method-name based compatible RPCs. */
public class TestRPCCompatibility {
private static final String ADDRESS = "0.0.0.0";
@@ -49,7 +41,7 @@ public class TestRPCCompatibility {
public static final Log LOG =
LogFactory.getLog(TestRPCCompatibility.class);
-
+
private static Configuration conf = new Configuration();
public interface TestProtocol0 extends VersionedProtocol {
@@ -120,6 +112,21 @@ public class TestRPCCompatibility {
@Before
public void setUp() {
ProtocolSignature.resetCache();
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol0.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol1.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol2.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol3.class, ProtobufRpcEngine.class);
+
+ RPC.setProtocolEngine(conf,
+ TestProtocol4.class, ProtobufRpcEngine.class);
}
@After
@@ -133,117 +140,7 @@ public class TestRPCCompatibility {
server = null;
}
}
-
- @Test // old client vs new server
- public void testVersion0ClientVersion1Server() throws Exception {
- // create a server with two handlers
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- proxy = RPC.getProtocolProxy(
- TestProtocol0.class, TestProtocol0.versionID, addr, conf);
-
- TestProtocol0 proxy0 = (TestProtocol0)proxy.getProxy();
- proxy0.ping();
- }
-
- @Test // old client vs new server
- public void testVersion1ClientVersion0Server() throws Exception {
- // create a server with two handlers
- server = new RPC.Builder(conf).setProtocol(TestProtocol0.class)
- .setInstance(new TestImpl0()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- proxy = RPC.getProtocolProxy(
- TestProtocol1.class, TestProtocol1.versionID, addr, conf);
-
- TestProtocol1 proxy1 = (TestProtocol1)proxy.getProxy();
- proxy1.ping();
- try {
- proxy1.echo("hello");
- fail("Echo should fail");
- } catch(IOException e) {
- }
- }
-
- private class Version2Client {
- private TestProtocol2 proxy2;
- private ProtocolProxy<TestProtocol2> serverInfo;
-
- private Version2Client() throws IOException {
- serverInfo = RPC.getProtocolProxy(
- TestProtocol2.class, TestProtocol2.versionID, addr, conf);
- proxy2 = serverInfo.getProxy();
- }
-
- public int echo(int value) throws IOException, NumberFormatException {
- if (serverInfo.isMethodSupported("echo", int.class)) {
-System.out.println("echo int is supported");
- return -value; // use version 3 echo long
- } else { // server is version 2
-System.out.println("echo int is NOT supported");
- return Integer.parseInt(proxy2.echo(String.valueOf(value)));
- }
- }
-
- public String echo(String value) throws IOException {
- return proxy2.echo(value);
- }
-
- public void ping() throws IOException {
- proxy2.ping();
- }
- }
-
- @Test // Compatible new client & old server
- public void testVersion2ClientVersion1Server() throws Exception {
- // create a server with two handlers
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
-
- Version2Client client = new Version2Client();
- client.ping();
- assertEquals("hello", client.echo("hello"));
-
- // echo(int) is not supported by server, so returning 3
- // This verifies that echo(int) and echo(String)'s hash codes are different
- assertEquals(3, client.echo(3));
- }
-
- @Test // equal version client and server
- public void testVersion2ClientVersion2Server() throws Exception {
- // create a server with two handlers
- TestImpl2 impl = new TestImpl2();
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- Version2Client client = new Version2Client();
-
- client.ping();
- assertEquals("hello", client.echo("hello"));
-
- // now that echo(int) is supported by the server, echo(int) should return -3
- assertEquals(-3, client.echo(3));
- }
-
public interface TestProtocol3 {
int echo(String value);
int echo(int value);
@@ -297,97 +194,4 @@ System.out.println("echo int is NOT supported");
@Override
int echo(int value) throws IOException;
}
-
- @Test
- public void testVersionMismatch() throws IOException {
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- TestProtocol4 proxy = RPC.getProxy(TestProtocol4.class,
- TestProtocol4.versionID, addr, conf);
- try {
- proxy.echo(21);
- fail("The call must throw VersionMismatch exception");
- } catch (RemoteException ex) {
- Assert.assertEquals(RPC.VersionMismatch.class.getName(),
- ex.getClassName());
- Assert.assertTrue(ex.getErrorCode().equals(
- RpcErrorCodeProto.ERROR_RPC_VERSION_MISMATCH));
- } catch (IOException ex) {
- fail("Expected version mismatch but got " + ex);
- }
- }
-
- @Test
- public void testIsMethodSupported() throws IOException {
- server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
- .setInstance(new TestImpl2()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
- server.start();
- addr = NetUtils.getConnectAddress(server);
-
- TestProtocol2 proxy = RPC.getProxy(TestProtocol2.class,
- TestProtocol2.versionID, addr, conf);
- boolean supported = RpcClientUtil.isMethodSupported(proxy,
- TestProtocol2.class, RPC.RpcKind.RPC_WRITABLE,
- RPC.getProtocolVersion(TestProtocol2.class), "echo");
- Assert.assertTrue(supported);
- supported = RpcClientUtil.isMethodSupported(proxy,
- TestProtocol2.class, RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- RPC.getProtocolVersion(TestProtocol2.class), "echo");
- Assert.assertFalse(supported);
- }
-
- /**
- * Verify that ProtocolMetaInfoServerSideTranslatorPB correctly looks up
- * the server registry to extract protocol signatures and versions.
- */
- @Test
- public void testProtocolMetaInfoSSTranslatorPB() throws Exception {
- TestImpl1 impl = new TestImpl1();
- server = new RPC.Builder(conf).setProtocol(TestProtocol1.class)
- .setInstance(impl).setBindAddress(ADDRESS).setPort(0).setNumHandlers(2)
- .setVerbose(false).build();
- server.addProtocol(RPC.RpcKind.RPC_WRITABLE, TestProtocol0.class, impl);
- server.start();
-
- ProtocolMetaInfoServerSideTranslatorPB xlator =
- new ProtocolMetaInfoServerSideTranslatorPB(server);
-
- GetProtocolSignatureResponseProto resp = xlator.getProtocolSignature(
- null,
- createGetProtocolSigRequestProto(TestProtocol1.class,
- RPC.RpcKind.RPC_PROTOCOL_BUFFER));
- //No signatures should be found
- Assert.assertEquals(0, resp.getProtocolSignatureCount());
- resp = xlator.getProtocolSignature(
- null,
- createGetProtocolSigRequestProto(TestProtocol1.class,
- RPC.RpcKind.RPC_WRITABLE));
- Assert.assertEquals(1, resp.getProtocolSignatureCount());
- ProtocolSignatureProto sig = resp.getProtocolSignatureList().get(0);
- Assert.assertEquals(TestProtocol1.versionID, sig.getVersion());
- boolean found = false;
- int expected = ProtocolSignature.getFingerprint(TestProtocol1.class
- .getMethod("echo", String.class));
- for (int m : sig.getMethodsList()) {
- if (expected == m) {
- found = true;
- break;
- }
- }
- Assert.assertTrue(found);
- }
-
- private GetProtocolSignatureRequestProto createGetProtocolSigRequestProto(
- Class<?> protocol, RPC.RpcKind rpcKind) {
- GetProtocolSignatureRequestProto.Builder builder =
- GetProtocolSignatureRequestProto.newBuilder();
- builder.setProtocol(protocol.getName());
- builder.setRpcKind(rpcKind.toString());
- return builder.build();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
index 5807998..b22f91b 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCWaitForProxy.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.ipc;
import org.apache.hadoop.conf.Configuration;
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.*;
-import org.apache.hadoop.ipc.TestRPC.TestProtocol;
import org.junit.Assert;
import org.junit.Test;
import org.slf4j.Logger;
@@ -30,11 +28,13 @@ import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.nio.channels.ClosedByInterruptException;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_KEY;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY;
+
/**
* tests that the proxy can be interrupted
*/
-public class TestRPCWaitForProxy extends Assert {
- private static final String ADDRESS = "0.0.0.0";
+public class TestRPCWaitForProxy extends TestRpcBase {
private static final Logger
LOG = LoggerFactory.getLogger(TestRPCWaitForProxy.class);
@@ -46,14 +46,15 @@ public class TestRPCWaitForProxy extends Assert {
*
* @throws Throwable any exception other than that which was expected
*/
- @Test(timeout = 10000)
+ @Test(timeout = 50000)
public void testWaitForProxy() throws Throwable {
RpcThread worker = new RpcThread(0);
worker.start();
worker.join();
Throwable caught = worker.getCaught();
- assertNotNull("No exception was raised", caught);
- if (!(caught instanceof ConnectException)) {
+ Throwable cause = caught.getCause();
+ Assert.assertNotNull("No exception was raised", cause);
+ if (!(cause instanceof ConnectException)) {
throw caught;
}
}
@@ -69,11 +70,11 @@ public class TestRPCWaitForProxy extends Assert {
RpcThread worker = new RpcThread(100);
worker.start();
Thread.sleep(1000);
- assertTrue("worker hasn't started", worker.waitStarted);
+ Assert.assertTrue("worker hasn't started", worker.waitStarted);
worker.interrupt();
worker.join();
Throwable caught = worker.getCaught();
- assertNotNull("No exception was raised", caught);
+ Assert.assertNotNull("No exception was raised", caught);
// looking for the root cause here, which can be wrapped
// as part of the NetUtils work. Having this test look
// a the type of exception there would be brittle to improvements
@@ -82,6 +83,8 @@ public class TestRPCWaitForProxy extends Assert {
if (cause == null) {
// no inner cause, use outer exception as root cause.
cause = caught;
+ } else if (cause.getCause() != null) {
+ cause = cause.getCause();
}
if (!(cause instanceof InterruptedIOException)
&& !(cause instanceof ClosedByInterruptException)) {
@@ -112,12 +115,16 @@ public class TestRPCWaitForProxy extends Assert {
IPC_CLIENT_CONNECT_MAX_RETRIES_ON_SOCKET_TIMEOUTS_KEY,
connectRetries);
waitStarted = true;
- TestProtocol proxy = RPC.waitForProxy(TestProtocol.class,
- TestProtocol.versionID,
- new InetSocketAddress(ADDRESS, 20),
- config,
- 15000L);
- proxy.echo("");
+
+ short invalidPort = 20;
+ InetSocketAddress invalidAddress = new InetSocketAddress(ADDRESS,
+ invalidPort);
+ TestRpcBase.TestRpcService proxy = RPC.getProxy(
+ TestRpcBase.TestRpcService.class,
+ 1L, invalidAddress, conf);
+ // Test echo method
+ proxy.echo(null, newEchoRequest("hello"));
+
} catch (Throwable throwable) {
caught = throwable;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
index bc604a4..5a8f8d0 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRpcBase.java
@@ -112,7 +112,8 @@ public class TestRpcBase {
return setupTestServer(builder);
}
- protected static RPC.Server setupTestServer(RPC.Builder builder) throws IOException {
+ protected static RPC.Server setupTestServer(
+ RPC.Builder builder) throws IOException {
RPC.Server server = builder.build();
server.start();
@@ -175,17 +176,21 @@ public class TestRpcBase {
public TestTokenIdentifier() {
this(new Text(), new Text());
}
+
public TestTokenIdentifier(Text tokenid) {
this(tokenid, new Text());
}
+
public TestTokenIdentifier(Text tokenid, Text realUser) {
this.tokenid = tokenid == null ? new Text() : tokenid;
this.realUser = realUser == null ? new Text() : realUser;
}
+
@Override
public Text getKind() {
return KIND_NAME;
}
+
@Override
public UserGroupInformation getUser() {
if (realUser.toString().isEmpty()) {
@@ -203,6 +208,7 @@ public class TestRpcBase {
tokenid.readFields(in);
realUser.readFields(in);
}
+
@Override
public void write(DataOutput out) throws IOException {
tokenid.write(out);
@@ -234,7 +240,7 @@ public class TestRpcBase {
@SuppressWarnings("unchecked")
@Override
public Token<TestTokenIdentifier> selectToken(Text service,
- Collection<Token<? extends TokenIdentifier>> tokens) {
+ Collection<Token<? extends TokenIdentifier>> tokens) {
if (service == null) {
return null;
}
@@ -388,19 +394,17 @@ public class TestRpcBase {
}
@Override
- public TestProtos.AuthUserResponseProto getAuthUser(
+ public TestProtos.UserResponseProto getAuthUser(
RpcController controller, TestProtos.EmptyRequestProto request)
throws ServiceException {
- UserGroupInformation authUser = null;
+ UserGroupInformation authUser;
try {
authUser = UserGroupInformation.getCurrentUser();
} catch (IOException e) {
throw new ServiceException(e);
}
- return TestProtos.AuthUserResponseProto.newBuilder()
- .setAuthUser(authUser.getUserName())
- .build();
+ return newUserResponse(authUser.getUserName());
}
@Override
@@ -432,6 +436,34 @@ public class TestRpcBase {
return TestProtos.EmptyResponseProto.newBuilder().build();
}
+
+ @Override
+ public TestProtos.UserResponseProto getCurrentUser(
+ RpcController controller,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ String user;
+ try {
+ user = UserGroupInformation.getCurrentUser().toString();
+ } catch (IOException e) {
+ throw new ServiceException("Failed to get current user", e);
+ }
+
+ return newUserResponse(user);
+ }
+
+ @Override
+ public TestProtos.UserResponseProto getServerRemoteUser(
+ RpcController controller,
+ TestProtos.EmptyRequestProto request) throws ServiceException {
+ String serverRemoteUser = Server.getRemoteUser().toString();
+ return newUserResponse(serverRemoteUser);
+ }
+
+ private TestProtos.UserResponseProto newUserResponse(String user) {
+ return TestProtos.UserResponseProto.newBuilder()
+ .setUser(user)
+ .build();
+ }
}
protected static TestProtos.EmptyRequestProto newEmptyRequest() {
@@ -478,8 +510,4 @@ public class TestRpcBase {
}
return null;
}
-
- protected static String convert(TestProtos.AuthUserResponseProto response) {
- return response.getAuthUser();
- }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
index 72371a7..c48ff2e 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestSaslRPC.java
@@ -45,30 +45,55 @@ import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
import org.junit.runners.Parameterized.Parameters;
-import javax.security.auth.callback.*;
-import javax.security.sasl.*;
+import javax.security.auth.callback.Callback;
+import javax.security.auth.callback.CallbackHandler;
+import javax.security.auth.callback.NameCallback;
+import javax.security.auth.callback.PasswordCallback;
+import javax.security.auth.callback.UnsupportedCallbackException;
+import javax.security.sasl.AuthorizeCallback;
+import javax.security.sasl.Sasl;
+import javax.security.sasl.SaslClient;
+import javax.security.sasl.SaslException;
+import javax.security.sasl.SaslServer;
import java.io.IOException;
import java.lang.annotation.Annotation;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.security.PrivilegedExceptionAction;
import java.security.Security;
-import java.util.*;
-import java.util.concurrent.*;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_RPC_PROTECTION;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.KERBEROS;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.SIMPLE;
+import static org.apache.hadoop.security.SaslRpcServer.AuthMethod.TOKEN;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
/** Unit tests for using Sasl over RPC. */
@RunWith(Parameterized.class)
public class TestSaslRPC extends TestRpcBase {
@Parameters
public static Collection<Object[]> data() {
- Collection<Object[]> params = new ArrayList<Object[]>();
+ Collection<Object[]> params = new ArrayList<>();
for (QualityOfProtection qop : QualityOfProtection.values()) {
params.add(new Object[]{ new QualityOfProtection[]{qop},qop, null });
}
@@ -114,7 +139,7 @@ public class TestSaslRPC extends TestRpcBase {
NONE(),
VALID(),
INVALID(),
- OTHER();
+ OTHER()
}
@BeforeClass
@@ -230,7 +255,7 @@ public class TestSaslRPC extends TestRpcBase {
final Server server = setupTestServer(conf, 5, sm);
doDigestRpc(server, sm);
} finally {
- SecurityUtil.setSecurityInfoProviders(new SecurityInfo[0]);
+ SecurityUtil.setSecurityInfoProviders();
}
}
@@ -259,7 +284,7 @@ public class TestSaslRPC extends TestRpcBase {
addr = NetUtils.getConnectAddress(server);
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId, sm);
+ Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
@@ -296,8 +321,8 @@ public class TestSaslRPC extends TestRpcBase {
// set doPing to true
newConf.setBoolean(CommonConfigurationKeys.IPC_CLIENT_PING_KEY, true);
- ConnectionId remoteId = ConnectionId.getConnectionId(new InetSocketAddress(0),
- TestRpcService.class, null, 0, null, newConf);
+ ConnectionId remoteId = ConnectionId.getConnectionId(
+ new InetSocketAddress(0), TestRpcService.class, null, 0, null, newConf);
assertEquals(CommonConfigurationKeys.IPC_PING_INTERVAL_DEFAULT,
remoteId.getPingInterval());
// set doPing to false
@@ -806,13 +831,13 @@ public class TestSaslRPC extends TestRpcBase {
final TestTokenSecretManager sm = new TestTokenSecretManager();
boolean useSecretManager = (serverAuth != SIMPLE);
if (enableSecretManager != null) {
- useSecretManager &= enableSecretManager.booleanValue();
+ useSecretManager &= enableSecretManager;
}
if (forceSecretManager != null) {
- useSecretManager |= forceSecretManager.booleanValue();
+ useSecretManager |= forceSecretManager;
}
final SecretManager<?> serverSm = useSecretManager ? sm : null;
-
+
Server server = serverUgi.doAs(new PrivilegedExceptionAction<Server>() {
@Override
public Server run() throws IOException {
@@ -867,13 +892,13 @@ public class TestSaslRPC extends TestRpcBase {
proxy.ping(null, newEmptyRequest());
// make sure the other side thinks we are who we said we are!!!
assertEquals(clientUgi.getUserName(),
- convert(proxy.getAuthUser(null, newEmptyRequest())));
+ proxy.getAuthUser(null, newEmptyRequest()).getUser());
AuthMethod authMethod =
convert(proxy.getAuthMethod(null, newEmptyRequest()));
// verify sasl completed with correct QOP
assertEquals((authMethod != SIMPLE) ? expectedQop.saslQop : null,
- RPC.getConnectionIdForProxy(proxy).getSaslQop());
- return authMethod.toString();
+ RPC.getConnectionIdForProxy(proxy).getSaslQop());
+ return authMethod != null ? authMethod.toString() : null;
} catch (ServiceException se) {
if (se.getCause() instanceof RemoteException) {
throw (RemoteException) se.getCause();
@@ -898,21 +923,18 @@ public class TestSaslRPC extends TestRpcBase {
String actual) {
assertEquals(expect.toString(), actual);
}
-
- private static void assertAuthEquals(Pattern expect,
- String actual) {
+
+ private static void assertAuthEquals(Pattern expect, String actual) {
// this allows us to see the regexp and the value it didn't match
if (!expect.matcher(actual).matches()) {
- assertEquals(expect, actual); // it failed
- } else {
- assertTrue(true); // it matched
+ fail(); // it failed
}
}
/*
* Class used to test overriding QOP values using SaslPropertiesResolver
*/
- static class AuthSaslPropertiesResolver extends SaslPropertiesResolver{
+ static class AuthSaslPropertiesResolver extends SaslPropertiesResolver {
@Override
public Map<String, String> getServerProperties(InetAddress address) {
@@ -921,7 +943,7 @@ public class TestSaslRPC extends TestRpcBase {
return newPropertes;
}
}
-
+
public static void main(String[] args) throws Exception {
System.out.println("Testing Kerberos authentication over RPC");
if (args.length != 2) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
index 50d389c..c4dbcac 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestDoAsEffectiveUser.java
@@ -17,40 +17,35 @@
*/
package org.apache.hadoop.security;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.net.NetworkInterface;
-import java.security.PrivilegedExceptionAction;
-import java.util.ArrayList;
-import java.util.Enumeration;
-
-import org.junit.Assert;
-
+import com.google.protobuf.ServiceException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.ProtocolSignature;
+import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.ipc.VersionedProtocol;
-import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.ipc.TestRpcBase;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authorize.DefaultImpersonationProvider;
import org.apache.hadoop.security.authorize.ProxyUsers;
import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.security.token.TokenInfo;
+import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenSecretManager;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
-import org.apache.hadoop.ipc.TestRpcBase.TestTokenSelector;
-import org.apache.commons.logging.*;
-import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+
+import java.io.IOException;
+import java.net.InetAddress;
+import java.net.NetworkInterface;
+import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
+import java.util.Enumeration;
/**
- *
+ * Test do as effective user.
*/
-public class TestDoAsEffectiveUser {
+public class TestDoAsEffectiveUser extends TestRpcBase {
final private static String REAL_USER_NAME = "realUser1@HADOOP.APACHE.ORG";
final private static String REAL_USER_SHORT_NAME = "realUser1";
final private static String PROXY_USER_NAME = "proxyUser";
@@ -58,8 +53,8 @@ public class TestDoAsEffectiveUser {
final private static String GROUP2_NAME = "group2";
final private static String[] GROUP_NAMES = new String[] { GROUP1_NAME,
GROUP2_NAME };
- private static final String ADDRESS = "0.0.0.0";
- private TestProtocol proxy;
+
+ private TestRpcService client;
private static final Configuration masterConf = new Configuration();
@@ -82,7 +77,7 @@ public class TestDoAsEffectiveUser {
private void configureSuperUserIPAddresses(Configuration conf,
String superUserShortName) throws IOException {
- ArrayList<String> ipList = new ArrayList<String>();
+ ArrayList<String> ipList = new ArrayList<>();
Enumeration<NetworkInterface> netInterfaceList = NetworkInterface
.getNetworkInterfaces();
while (netInterfaceList.hasMoreElements()) {
@@ -130,50 +125,19 @@ public class TestDoAsEffectiveUser {
curUGI.toString());
}
- @TokenInfo(TestTokenSelector.class)
- public interface TestProtocol extends VersionedProtocol {
- public static final long versionID = 1L;
-
- String aMethod() throws IOException;
- String getServerRemoteUser() throws IOException;
- }
-
- public class TestImpl implements TestProtocol {
-
- @Override
- public String aMethod() throws IOException {
- return UserGroupInformation.getCurrentUser().toString();
- }
-
- @Override
- public String getServerRemoteUser() throws IOException {
- return Server.getRemoteUser().toString();
- }
-
- @Override
- public long getProtocolVersion(String protocol, long clientVersion)
- throws IOException {
- return TestProtocol.versionID;
- }
-
- @Override
- public ProtocolSignature getProtocolSignature(String protocol,
- long clientVersion, int clientMethodsHash) throws IOException {
- return new ProtocolSignature(TestProtocol.versionID, null);
- }
- }
-
- private void checkRemoteUgi(final Server server,
- final UserGroupInformation ugi, final Configuration conf)
- throws Exception {
+ private void checkRemoteUgi(final UserGroupInformation ugi,
+ final Configuration conf) throws Exception {
ugi.doAs(new PrivilegedExceptionAction<Void>() {
@Override
- public Void run() throws IOException {
- proxy = RPC.getProxy(
- TestProtocol.class, TestProtocol.versionID,
- NetUtils.getConnectAddress(server), conf);
- Assert.assertEquals(ugi.toString(), proxy.aMethod());
- Assert.assertEquals(ugi.toString(), proxy.getServerRemoteUser());
+ public Void run() throws ServiceException {
+ client = getClient(addr, conf);
+ String currentUser = client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
+ String serverRemoteUser = client.getServerRemoteUser(null,
+ newEmptyRequest()).getUser();
+
+ Assert.assertEquals(ugi.toString(), currentUser);
+ Assert.assertEquals(ugi.toString(), serverRemoteUser);
return null;
}
});
@@ -185,29 +149,27 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).build();
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(server, realUserUgi, conf);
+ checkRemoteUgi(realUserUgi, conf);
- UserGroupInformation proxyUserUgi = UserGroupInformation.createProxyUserForTesting(
+ UserGroupInformation proxyUserUgi =
+ UserGroupInformation.createProxyUserForTesting(
PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(server, proxyUserUgi, conf);
+ checkRemoteUgi(proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -218,29 +180,25 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
- checkRemoteUgi(server, realUserUgi, conf);
+ checkRemoteUgi(realUserUgi, conf);
UserGroupInformation proxyUserUgi = UserGroupInformation
.createProxyUserForTesting(PROXY_USER_NAME, realUserUgi, GROUP_NAMES);
- checkRemoteUgi(server, proxyUserUgi, conf);
+ checkRemoteUgi(proxyUserUgi, conf);
} catch (Exception e) {
e.printStackTrace();
Assert.fail();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -256,17 +214,14 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 5);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -275,11 +230,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -287,10 +241,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -299,17 +250,14 @@ public class TestDoAsEffectiveUser {
final Configuration conf = new Configuration();
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME), "group1");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -318,11 +266,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -330,10 +277,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -341,15 +285,12 @@ public class TestDoAsEffectiveUser {
public void testRealUserGroupNotSpecified() throws IOException {
final Configuration conf = new Configuration();
configureSuperUserIPAddresses(conf, REAL_USER_SHORT_NAME);
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -358,11 +299,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = (TestProtocol) RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -370,10 +310,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -384,17 +321,14 @@ public class TestDoAsEffectiveUser {
conf.setStrings(DefaultImpersonationProvider.getTestProvider().
getProxySuperuserGroupConfKey(REAL_USER_SHORT_NAME),
"group3");
- Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(2).setVerbose(false).build();
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
+ UserGroupInformation.setConfiguration(conf);
+ final Server server = setupTestServer(conf, 2);
refreshConf(conf);
try {
- server.start();
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
-
UserGroupInformation realUserUgi = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
@@ -403,11 +337,10 @@ public class TestDoAsEffectiveUser {
String retVal = proxyUserUgi
.doAs(new PrivilegedExceptionAction<String>() {
@Override
- public String run() throws IOException {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ public String run() throws ServiceException {
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
}
});
@@ -415,10 +348,7 @@ public class TestDoAsEffectiveUser {
} catch (Exception e) {
e.printStackTrace();
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
@@ -432,20 +362,17 @@ public class TestDoAsEffectiveUser {
final Configuration conf = new Configuration(masterConf);
TestTokenSecretManager sm = new TestTokenSecretManager();
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ RPC.setProtocolEngine(conf, TestRpcService.class,
+ ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(conf);
- final Server server = new RPC.Builder(conf).setProtocol(TestProtocol.class)
- .setInstance(new TestImpl()).setBindAddress(ADDRESS).setPort(0)
- .setNumHandlers(5).setVerbose(true).setSecretManager(sm).build();
-
- server.start();
+ final Server server = setupTestServer(conf, 5, sm);
final UserGroupInformation current = UserGroupInformation
.createRemoteUser(REAL_USER_NAME);
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
+ Token<TestTokenIdentifier> token = new Token<>(tokenId,
sm);
SecurityUtil.setTokenService(token, addr);
UserGroupInformation proxyUserUgi = UserGroupInformation
@@ -453,23 +380,19 @@ public class TestDoAsEffectiveUser {
proxyUserUgi.addToken(token);
refreshConf(conf);
-
+
String retVal = proxyUserUgi.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, conf);
- String ret = proxy.aMethod();
- return ret;
+ client = getClient(addr, conf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
});
@@ -486,42 +409,34 @@ public class TestDoAsEffectiveUser {
TestTokenSecretManager sm = new TestTokenSecretManager();
final Configuration newConf = new Configuration(masterConf);
SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, newConf);
+ // Set RPC engine to protobuf RPC engine
+ RPC.setProtocolEngine(newConf, TestRpcService.class,
+ ProtobufRpcEngine.class);
UserGroupInformation.setConfiguration(newConf);
- final Server server = new RPC.Builder(newConf)
- .setProtocol(TestProtocol.class).setInstance(new TestImpl())
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(sm).build();
-
- server.start();
+ final Server server = setupTestServer(newConf, 5, sm);
final UserGroupInformation current = UserGroupInformation
.createUserForTesting(REAL_USER_NAME, GROUP_NAMES);
refreshConf(newConf);
-
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
+
TestTokenIdentifier tokenId = new TestTokenIdentifier(new Text(current
.getUserName()), new Text("SomeSuperUser"));
- Token<TestTokenIdentifier> token = new Token<TestTokenIdentifier>(tokenId,
- sm);
+ Token<TestTokenIdentifier> token = new Token<>(tokenId, sm);
SecurityUtil.setTokenService(token, addr);
current.addToken(token);
String retVal = current.doAs(new PrivilegedExceptionAction<String>() {
@Override
public String run() throws Exception {
try {
- proxy = RPC.getProxy(TestProtocol.class,
- TestProtocol.versionID, addr, newConf);
- String ret = proxy.aMethod();
- return ret;
+ client = getClient(addr, newConf);
+ return client.getCurrentUser(null,
+ newEmptyRequest()).getUser();
} catch (Exception e) {
e.printStackTrace();
throw e;
} finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
+ stop(server, client);
}
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index ef5c6a2..a306d35 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -20,6 +20,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
+import org.apache.hadoop.ipc.TestRpcBase.TestTokenIdentifier;
import org.apache.hadoop.metrics2.MetricsRecordBuilder;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
@@ -28,7 +29,11 @@ import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
-import org.junit.*;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
@@ -49,9 +54,22 @@ import java.util.Set;
import static org.apache.hadoop.fs.CommonConfigurationKeys.HADOOP_USER_GROUP_METRICS_PERCENTILES_INTERVALS;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTH_TO_LOCAL;
-import static org.apache.hadoop.ipc.TestSaslRPC.*;
-import static org.apache.hadoop.test.MetricsAsserts.*;
-import static org.junit.Assert.*;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounter;
+import static org.apache.hadoop.test.MetricsAsserts.assertCounterGt;
+import static org.apache.hadoop.test.MetricsAsserts.assertGaugeGt;
+import static org.apache.hadoop.test.MetricsAsserts.assertQuantileGauges;
+import static org.apache.hadoop.test.MetricsAsserts.getDoubleGauge;
+import static org.apache.hadoop.test.MetricsAsserts.getMetrics;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNotSame;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertSame;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -108,7 +126,7 @@ public class TestUserGroupInformation {
UserGroupInformation.setLoginUser(null);
}
- @Test (timeout = 30000)
+ @Test(timeout = 30000)
public void testSimpleLogin() throws IOException {
tryLoginAuthenticationMethod(AuthenticationMethod.SIMPLE, true);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/proto/test.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test.proto b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
index 99cd93d..6411f97 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test.proto
@@ -88,6 +88,6 @@ message AuthMethodResponseProto {
required string mechanismName = 2;
}
-message AuthUserResponseProto {
- required string authUser = 1;
+message UserResponseProto {
+ required string user = 1;
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
index 3292115..06f6c4f 100644
--- a/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
+++ b/hadoop-common-project/hadoop-common/src/test/proto/test_rpc_service.proto
@@ -40,9 +40,11 @@ service TestProtobufRpcProto {
rpc exchange(ExchangeRequestProto) returns (ExchangeResponseProto);
rpc sleep(SleepRequestProto) returns (EmptyResponseProto);
rpc getAuthMethod(EmptyRequestProto) returns (AuthMethodResponseProto);
- rpc getAuthUser(EmptyRequestProto) returns (AuthUserResponseProto);
+ rpc getAuthUser(EmptyRequestProto) returns (UserResponseProto);
rpc echoPostponed(EchoRequestProto) returns (EchoResponseProto);
rpc sendPostponed(EmptyRequestProto) returns (EmptyResponseProto);
+ rpc getCurrentUser(EmptyRequestProto) returns (UserResponseProto);
+ rpc getServerRemoteUser(EmptyRequestProto) returns (UserResponseProto);
}
service TestProtobufRpc2Proto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 6b52949..57f7cb1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -168,7 +168,6 @@ import org.apache.hadoop.ipc.RetryCache.CacheEntry;
import org.apache.hadoop.ipc.RetryCache.CacheEntryWithPayload;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.ipc.StandbyException;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.ipc.RefreshRegistry;
import org.apache.hadoop.ipc.RefreshResponse;
import org.apache.hadoop.net.Node;
@@ -317,8 +316,6 @@ public class NameNodeRpcServer implements NamenodeProtocols {
new TraceAdminProtocolServerSideTranslatorPB(this);
BlockingService traceAdminService = TraceAdminService
.newReflectiveBlockingService(traceAdminXlator);
-
- WritableRpcEngine.ensureInitialized();
InetSocketAddress serviceRpcAddr = nn.getServiceRpcServerAddress(conf);
if (serviceRpcAddr != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
deleted file mode 100644
index 0b7ee33..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/security/TestClientProtocolWithDelegationToken.java
+++ /dev/null
@@ -1,119 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.security;
-
-import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION;
-import static org.mockito.Mockito.mock;
-
-import java.net.InetSocketAddress;
-import java.security.PrivilegedExceptionAction;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
-import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenSecretManager;
-import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
-import org.apache.hadoop.io.Text;
-import org.apache.hadoop.ipc.Client;
-import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.Server;
-import org.apache.hadoop.net.NetUtils;
-import org.apache.hadoop.security.SaslInputStream;
-import org.apache.hadoop.security.SaslRpcClient;
-import org.apache.hadoop.security.SaslRpcServer;
-import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
-import org.apache.hadoop.test.GenericTestUtils;
-import org.apache.log4j.Level;
-import org.junit.Test;
-
-/** Unit tests for using Delegation Token over RPC. */
-public class TestClientProtocolWithDelegationToken {
- private static final String ADDRESS = "0.0.0.0";
-
- public static final Log LOG = LogFactory
- .getLog(TestClientProtocolWithDelegationToken.class);
-
- private static final Configuration conf;
- static {
- conf = new Configuration();
- conf.set(HADOOP_SECURITY_AUTHENTICATION, "kerberos");
- UserGroupInformation.setConfiguration(conf);
- }
-
- static {
- GenericTestUtils.setLogLevel(Client.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(Server.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslRpcClient.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslRpcServer.LOG, Level.ALL);
- GenericTestUtils.setLogLevel(SaslInputStream.LOG, Level.ALL);
- }
-
- @Test
- public void testDelegationTokenRpc() throws Exception {
- ClientProtocol mockNN = mock(ClientProtocol.class);
- FSNamesystem mockNameSys = mock(FSNamesystem.class);
-
- DelegationTokenSecretManager sm = new DelegationTokenSecretManager(
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_KEY_UPDATE_INTERVAL_DEFAULT,
- DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_DEFAULT,
- 3600000, mockNameSys);
- sm.startThreads();
- final Server server = new RPC.Builder(conf)
- .setProtocol(ClientProtocol.class).setInstance(mockNN)
- .setBindAddress(ADDRESS).setPort(0).setNumHandlers(5).setVerbose(true)
- .setSecretManager(sm).build();
-
- server.start();
-
- final UserGroupInformation current = UserGroupInformation.getCurrentUser();
- final InetSocketAddress addr = NetUtils.getConnectAddress(server);
- String user = current.getUserName();
- Text owner = new Text(user);
- DelegationTokenIdentifier dtId = new DelegationTokenIdentifier(owner, owner, null);
- Token<DelegationTokenIdentifier> token = new Token<DelegationTokenIdentifier>(
- dtId, sm);
- SecurityUtil.setTokenService(token, addr);
- LOG.info("Service for token is " + token.getService());
- current.addToken(token);
- current.doAs(new PrivilegedExceptionAction<Object>() {
- @Override
- public Object run() throws Exception {
- ClientProtocol proxy = null;
- try {
- proxy = RPC.getProxy(ClientProtocol.class,
- ClientProtocol.versionID, addr, conf);
- proxy.getServerDefaults();
- } finally {
- server.stop();
- if (proxy != null) {
- RPC.stopProxy(proxy);
- }
- }
- return null;
- }
- });
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62a96671/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
index 3fef5e2..729af0a 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/server/HSAdminServer.java
@@ -29,7 +29,6 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.ipc.ProtobufRpcEngine;
import org.apache.hadoop.ipc.RPC;
-import org.apache.hadoop.ipc.WritableRpcEngine;
import org.apache.hadoop.mapreduce.v2.jobhistory.JHAdminConfig;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.security.Groups;
@@ -98,8 +97,6 @@ public class HSAdminServer extends AbstractService implements HSAdminProtocol {
BlockingService refreshHSAdminProtocolService = HSAdminRefreshProtocolService
.newReflectiveBlockingService(refreshHSAdminProtocolXlator);
- WritableRpcEngine.ensureInitialized();
-
clientRpcAddress = conf.getSocketAddr(
JHAdminConfig.MR_HISTORY_BIND_HOST,
JHAdminConfig.JHS_ADMIN_ADDRESS,
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/48] hadoop git commit: YARN-5555. Scheduler UI: "% of Queue" is
inaccurate if leaf queue is hierarchically nested. Contributed by Eric Payne.
Posted by aw...@apache.org.
YARN-5555. Scheduler UI: "% of Queue" is inaccurate if leaf queue is hierarchically nested. Contributed by Eric Payne.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/05f5c0f6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/05f5c0f6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/05f5c0f6
Branch: refs/heads/HADOOP-13341
Commit: 05f5c0f631680cffc36a79550c351620615445db
Parents: 0690f09
Author: Varun Vasudev <vv...@apache.org>
Authored: Fri Sep 2 16:02:01 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Fri Sep 2 16:02:01 2016 +0530
----------------------------------------------------------------------
.../scheduler/common/fica/FiCaSchedulerApp.java | 27 ++++++
.../scheduler/capacity/TestLeafQueue.java | 87 ++++++++++++++++++++
2 files changed, 114 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05f5c0f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
index 33dee80..9c84a23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
@@ -29,6 +29,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Container;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -57,6 +58,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.activities.ActivitiesManager;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.AbstractCSQueue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAMContainerLaunchDiagnosticsConstants;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CSAssignment;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityHeadroomProvider;
@@ -617,4 +619,29 @@ public class FiCaSchedulerApp extends SchedulerApplicationAttempt {
updateAMContainerDiagnostics(AMState.ACTIVATED, diagnosticMessageBldr.toString());
}
}
+
+ /**
+ * Recalculates the per-app, percent of queue metric, specific to the
+ * Capacity Scheduler.
+ */
+ @Override
+ public synchronized ApplicationResourceUsageReport getResourceUsageReport() {
+ ApplicationResourceUsageReport report = super.getResourceUsageReport();
+ Resource cluster = rmContext.getScheduler().getClusterResource();
+ Resource totalPartitionRes =
+ rmContext.getNodeLabelManager()
+ .getResourceByLabel(getAppAMNodePartitionName(), cluster);
+ ResourceCalculator calc = rmContext.getScheduler().getResourceCalculator();
+ if (!calc.isInvalidDivisor(totalPartitionRes)) {
+ float queueAbsMaxCapPerPartition =
+ ((AbstractCSQueue)getQueue()).getQueueCapacities()
+ .getAbsoluteCapacity(getAppAMNodePartitionName());
+ float queueUsagePerc =
+ calc.divide(totalPartitionRes, report.getUsedResources(),
+ Resources.multiply(totalPartitionRes,
+ queueAbsMaxCapPerPartition)) * 100;
+ report.setQueueUsagePercentage(queueUsagePerc);
+ }
+ return report;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/05f5c0f6/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
index b2c53da..9134889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
@@ -51,6 +51,8 @@ import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.QueueACL;
+import org.apache.hadoop.yarn.api.records.QueueInfo;
+import org.apache.hadoop.yarn.api.records.QueueState;
import org.apache.hadoop.yarn.api.records.QueueUserACLInfo;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
@@ -67,6 +69,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ActiveUsersManager;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceLimits;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceUsage;
@@ -3312,6 +3315,90 @@ public class TestLeafQueue {
return csContext;
}
+ @Test
+ public void testApplicationQueuePercent()
+ throws Exception {
+ Resource res = Resource.newInstance(10 * 1024, 10);
+ CapacityScheduler scheduler = mock(CapacityScheduler.class);
+ when(scheduler.getClusterResource()).thenReturn(res);
+ when(scheduler.getResourceCalculator())
+ .thenReturn(new DefaultResourceCalculator());
+
+ ApplicationAttemptId appAttId = createAppAttemptId(0, 0);
+ RMContext rmContext = mock(RMContext.class);
+ when(rmContext.getEpoch()).thenReturn(3L);
+ when(rmContext.getScheduler()).thenReturn(scheduler);
+ when(rmContext.getRMApps())
+ .thenReturn(new ConcurrentHashMap<ApplicationId, RMApp>());
+ RMNodeLabelsManager nlm = mock(RMNodeLabelsManager.class);
+ when(nlm.getResourceByLabel(any(), any())).thenReturn(res);
+ when(rmContext.getNodeLabelManager()).thenReturn(nlm);
+
+ // Queue "test" consumes 100% of the cluster, so its capacity and absolute
+ // capacity are both 1.0f.
+ Queue queue = createQueue("test", null, 1.0f, 1.0f);
+ final String user = "user1";
+ FiCaSchedulerApp app =
+ new FiCaSchedulerApp(appAttId, user, queue,
+ queue.getActiveUsersManager(), rmContext);
+
+ // Resource request
+ Resource requestedResource = Resource.newInstance(1536, 2);
+ app.getAppAttemptResourceUsage().incUsed(requestedResource);
+ // In "test" queue, 1536 used is 15% of both the queue and the cluster
+ assertEquals(15.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+
+ // Queue "test2" is a child of root and its capacity is 50% of root. As a
+ // child of root, its absolute capaicty is also 50%.
+ queue = createQueue("test2", null, 0.5f, 0.5f);
+ app = new FiCaSchedulerApp(appAttId, user, queue,
+ queue.getActiveUsersManager(), rmContext);
+ app.getAppAttemptResourceUsage().incUsed(requestedResource);
+ // In "test2" queue, 1536 used is 30% of "test2" and 15% of the cluster.
+ assertEquals(30.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+
+ // Queue "test2.1" is 50% of queue "test2", which is 50% of the cluster.
+ // Therefore, "test2.1" capacity is 50% and absolute capacity is 25%.
+ AbstractCSQueue qChild = createQueue("test2.1", queue, 0.5f, 0.25f);
+ app = new FiCaSchedulerApp(appAttId, user, qChild,
+ qChild.getActiveUsersManager(), rmContext);
+ app.getAppAttemptResourceUsage().incUsed(requestedResource);
+ // In "test2.1" queue, 1536 used is 60% of "test2.1" and 15% of the cluster.
+ assertEquals(60.0f, app.getResourceUsageReport().getQueueUsagePercentage(),
+ 0.01f);
+ assertEquals(15.0f,
+ app.getResourceUsageReport().getClusterUsagePercentage(), 0.01f);
+ }
+
+ private ApplicationAttemptId createAppAttemptId(int appId, int attemptId) {
+ ApplicationId appIdImpl = ApplicationId.newInstance(0, appId);
+ ApplicationAttemptId attId =
+ ApplicationAttemptId.newInstance(appIdImpl, attemptId);
+ return attId;
+ }
+
+ private AbstractCSQueue createQueue(String name, Queue parent, float capacity,
+ float absCap) {
+ CSQueueMetrics metrics = CSQueueMetrics.forQueue(name, parent, false, cs.getConf());
+ QueueInfo queueInfo = QueueInfo.newInstance(name, capacity, 1.0f, 0, null,
+ null, QueueState.RUNNING, null, "", null, false);
+ ActiveUsersManager activeUsersManager = new ActiveUsersManager(metrics);
+ AbstractCSQueue queue = mock(AbstractCSQueue.class);
+ when(queue.getMetrics()).thenReturn(metrics);
+ when(queue.getActiveUsersManager()).thenReturn(activeUsersManager);
+ when(queue.getQueueInfo(false, false)).thenReturn(queueInfo);
+ QueueCapacities qCaps = mock(QueueCapacities.class);
+ when(qCaps.getAbsoluteCapacity(any())).thenReturn(absCap);
+ when(queue.getQueueCapacities()).thenReturn(qCaps);
+ return queue;
+ }
+
@After
public void tearDown() throws Exception {
if (cs != null) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/48] hadoop git commit: HADOOP-13356. Add a function to handle
command_subcommand_OPTS (aw)
Posted by aw...@apache.org.
HADOOP-13356. Add a function to handle command_subcommand_OPTS (aw)
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f462c9d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f462c9d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f462c9d7
Branch: refs/heads/HADOOP-13341
Commit: f462c9d73467eec3be19830829005f1136577e0b
Parents: 63f5948
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Jul 8 09:25:09 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../src/main/bin/hadoop-functions.sh | 62 ++++++++++++++++
.../test/scripts/hadoop_subcommand_opts.bats | 76 ++++++++++++++++++++
2 files changed, 138 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f462c9d7/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 75554f0..6e58dca 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -1974,6 +1974,68 @@ function hadoop_verify_user
fi
}
+## @description Add custom (program)_(command)_OPTS to HADOOP_OPTS.
+## @description Also handles the deprecated cases from pre-3.x.
+## @audience public
+## @stability stable
+## @replaceable yes
+## @param program
+## @param subcommand
+## @return will exit on failure conditions
+function hadoop_subcommand_opts
+{
+ declare program=$1
+ declare command=$2
+ declare var
+ declare uvar
+ declare uprogram
+ declare ucommand
+
+ if [[ -z "${program}" || -z "${command}" ]]; then
+ return 1
+ fi
+
+ # bash 4 and up have built-in ways to upper and lower
+ # case the contents of vars. This is faster than
+ # calling tr.
+
+ if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
+ ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
+ else
+ uprogram=${program^^}
+ ucommand=${command^^}
+ fi
+
+ # HDFS_namenode_OPTS
+ # HADOOP_distcp_OPTS
+ # MAPRED_distcp_OPTS
+ # YARN_sharedcachemanger_OPTS
+ # ...
+ var="${uprogram}_${command}_OPTS"
+
+ # Let's handle all of the deprecation cases early
+ # HADOOP_NAMENODE_OPTS -> HDFS_namenode_OPTS
+ # YARN_RESOURCEMANAGER_OPTS -> YARN_resourcemanager_OPTS
+
+ uvar="${uprogram}_${ucommand}_OPTS"
+ if [[ -n ${!uvar} ]]; then
+ hadoop_deprecate_envvar "${uvar}" "${var}"
+ fi
+
+ uvar="HADOOP_${ucommand}_OPTS"
+ if [[ -n ${!uvar} ]]; then
+ hadoop_deprecate_envvar "${uvar}" "${var}"
+ fi
+
+ if [[ -n ${!var} ]]; then
+ hadoop_debug "Appending ${!var} onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${!var}"
+ return 0
+ fi
+}
+
## @description Perform the 'hadoop classpath', etc subcommand with the given
## @description parameters
## @audience private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f462c9d7/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
new file mode 100644
index 0000000..1fbf343
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_subcommand_opts.bats
@@ -0,0 +1,76 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_subcommand_opts (missing param)" {
+ HADOOP_OPTS="x"
+ run hadoop_subcommand_opts testvar
+ [ "${status}" = "1" ]
+}
+
+@test "hadoop_subcommand_opts (simple not exist)" {
+ HADOOP_OPTS="x"
+ hadoop_subcommand_opts hadoop subcommand
+ [ "${HADOOP_OPTS}" = "x" ]
+}
+
+@test "hadoop_subcommand_opts (hadoop simple exist)" {
+ HADOOP_OPTS="x"
+ HADOOP_test_OPTS="y"
+ hadoop_subcommand_opts hadoop test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
+
+@test "hadoop_subcommand_opts (hadoop complex exist)" {
+ HADOOP_OPTS="x"
+ HADOOP_test_OPTS="y z"
+ hadoop_subcommand_opts hadoop test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y z" ]
+}
+
+@test "hadoop_subcommand_opts (hdfs simple exist)" {
+ HADOOP_OPTS="x"
+ HDFS_test_OPTS="y"
+ hadoop_subcommand_opts hdfs test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
+
+@test "hadoop_subcommand_opts (yarn simple exist)" {
+ HADOOP_OPTS="x"
+ YARN_test_OPTS="y"
+ hadoop_subcommand_opts yarn test
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
+
+@test "hadoop_subcommand_opts (deprecation case #1)" {
+ HADOOP_OPTS="x"
+ HADOOP_NAMENODE_OPTS="y"
+ hadoop_subcommand_opts hdfs namenode
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
+
+@test "hadoop_subcommand_opts (deprecation case #2)" {
+ HADOOP_OPTS="x"
+ YARN_RESOURCEMANAGER_OPTS="y"
+ hadoop_subcommand_opts yarn resourcemanager
+ echo "${HADOOP_OPTS}"
+ [ "${HADOOP_OPTS}" = "x y" ]
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/48] hadoop git commit: HDFS-10835. Fix typos in httpfs.sh.
Contributed by John Zhuge.
Posted by aw...@apache.org.
HDFS-10835. Fix typos in httpfs.sh. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7c213c34
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7c213c34
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7c213c34
Branch: refs/heads/HADOOP-13341
Commit: 7c213c341d047362e6f8b66ab3ffafaf639d8d31
Parents: f0d5382
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 11:44:26 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 11:44:26 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7c213c34/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
index a6c9f03..64459bf 100755
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/main/sbin/httpfs.sh
@@ -17,15 +17,15 @@ MYNAME="${BASH_SOURCE-$0}"
function hadoop_usage
{
- hadoop_add_subcommand "run" "Start kms in the current window"
+ hadoop_add_subcommand "run" "Start HttpFS in the current window"
hadoop_add_subcommand "run -security" "Start in the current window with security manager"
- hadoop_add_subcommand "start" "Start kms in a separate window"
+ hadoop_add_subcommand "start" "Start HttpFS in a separate window"
hadoop_add_subcommand "start -security" "Start in a separate window with security manager"
hadoop_add_subcommand "status" "Return the LSB compliant status"
- hadoop_add_subcommand "stop" "Stop kms, waiting up to 5 seconds for the process to end"
- hadoop_add_subcommand "top n" "Stop kms, waiting up to n seconds for the process to end"
- hadoop_add_subcommand "stop -force" "Stop kms, wait up to 5 seconds and then use kill -KILL if still running"
- hadoop_add_subcommand "stop n -force" "Stop kms, wait up to n seconds and then use kill -KILL if still running"
+ hadoop_add_subcommand "stop" "Stop HttpFS, waiting up to 5 seconds for the process to end"
+ hadoop_add_subcommand "stop n" "Stop HttpFS, waiting up to n seconds for the process to end"
+ hadoop_add_subcommand "stop -force" "Stop HttpFS, wait up to 5 seconds and then use kill -KILL if still running"
+ hadoop_add_subcommand "stop n -force" "Stop HttpFS, wait up to n seconds and then use kill -KILL if still running"
hadoop_generate_usage "${MYNAME}" false
}
@@ -47,7 +47,7 @@ else
exit 1
fi
-# The Java System property 'httpfs.http.port' it is not used by Kms,
+# The Java System property 'httpfs.http.port' it is not used by HttpFS,
# it is used in Tomcat's server.xml configuration file
#
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/48] hadoop git commit: YARN-5549.
AMLauncher#createAMContainerLaunchContext() should not log the command to be
launched indiscriminately. (Daniel Templeton via rchiang)
Posted by aw...@apache.org.
YARN-5549. AMLauncher#createAMContainerLaunchContext() should not log the command to be launched indiscriminately. (Daniel Templeton via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/378f624a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/378f624a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/378f624a
Branch: refs/heads/HADOOP-13341
Commit: 378f624a392550770d1db33cb4cee3ef7d5facd4
Parents: 5a8c506
Author: Ray Chiang <rc...@apache.org>
Authored: Fri Sep 2 11:07:39 2016 -0700
Committer: Ray Chiang <rc...@apache.org>
Committed: Fri Sep 2 11:14:35 2016 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 12 +++++++++
.../src/main/resources/yarn-default.xml | 13 ++++++++++
.../resourcemanager/amlauncher/AMLauncher.java | 26 ++++++++++++++++----
3 files changed, 46 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 46e3323..86e8a95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -534,6 +534,18 @@ public class YarnConfiguration extends Configuration {
public static final int
DEFAULT_RM_SYSTEM_METRICS_PUBLISHER_DISPATCHER_POOL_SIZE = 10;
+ /**
+ * The {@code AMLauncher.createAMContainerLaunchContext()} method will log the
+ * command being executed to the RM log if this property is true. Commands
+ * may contain sensitive information, such as application or service
+ * passwords, making logging the commands a security risk. In cases where
+ * the cluster may be running applications with such commands, this property
+ * should be set to false. Commands are only logged at the debug level.
+ */
+ public static final String RM_AMLAUNCHER_LOG_COMMAND =
+ RM_PREFIX + "amlauncher.log.command";
+ public static final boolean DEFAULT_RM_AMLAUNCHER_LOG_COMMAND = false;
+
//RM delegation token related keys
public static final String RM_DELEGATION_KEY_UPDATE_INTERVAL_KEY =
RM_PREFIX + "delegation.key.update-interval";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index e956507..423b78b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -299,6 +299,19 @@
</property>
<property>
+ <description>
+ The resource manager will log all commands being executed to the RM log
+ if this property is true. Commands may contain sensitive information,
+ such as application or service passwords, making logging the commands a
+ security risk. In cases where the cluster may be running applications with
+ such commands this property should be set to false. Commands are only
+ logged at the debug level.
+ </description>
+ <name>yarn.resourcemanager.amlauncher.log.command</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>The class to use as the resource scheduler.</description>
<name>yarn.resourcemanager.scheduler.class</name>
<value>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.CapacityScheduler</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/378f624a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
index 4aace2c..181463a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/amlauncher/AMLauncher.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
/**
* The launch of the AM itself.
@@ -81,6 +82,7 @@ public class AMLauncher implements Runnable {
private final AMLauncherEventType eventType;
private final RMContext rmContext;
private final Container masterContainer;
+ private final boolean logCommandLine;
@SuppressWarnings("rawtypes")
private final EventHandler handler;
@@ -93,6 +95,9 @@ public class AMLauncher implements Runnable {
this.rmContext = rmContext;
this.handler = rmContext.getDispatcher().getEventHandler();
this.masterContainer = application.getMasterContainer();
+ this.logCommandLine =
+ conf.getBoolean(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND,
+ YarnConfiguration.DEFAULT_RM_AMLAUNCHER_LOG_COMMAND);
}
private void connect() throws IOException {
@@ -188,11 +193,22 @@ public class AMLauncher implements Runnable {
// Construct the actual Container
ContainerLaunchContext container =
applicationMasterContext.getAMContainerSpec();
- LOG.info("Command to launch container "
- + containerID
- + " : "
- + StringUtils.arrayToString(container.getCommands().toArray(
- new String[0])));
+
+ if (LOG.isDebugEnabled()) {
+ StringBuilder message = new StringBuilder("Command to launch container ");
+
+ message.append(containerID).append(" : ");
+
+ if (logCommandLine) {
+ message.append(Joiner.on(",").join(container.getCommands()));
+ } else {
+ message.append("<REDACTED> -- Set ");
+ message.append(YarnConfiguration.RM_AMLAUNCHER_LOG_COMMAND);
+ message.append(" to true to reenable command logging");
+ }
+
+ LOG.debug(message.toString());
+ }
// Populate the current queue name in the environment variable.
setupQueueNameEnv(container, applicationMasterContext);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/48] hadoop git commit: YARN-5576. Allow resource localization
while container is running. Contributed by Jian He.
Posted by aw...@apache.org.
YARN-5576. Allow resource localization while container is running. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e6fcfe28
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e6fcfe28
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e6fcfe28
Branch: refs/heads/HADOOP-13341
Commit: e6fcfe28e304062c7d09231db757acb2953703ce
Parents: 62a9667
Author: Varun Vasudev <vv...@apache.org>
Authored: Tue Sep 6 20:01:45 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Tue Sep 6 20:01:45 2016 +0530
----------------------------------------------------------------------
.../server/nodemanager/ContainerExecutor.java | 9 +
.../hadoop/yarn/server/nodemanager/Context.java | 2 +
.../nodemanager/DefaultContainerExecutor.java | 6 +
.../nodemanager/DockerContainerExecutor.java | 6 +
.../nodemanager/LinuxContainerExecutor.java | 5 +
.../yarn/server/nodemanager/NodeManager.java | 12 ++
.../containermanager/ContainerManagerImpl.java | 70 +++++--
.../containermanager/container/Container.java | 8 +-
.../container/ContainerImpl.java | 207 ++++++++-----------
.../launcher/ContainerLaunch.java | 4 +-
.../localizer/ResourceLocalizationService.java | 21 +-
.../containermanager/localizer/ResourceSet.java | 192 +++++++++++++++++
.../ContainerLocalizationRequestEvent.java | 2 +-
.../localizer/event/LocalizationEventType.java | 2 +-
.../timelineservice/NMTimelinePublisher.java | 2 +-
.../nodemanager/DummyContainerManager.java | 2 +-
.../TestContainerManagerWithLCE.java | 13 +-
.../amrmproxy/BaseAMRMProxyTest.java | 6 +
.../containermanager/TestContainerManager.java | 137 +++++++++++-
.../TestResourceLocalizationService.java | 7 +-
.../TestContainersMonitorResourceChange.java | 7 +
.../nodemanager/webapp/MockContainer.java | 17 +-
22 files changed, 577 insertions(+), 160 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 818b0ea..918c30a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -182,6 +182,15 @@ public abstract class ContainerExecutor implements Configurable {
throws IOException, InterruptedException;
/**
+ * Create a symlink file which points to the target.
+ * @param target The target for symlink
+ * @param symlink the symlink file
+ * @throws IOException Error when creating symlinks
+ */
+ public abstract void symLink(String target, String symlink)
+ throws IOException;
+
+ /**
* Check if a container is alive.
* @param ctx Encapsulates information necessary for container liveness check.
* @return true if container is still alive
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
index 88bc29c..e888393 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
@@ -126,4 +126,6 @@ public interface Context {
void setNMTimelinePublisher(NMTimelinePublisher nmMetricsPublisher);
NMTimelinePublisher getNMTimelinePublisher();
+
+ ContainerExecutor getContainerExecutor();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 13ad9ac..9a0549d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -38,6 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -511,6 +512,11 @@ public class DefaultContainerExecutor extends ContainerExecutor {
}
}
+ @Override
+ public void symLink(String target, String symlink) throws IOException {
+ FileUtil.symLink(target, symlink);
+ }
+
/** Permissions for user dir.
* $local.dir/usercache/$user */
static final short USER_PERM = (short)0750;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
index ebf9566..1390214 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DockerContainerExecutor.java
@@ -490,6 +490,12 @@ public class DockerContainerExecutor extends ContainerExecutor {
}
}
+ @Override
+ public void symLink(String target, String symlink)
+ throws IOException {
+
+ }
+
/**
* Converts a directory list to a docker mount string
* @param dirs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index 6890b25..cc12b20 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -683,6 +683,11 @@ public class LinuxContainerExecutor extends ContainerExecutor {
}
@Override
+ public void symLink(String target, String symlink) {
+
+ }
+
+ @Override
public boolean isContainerAlive(ContainerLivenessContext ctx)
throws IOException {
String user = ctx.getUser();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 280a086..7f13334 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -343,6 +343,9 @@ public class NodeManager extends CompositeService
this.context = createNMContext(containerTokenSecretManager,
nmTokenSecretManager, nmStore, isDistSchedulingEnabled, conf);
+
+ ((NMContext)context).setContainerExecutor(exec);
+
nodeLabelsProvider = createNodeLabelsProvider(conf);
if (null == nodeLabelsProvider) {
@@ -509,6 +512,7 @@ public class NodeManager extends CompositeService
private OpportunisticContainerAllocator containerAllocator;
private final QueuingContext queuingContext;
+ private ContainerExecutor executor;
private NMTimelinePublisher nmTimelinePublisher;
@@ -702,6 +706,14 @@ public class NodeManager extends CompositeService
public NMTimelinePublisher getNMTimelinePublisher() {
return nmTimelinePublisher;
}
+
+ public ContainerExecutor getContainerExecutor() {
+ return this.executor;
+ }
+
+ public void setContainerExecutor(ContainerExecutor executor) {
+ this.executor = executor;
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 8c060bc..52d8566 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -18,25 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
-import static org.apache.hadoop.service.Service.STATE.STARTED;
-
-import java.io.DataInputStream;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.net.URISyntaxException;
-import java.nio.ByteBuffer;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.locks.ReentrantReadWriteLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
-import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
-
+import com.google.common.annotations.VisibleForTesting;
+import com.google.protobuf.ByteString;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@@ -76,6 +59,7 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
@@ -129,7 +113,9 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncher;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.SignalContainersLauncherEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.sharedcache.SharedCacheUploadEventType;
@@ -154,8 +140,25 @@ import org.apache.hadoop.yarn.server.utils.YarnServerSecurityUtils;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
-import com.google.common.annotations.VisibleForTesting;
-import com.google.protobuf.ByteString;
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.net.InetSocketAddress;
+import java.net.URISyntaxException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+
+import static org.apache.hadoop.service.Service.STATE.STARTED;
public class ContainerManagerImpl extends CompositeService implements
ContainerManager {
@@ -1525,6 +1528,31 @@ public class ContainerManagerImpl extends CompositeService implements
public ResourceLocalizationResponse localize(
ResourceLocalizationRequest request) throws YarnException, IOException {
+ ContainerId containerId = request.getContainerId();
+ Container container = context.getContainers().get(containerId);
+ if (container == null) {
+ throw new YarnException("Specified " + containerId + " does not exist!");
+ }
+ if (!container.getContainerState()
+ .equals(org.apache.hadoop.yarn.server.nodemanager.
+ containermanager.container.ContainerState.RUNNING)) {
+ throw new YarnException(
+ containerId + " is at " + container.getContainerState()
+ + " state. Not able to localize new resources.");
+ }
+
+ try {
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ container.getResourceSet().addResources(request.getLocalResources());
+ if (req != null && !req.isEmpty()) {
+ dispatcher.getEventHandler()
+ .handle(new ContainerLocalizationRequestEvent(container, req));
+ }
+ } catch (URISyntaxException e) {
+ LOG.info("Error when parsing local resource URI for " + containerId, e);
+ throw new YarnException(e);
+ }
+
return ResourceLocalizationResponse.newInstance();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index 84d3cb2..c4cea18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -31,6 +28,10 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.event.EventHandler;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
+
+import java.util.List;
+import java.util.Map;
public interface Container extends EventHandler<ContainerEvent> {
@@ -74,4 +75,5 @@ public interface Container extends EventHandler<ContainerEvent> {
Priority getPriority();
+ ResourceSet getResourceSet();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 0244d90..ce9e581 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -18,18 +18,15 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+import java.io.File;
import java.io.IOException;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
-import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
-import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
@@ -65,6 +62,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Ap
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainersLauncherEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.LocalResourceRequest;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent;
@@ -124,20 +122,7 @@ public class ContainerImpl implements Container {
private final Configuration daemonConf;
private static final Log LOG = LogFactory.getLog(ContainerImpl.class);
- private final Map<LocalResourceRequest,List<String>> pendingResources =
- new HashMap<LocalResourceRequest,List<String>>();
- private final Map<Path,List<String>> localizedResources =
- new HashMap<Path,List<String>>();
- private final List<LocalResourceRequest> publicRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final List<LocalResourceRequest> privateRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final List<LocalResourceRequest> appRsrcs =
- new ArrayList<LocalResourceRequest>();
- private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
- new ConcurrentHashMap<LocalResourceRequest, Path>();
- private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
- new ConcurrentHashMap<LocalResourceRequest, Boolean>();
+
// whether container has been recovered after a restart
private RecoveredContainerStatus recoveredStatus =
@@ -145,6 +130,7 @@ public class ContainerImpl implements Container {
// whether container was marked as killed after recovery
private boolean recoveredAsKilled = false;
private Context context;
+ private ResourceSet resourceSet;
public ContainerImpl(Configuration conf, Dispatcher dispatcher,
ContainerLaunchContext launchContext, Credentials creds,
@@ -204,6 +190,7 @@ public class ContainerImpl implements Container {
stateMachine = stateMachineFactory.make(this);
this.context = context;
+ this.resourceSet = new ResourceSet();
}
// constructor for a recovered container
@@ -313,6 +300,12 @@ public class ContainerImpl implements Container {
ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
new RetryFailureTransition())
.addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.RESOURCE_LOCALIZED,
+ new ResourceLocalizedWhileRunningTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
+ ContainerEventType.RESOURCE_FAILED,
+ new ResourceLocalizationFailedWhileRunningTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.RUNNING,
ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.RUNNING, ContainerState.KILLING,
@@ -470,7 +463,7 @@ public class ContainerImpl implements Container {
try {
if (ContainerState.LOCALIZED == getContainerState()
|| ContainerState.RELAUNCHING == getContainerState()) {
- return localizedResources;
+ return resourceSet.getLocalizedResources();
} else {
return null;
}
@@ -591,6 +584,11 @@ public class ContainerImpl implements Container {
this.logDir = logDir;
}
+ @Override
+ public ResourceSet getResourceSet() {
+ return this.resourceSet;
+ }
+
@SuppressWarnings("unchecked")
private void sendFinishedEvents() {
// Inform the application
@@ -653,7 +651,7 @@ public class ContainerImpl implements Container {
for (String s : diags) {
this.diagnostics.append(s);
}
- if (isRetryContextSet() && diagnostics.length() > diagnosticsMaxSize) {
+ if (diagnostics.length() > diagnosticsMaxSize) {
diagnostics.delete(0, diagnostics.length() - diagnosticsMaxSize);
}
try {
@@ -667,17 +665,7 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked") // dispatcher not typed
public void cleanup() {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
- new HashMap<LocalResourceVisibility,
- Collection<LocalResourceRequest>>();
- if (!publicRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
- }
- if (!privateRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
- }
- if (!appRsrcs.isEmpty()) {
- rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
- }
+ resourceSet.getAllResourcesByVisibility();
dispatcher.getEventHandler().handle(
new ContainerLocalizationCleanupEvent(this, rsrc));
}
@@ -697,7 +685,7 @@ public class ContainerImpl implements Container {
* message.
*
* If there are resources to localize, sends a
- * ContainerLocalizationRequest (INIT_CONTAINER_RESOURCES)
+ * ContainerLocalizationRequest (LOCALIZE_CONTAINER_RESOURCES)
* to the ResourceLocalizationManager and enters LOCALIZING state.
*
* If there are no resources to localize, sends LAUNCH_CONTAINER event
@@ -749,39 +737,15 @@ public class ContainerImpl implements Container {
}
container.containerLocalizationStartTime = clock.getTime();
+
// Send requests for public, private resources
Map<String,LocalResource> cntrRsrc = ctxt.getLocalResources();
if (!cntrRsrc.isEmpty()) {
try {
- for (Map.Entry<String,LocalResource> rsrc : cntrRsrc.entrySet()) {
- try {
- LocalResourceRequest req =
- new LocalResourceRequest(rsrc.getValue());
- List<String> links = container.pendingResources.get(req);
- if (links == null) {
- links = new ArrayList<String>();
- container.pendingResources.put(req, links);
- }
- links.add(rsrc.getKey());
- storeSharedCacheUploadPolicy(container, req, rsrc.getValue()
- .getShouldBeUploadedToSharedCache());
- switch (rsrc.getValue().getVisibility()) {
- case PUBLIC:
- container.publicRsrcs.add(req);
- break;
- case PRIVATE:
- container.privateRsrcs.add(req);
- break;
- case APPLICATION:
- container.appRsrcs.add(req);
- break;
- }
- } catch (URISyntaxException e) {
- LOG.info("Got exception parsing " + rsrc.getKey()
- + " and value " + rsrc.getValue());
- throw e;
- }
- }
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ container.resourceSet.addResources(ctxt.getLocalResources());
+ container.dispatcher.getEventHandler().handle(
+ new ContainerLocalizationRequestEvent(container, req));
} catch (URISyntaxException e) {
// malformed resource; abort container launch
LOG.warn("Failed to parse resource-request", e);
@@ -789,21 +753,6 @@ public class ContainerImpl implements Container {
container.metrics.endInitingContainer();
return ContainerState.LOCALIZATION_FAILED;
}
- Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
- new LinkedHashMap<LocalResourceVisibility,
- Collection<LocalResourceRequest>>();
- if (!container.publicRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.PUBLIC, container.publicRsrcs);
- }
- if (!container.privateRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.PRIVATE, container.privateRsrcs);
- }
- if (!container.appRsrcs.isEmpty()) {
- req.put(LocalResourceVisibility.APPLICATION, container.appRsrcs);
- }
-
- container.dispatcher.getEventHandler().handle(
- new ContainerLocalizationRequestEvent(container, req));
return ContainerState.LOCALIZING;
} else {
container.sendLaunchEvent();
@@ -814,27 +763,6 @@ public class ContainerImpl implements Container {
}
/**
- * Store the resource's shared cache upload policies
- * Given LocalResourceRequest can be shared across containers in
- * LocalResourcesTrackerImpl, we preserve the upload policies here.
- * In addition, it is possible for the application to create several
- * "identical" LocalResources as part of
- * ContainerLaunchContext.setLocalResources with different symlinks.
- * There is a corner case where these "identical" local resources have
- * different upload policies. For that scenario, upload policy will be set to
- * true as long as there is at least one LocalResource entry with
- * upload policy set to true.
- */
- private static void storeSharedCacheUploadPolicy(ContainerImpl container,
- LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
- Boolean storedUploadPolicy =
- container.resourcesUploadPolicies.get(resourceRequest);
- if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
- container.resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
- }
- }
-
- /**
* Transition when one of the requested resources for this container
* has been successfully localized.
*/
@@ -847,22 +775,21 @@ public class ContainerImpl implements Container {
ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
LocalResourceRequest resourceRequest = rsrcEvent.getResource();
Path location = rsrcEvent.getLocation();
- List<String> syms = container.pendingResources.remove(resourceRequest);
+ List<String> syms =
+ container.resourceSet.resourceLocalized(resourceRequest, location);
if (null == syms) {
- LOG.warn("Localized unknown resource " + resourceRequest +
- " for container " + container.containerId);
- assert false;
- // fail container?
+ LOG.info("Localized resource " + resourceRequest +
+ " for container " + container.containerId);
return ContainerState.LOCALIZING;
}
- container.localizedResources.put(location, syms);
// check to see if this resource should be uploaded to the shared cache
// as well
if (shouldBeUploadedToSharedCache(container, resourceRequest)) {
- container.resourcesToBeUploaded.put(resourceRequest, location);
+ container.resourceSet.getResourcesToBeUploaded()
+ .put(resourceRequest, location);
}
- if (!container.pendingResources.isEmpty()) {
+ if (!container.resourceSet.getPendingResources().isEmpty()) {
return ContainerState.LOCALIZING;
}
@@ -884,7 +811,8 @@ public class ContainerImpl implements Container {
&& container.recoveredStatus != RecoveredContainerStatus.COMPLETED) {
// kick off uploads to the shared cache
container.dispatcher.getEventHandler().handle(
- new SharedCacheUploadEvent(container.resourcesToBeUploaded, container
+ new SharedCacheUploadEvent(
+ container.resourceSet.getResourcesToBeUploaded(), container
.getLaunchContext(), container.getUser(),
SharedCacheUploadEventType.UPLOAD));
}
@@ -894,6 +822,56 @@ public class ContainerImpl implements Container {
}
/**
+ * Resource is localized while the container is running - create symlinks
+ */
+ static class ResourceLocalizedWhileRunningTransition
+ extends ContainerTransition {
+
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerResourceLocalizedEvent rsrcEvent =
+ (ContainerResourceLocalizedEvent) event;
+ List<String> links = container.resourceSet
+ .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation());
+ // creating symlinks.
+ for (String link : links) {
+ try {
+ String linkFile = new Path(container.workDir, link).toString();
+ if (new File(linkFile).exists()) {
+ LOG.info("Symlink file already exists: " + linkFile);
+ } else {
+ container.context.getContainerExecutor()
+ .symLink(rsrcEvent.getLocation().toString(), linkFile);
+ LOG.info("Created symlink: " + linkFile + " -> " + rsrcEvent
+ .getLocation());
+ }
+ } catch (IOException e) {
+ String message = String
+ .format("Error when creating symlink %s -> %s", link,
+ rsrcEvent.getLocation());
+ LOG.error(message, e);
+ }
+ }
+ }
+ }
+
+ /**
+ * Resource localization failed while the container is running.
+ */
+ static class ResourceLocalizationFailedWhileRunningTransition
+ extends ContainerTransition {
+
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ ContainerResourceFailedEvent failedEvent =
+ (ContainerResourceFailedEvent) event;
+ container.resourceSet
+ .resourceLocalizationFailed(failedEvent.getResource());
+ container.addDiagnostics(failedEvent.getDiagnosticMessage());
+ }
+ }
+
+ /**
* Transition from LOCALIZED state to RUNNING state upon receiving
* a CONTAINER_LAUNCHED event
*/
@@ -1136,17 +1114,10 @@ public class ContainerImpl implements Container {
SingleArcTransition<ContainerImpl, ContainerEvent> {
@Override
public void transition(ContainerImpl container, ContainerEvent event) {
- ContainerResourceLocalizedEvent rsrcEvent = (ContainerResourceLocalizedEvent) event;
- List<String> syms =
- container.pendingResources.remove(rsrcEvent.getResource());
- if (null == syms) {
- LOG.warn("Localized unknown resource " + rsrcEvent.getResource() +
- " for container " + container.containerId);
- assert false;
- // fail container?
- return;
- }
- container.localizedResources.put(rsrcEvent.getLocation(), syms);
+ ContainerResourceLocalizedEvent rsrcEvent =
+ (ContainerResourceLocalizedEvent) event;
+ container.resourceSet
+ .resourceLocalized(rsrcEvent.getResource(), rsrcEvent.getLocation());
}
}
@@ -1402,7 +1373,7 @@ public class ContainerImpl implements Container {
*/
private static boolean shouldBeUploadedToSharedCache(ContainerImpl container,
LocalResourceRequest resource) {
- return container.resourcesUploadPolicies.get(resource);
+ return container.resourceSet.getResourcesUploadPolicies().get(resource);
}
@VisibleForTesting
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 14190fc..d8239ef 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -1201,16 +1201,16 @@ public class ContainerLaunch implements Callable<Integer> {
private void recordContainerLogDir(ContainerId containerId,
String logDir) throws IOException{
+ container.setLogDir(logDir);
if (container.isRetryContextSet()) {
- container.setLogDir(logDir);
context.getNMStateStore().storeContainerLogDir(containerId, logDir);
}
}
private void recordContainerWorkDir(ContainerId containerId,
String workDir) throws IOException{
+ container.setWorkDir(workDir);
if (container.isRetryContextSet()) {
- container.setWorkDir(workDir);
context.getNMStateStore().storeContainerWorkDir(containerId, workDir);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 0ec2f29..b281ef5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -112,6 +112,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerResourceFailedEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationCleanupEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationEvent;
@@ -135,7 +136,6 @@ import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.Re
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredUserResources;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
import org.apache.hadoop.yarn.server.nodemanager.util.NodeManagerBuilderUtils;
-import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
import com.google.common.annotations.VisibleForTesting;
@@ -419,7 +419,7 @@ public class ResourceLocalizationService extends CompositeService
handleInitApplicationResources(
((ApplicationLocalizationEvent)event).getApplication());
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
handleInitContainerResources((ContainerLocalizationRequestEvent) event);
break;
case CONTAINER_RESOURCES_LOCALIZED:
@@ -469,6 +469,13 @@ public class ResourceLocalizationService extends CompositeService
private void handleInitContainerResources(
ContainerLocalizationRequestEvent rsrcReqs) {
Container c = rsrcReqs.getContainer();
+ EnumSet<ContainerState> set =
+ EnumSet.of(ContainerState.LOCALIZING, ContainerState.RUNNING);
+ if (!set.contains(c.getContainerState())) {
+ LOG.warn(c.getContainerId() + " is at " + c.getContainerState()
+ + " state, do not localize resources.");
+ return;
+ }
// create a loading cache for the file statuses
LoadingCache<Path,Future<FileStatus>> statCache =
CacheBuilder.newBuilder().build(FSDownload.createStatusCacheLoader(getConfig()));
@@ -538,7 +545,7 @@ public class ResourceLocalizationService extends CompositeService
}
String locId = c.getContainerId().toString();
localizerTracker.cleanupPrivLocalizers(locId);
-
+
// Delete the container directories
String userName = c.getUser();
String containerIDStr = c.toString();
@@ -747,6 +754,14 @@ public class ResourceLocalizationService extends CompositeService
case APPLICATION:
synchronized (privLocalizers) {
LocalizerRunner localizer = privLocalizers.get(locId);
+ if (localizer != null && localizer.killContainerLocalizer.get()) {
+ // Old localizer thread has been stopped, remove it and creates
+ // a new localizer thread.
+ LOG.info("New " + event.getType() + " localize request for "
+ + locId + ", remove old private localizer.");
+ cleanupPrivLocalizers(locId);
+ localizer = null;
+ }
if (null == localizer) {
LOG.info("Created localizer for " + locId);
localizer = new LocalizerRunner(req.getContext(), locId);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
new file mode 100644
index 0000000..a41ee20
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
@@ -0,0 +1,192 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+
+import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * All Resources requested by the container.
+ */
+public class ResourceSet {
+
+ private static final Log LOG = LogFactory.getLog(ResourceSet.class);
+
+ // resources by localization state (localized, pending, failed)
+ private Map<Path, List<String>> localizedResources =
+ new ConcurrentHashMap<>();
+ private Map<LocalResourceRequest, List<String>> pendingResources =
+ new ConcurrentHashMap<>();
+ private Set<LocalResourceRequest> resourcesFailedToBeLocalized =
+ new HashSet<>();
+
+ // resources by visibility (public, private, app)
+ private final List<LocalResourceRequest> publicRsrcs =
+ new ArrayList<>();
+ private final List<LocalResourceRequest> privateRsrcs =
+ new ArrayList<>();
+ private final List<LocalResourceRequest> appRsrcs =
+ new ArrayList<>();
+
+ private final Map<LocalResourceRequest, Path> resourcesToBeUploaded =
+ new ConcurrentHashMap<>();
+ private final Map<LocalResourceRequest, Boolean> resourcesUploadPolicies =
+ new ConcurrentHashMap<>();
+
+ public Map<LocalResourceVisibility, Collection<LocalResourceRequest>>
+ addResources(Map<String, LocalResource> localResourceMap)
+ throws URISyntaxException {
+ if (localResourceMap == null || localResourceMap.isEmpty()) {
+ return null;
+ }
+ Map<LocalResourceRequest, List<String>> allResources = new HashMap<>();
+ List<LocalResourceRequest> publicList = new ArrayList<>();
+ List<LocalResourceRequest> privateList = new ArrayList<>();
+ List<LocalResourceRequest> appList = new ArrayList<>();
+
+ for (Map.Entry<String, LocalResource> rsrc : localResourceMap.entrySet()) {
+ LocalResource resource = rsrc.getValue();
+ LocalResourceRequest req = new LocalResourceRequest(rsrc.getValue());
+ allResources.putIfAbsent(req, new ArrayList<>());
+ allResources.get(req).add(rsrc.getKey());
+ storeSharedCacheUploadPolicy(req,
+ resource.getShouldBeUploadedToSharedCache());
+ switch (resource.getVisibility()) {
+ case PUBLIC:
+ publicList.add(req);
+ break;
+ case PRIVATE:
+ privateList.add(req);
+ break;
+ case APPLICATION:
+ appList.add(req);
+ break;
+ default:
+ break;
+ }
+ }
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req =
+ new LinkedHashMap<>();
+ if (!publicList.isEmpty()) {
+ publicRsrcs.addAll(publicList);
+ req.put(LocalResourceVisibility.PUBLIC, publicList);
+ }
+ if (!privateList.isEmpty()) {
+ privateRsrcs.addAll(privateList);
+ req.put(LocalResourceVisibility.PRIVATE, privateList);
+ }
+ if (!appList.isEmpty()) {
+ appRsrcs.addAll(appList);
+ req.put(LocalResourceVisibility.APPLICATION, appList);
+ }
+ if (!allResources.isEmpty()) {
+ this.pendingResources.putAll(allResources);
+ }
+ return req;
+ }
+
+ /**
+ * Called when resource localized.
+ * @param request The original request for the localized resource
+ * @param location The path where the resource is localized
+ * @return The list of symlinks for the localized resources.
+ */
+ public List<String> resourceLocalized(LocalResourceRequest request,
+ Path location) {
+ List<String> symlinks = pendingResources.remove(request);
+ if (symlinks == null) {
+ return null;
+ } else {
+ localizedResources.put(location, symlinks);
+ return symlinks;
+ }
+ }
+
+ public void resourceLocalizationFailed(LocalResourceRequest request) {
+ pendingResources.remove(request);
+ resourcesFailedToBeLocalized.add(request);
+ }
+
+ public synchronized Map<LocalResourceVisibility,
+ Collection<LocalResourceRequest>> getAllResourcesByVisibility() {
+
+ Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc =
+ new HashMap<>();
+ if (!publicRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.PUBLIC, publicRsrcs);
+ }
+ if (!privateRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.PRIVATE, privateRsrcs);
+ }
+ if (!appRsrcs.isEmpty()) {
+ rsrc.put(LocalResourceVisibility.APPLICATION, appRsrcs);
+ }
+ return rsrc;
+ }
+
+ /**
+ * Store the resource's shared cache upload policies
+ * Given LocalResourceRequest can be shared across containers in
+ * LocalResourcesTrackerImpl, we preserve the upload policies here.
+ * In addition, it is possible for the application to create several
+ * "identical" LocalResources as part of
+ * ContainerLaunchContext.setLocalResources with different symlinks.
+ * There is a corner case where these "identical" local resources have
+ * different upload policies. For that scenario, upload policy will be set to
+ * true as long as there is at least one LocalResource entry with
+ * upload policy set to true.
+ */
+ private void storeSharedCacheUploadPolicy(
+ LocalResourceRequest resourceRequest, Boolean uploadPolicy) {
+ Boolean storedUploadPolicy = resourcesUploadPolicies.get(resourceRequest);
+ if (storedUploadPolicy == null || (!storedUploadPolicy && uploadPolicy)) {
+ resourcesUploadPolicies.put(resourceRequest, uploadPolicy);
+ }
+ }
+
+ public Map<Path, List<String>> getLocalizedResources() {
+ return localizedResources;
+ }
+
+ public Map<LocalResourceRequest, Path> getResourcesToBeUploaded() {
+ return resourcesToBeUploaded;
+ }
+
+ public Map<LocalResourceRequest, Boolean> getResourcesUploadPolicies() {
+ return resourcesUploadPolicies;
+ }
+
+ public Map<LocalResourceRequest, List<String>> getPendingResources() {
+ return pendingResources;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
index 11bb25e..43a2f33 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
@@ -44,7 +44,7 @@ public class ContainerLocalizationRequestEvent extends
*/
public ContainerLocalizationRequestEvent(Container c,
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrc) {
- super(LocalizationEventType.INIT_CONTAINER_RESOURCES, c);
+ super(LocalizationEventType.LOCALIZE_CONTAINER_RESOURCES, c);
this.rsrc = rsrc;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
index 4785fba..cb9842d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
@@ -19,7 +19,7 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.eve
public enum LocalizationEventType {
INIT_APPLICATION_RESOURCES,
- INIT_CONTAINER_RESOURCES,
+ LOCALIZE_CONTAINER_RESOURCES,
CACHE_CLEANUP,
CLEANUP_CONTAINER_RESOURCES,
DESTROY_APPLICATION_RESOURCES,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 0371863..8e68889 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -326,7 +326,7 @@ public class NMTimelinePublisher extends CompositeService {
publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
ContainerMetricsConstants.LOCALIZATION_FINISHED_EVENT_TYPE);
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
publishContainerLocalizationEvent((ContainerLocalizationEvent) event,
ContainerMetricsConstants.LOCALIZATION_START_EVENT_TYPE);
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index 4aeb2e2..740ed19 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -86,7 +86,7 @@ public class DummyContainerManager extends ContainerManagerImpl {
dispatcher.getEventHandler().handle(new ApplicationInitedEvent(
app.getAppId()));
break;
- case INIT_CONTAINER_RESOURCES:
+ case LOCALIZE_CONTAINER_RESOURCES:
ContainerLocalizationRequestEvent rsrcReqs =
(ContainerLocalizationRequestEvent) event;
// simulate localization of all requested resources
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 3e00885..aa0d975 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -131,7 +131,18 @@ public class TestContainerManagerWithLCE extends TestContainerManager {
LOG.info("Running testContainerLaunchAndExitFailure");
super.testContainerLaunchAndExitFailure();
}
-
+
+ @Override
+ public void testLocalingResourceWhileContainerRunning()
+ throws Exception {
+ // Don't run the test if the binary is not available.
+ if (!shouldRunTest()) {
+ LOG.info("LCE binary path is not passed. Not running the test");
+ return;
+ }
+ super.testLocalingResourceWhileContainerRunning();
+ }
+
@Override
public void testLocalFilesCleanup() throws InterruptedException,
IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index d8660dd..031300f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.api.protocolrecords.LogAggregationReport;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
+import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.Context;
import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.NodeResourceMonitor;
@@ -715,6 +716,11 @@ public abstract class BaseAMRMProxyTest {
@Override
public NMTimelinePublisher getNMTimelinePublisher() {
+ return null;
+ }
+
+ @Override
+ public ContainerExecutor getContainerExecutor() {
return null;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 1f803b4..5785e1f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -36,17 +36,20 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import com.google.common.base.Supplier;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.Service;
+import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetContainerStatusesResponse;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceRequest;
import org.apache.hadoop.yarn.api.protocolrecords.IncreaseContainersResourceResponse;
+import org.apache.hadoop.yarn.api.protocolrecords.ResourceLocalizationRequest;
import org.apache.hadoop.yarn.api.protocolrecords.SignalContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainerRequest;
import org.apache.hadoop.yarn.api.protocolrecords.StartContainersRequest;
@@ -83,6 +86,7 @@ import org.apache.hadoop.yarn.server.nodemanager.CMgrSignalContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor.Signal;
import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
+import org.apache.hadoop.yarn.server.nodemanager.NodeManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.TestAuxServices.ServiceA;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
@@ -459,7 +463,138 @@ public class TestContainerManager extends BaseContainerManagerTest {
// and verify exit code returned
testContainerLaunchAndExit(exitCode);
}
-
+
+ private Map<String, LocalResource> setupLocalResources(String fileName,
+ String symLink) throws Exception {
+ // ////// Create the resources for the container
+ File dir = new File(tmpDir, "dir");
+ dir.mkdirs();
+ File file = new File(dir, fileName);
+ PrintWriter fileWriter = new PrintWriter(file);
+ fileWriter.write("Hello World!");
+ fileWriter.close();
+
+ URL resourceURL = URL.fromPath(FileContext.getLocalFSFileContext()
+ .makeQualified(new Path(file.getAbsolutePath())));
+ LocalResource resource =
+ recordFactory.newRecordInstance(LocalResource.class);
+ resource.setResource(resourceURL);
+ resource.setSize(-1);
+ resource.setVisibility(LocalResourceVisibility.APPLICATION);
+ resource.setType(LocalResourceType.FILE);
+ resource.setTimestamp(file.lastModified());
+ Map<String, LocalResource> localResources =
+ new HashMap<String, LocalResource>();
+ localResources.put(symLink, resource);
+ return localResources;
+ }
+
+ // Start the container
+ // While the container is running, localize new resources.
+ // Verify the symlink is created properly
+ @Test
+ public void testLocalingResourceWhileContainerRunning() throws Exception {
+ // Real del service
+ delSrvc = new DeletionService(exec);
+ delSrvc.init(conf);
+
+ ((NodeManager.NMContext)context).setContainerExecutor(exec);
+ containerManager = createContainerManager(delSrvc);
+ containerManager.init(conf);
+ containerManager.start();
+ // set up local resources
+ Map<String, LocalResource> localResource =
+ setupLocalResources("file", "symLink1");
+ ContainerLaunchContext context =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+ context.setLocalResources(localResource);
+
+ // a long running container - sleep
+ context.setCommands(Arrays.asList("sleep 6"));
+ ContainerId cId = createContainerId(0);
+
+ // start the container
+ StartContainerRequest scRequest = StartContainerRequest.newInstance(context,
+ createContainerToken(cId, DUMMY_RM_IDENTIFIER, this.context.getNodeId(),
+ user, this.context.getContainerTokenSecretManager()));
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(Arrays.asList(scRequest));
+ containerManager.startContainers(allRequests);
+ BaseContainerManagerTest
+ .waitForContainerState(containerManager, cId, ContainerState.RUNNING);
+
+ BaseContainerManagerTest.waitForApplicationState(containerManager,
+ cId.getApplicationAttemptId().getApplicationId(),
+ ApplicationState.RUNNING);
+ checkResourceLocalized(cId, "symLink1");
+
+ // Localize new local resources while container is running
+ Map<String, LocalResource> localResource2 =
+ setupLocalResources("file2", "symLink2");
+
+ ResourceLocalizationRequest request =
+ ResourceLocalizationRequest.newInstance(cId, localResource2);
+ containerManager.localize(request);
+
+ // Verify resource is localized and symlink is created.
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ public Boolean get() {
+ try {
+ checkResourceLocalized(cId, "symLink2");
+ return true;
+ } catch (Throwable e) {
+ return false;
+ }
+ }
+ }, 500, 20000);
+
+ BaseContainerManagerTest
+ .waitForContainerState(containerManager, cId, ContainerState.COMPLETE);
+ // Verify container cannot localize resources while at non-running state.
+ try{
+ containerManager.localize(request);
+ Assert.fail();
+ } catch (YarnException e) {
+ Assert.assertTrue(
+ e.getMessage().contains("Not able to localize new resources"));
+ }
+ }
+
+ private void checkResourceLocalized(ContainerId containerId, String symLink) {
+ String appId =
+ containerId.getApplicationAttemptId().getApplicationId().toString();
+ File userCacheDir = new File(localDir, ContainerLocalizer.USERCACHE);
+ File userDir = new File(userCacheDir, user);
+ File appCache = new File(userDir, ContainerLocalizer.APPCACHE);
+ // localDir/usercache/nobody/appcache/application_0_0000
+ File appDir = new File(appCache, appId);
+ // localDir/usercache/nobody/appcache/application_0_0000/container_0_0000_01_000000
+ File containerDir = new File(appDir, containerId.toString());
+ // localDir/usercache/nobody/appcache/application_0_0000/container_0_0000_01_000000/symLink1
+ File targetFile = new File(containerDir, symLink);
+
+ File sysDir =
+ new File(localDir, ResourceLocalizationService.NM_PRIVATE_DIR);
+ // localDir/nmPrivate/application_0_0000
+ File appSysDir = new File(sysDir, appId);
+ // localDir/nmPrivate/application_0_0000/container_0_0000_01_000000
+ File containerSysDir = new File(appSysDir, containerId.toString());
+
+ Assert.assertTrue("AppDir " + appDir.getAbsolutePath() + " doesn't exist!!",
+ appDir.exists());
+ Assert.assertTrue(
+ "AppSysDir " + appSysDir.getAbsolutePath() + " doesn't exist!!",
+ appSysDir.exists());
+ Assert.assertTrue(
+ "containerDir " + containerDir.getAbsolutePath() + " doesn't exist !",
+ containerDir.exists());
+ Assert.assertTrue("containerSysDir " + containerSysDir.getAbsolutePath()
+ + " doesn't exist !", containerDir.exists());
+ Assert.assertTrue(
+ "targetFile " + targetFile.getAbsolutePath() + " doesn't exist !!",
+ targetFile.exists());
+ }
+
@Test
public void testLocalFilesCleanup() throws InterruptedException,
IOException, YarnException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
index 23786fe..7aa657e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
@@ -66,6 +66,7 @@ import java.util.concurrent.Future;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.Options;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.executor.LocalizerStartContext;
import org.junit.Assert;
@@ -1767,7 +1768,7 @@ public class TestResourceLocalizationService {
// creating new containers and populating corresponding localizer runners
// Container - 1
- ContainerImpl container1 = createMockContainer(user, 1);
+ Container container1 = createMockContainer(user, 1);
String localizerId1 = container1.getContainerId().toString();
rls.getPrivateLocalizers().put(
localizerId1,
@@ -2292,7 +2293,7 @@ public class TestResourceLocalizationService {
}
private ContainerLocalizationRequestEvent createContainerLocalizationEvent(
- ContainerImpl container, LocalResourceVisibility vis,
+ Container container, LocalResourceVisibility vis,
LocalResourceRequest req) {
Map<LocalResourceVisibility, Collection<LocalResourceRequest>> reqs =
new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
@@ -2310,6 +2311,7 @@ public class TestResourceLocalizationService {
when(container.getUser()).thenReturn(user);
Credentials mockCredentials = mock(Credentials.class);
when(container.getCredentials()).thenReturn(mockCredentials);
+ when(container.getContainerState()).thenReturn(ContainerState.LOCALIZING);
return container;
}
@@ -2358,6 +2360,7 @@ public class TestResourceLocalizationService {
creds.addToken(new Text("tok" + id), tk);
when(c.getCredentials()).thenReturn(creds);
when(c.toString()).thenReturn(cId.toString());
+ when(c.getContainerState()).thenReturn(ContainerState.LOCALIZING);
return c;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
index 2df0c98..d24f89d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitorResourceChange.java
@@ -89,6 +89,13 @@ public class TestContainersMonitorResourceChange {
public void deleteAsUser(DeletionAsUserContext ctx)
throws IOException, InterruptedException {
}
+
+ @Override
+ public void symLink(String target, String symlink)
+ throws IOException {
+
+ }
+
@Override
public String getProcessId(ContainerId containerId) {
return String.valueOf(containerId.getContainerId());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e6fcfe28/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 8332b2a..c176556 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,11 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import java.io.IOException;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
@@ -41,8 +37,14 @@ import org.apache.hadoop.yarn.server.api.protocolrecords.NMContainerStatus;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceSet;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
public class MockContainer implements Container {
private ContainerId id;
@@ -119,6 +121,11 @@ public class MockContainer implements Container {
}
@Override
+ public ResourceSet getResourceSet() {
+ return null;
+ }
+
+ @Override
public void handle(ContainerEvent event) {
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/48] hadoop git commit: HDFS-10778. Add -format option to make the
output of FileDistribution processor human-readable in OfflineImageViewer.
Posted by aw...@apache.org.
HDFS-10778. Add -format option to make the output of FileDistribution processor human-readable in OfflineImageViewer.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/63f59489
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/63f59489
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/63f59489
Branch: refs/heads/HADOOP-13341
Commit: 63f594892ecd4687e37a99790288e36eb278849f
Parents: d355573
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Sep 8 15:13:43 2016 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Sep 8 15:13:43 2016 +0900
----------------------------------------------------------------------
.../FileDistributionCalculator.java | 20 +++-
.../FileDistributionVisitor.java | 28 ++++-
.../offlineImageViewer/OfflineImageViewer.java | 116 ++++++++++---------
.../OfflineImageViewerPB.java | 78 +++++++------
.../src/site/markdown/HDFSCommands.md | 1 +
.../src/site/markdown/HdfsImageViewer.md | 1 +
.../TestOfflineImageViewer.java | 24 +++-
7 files changed, 164 insertions(+), 104 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
index 33ab641..71fb822 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionCalculator.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hdfs.server.namenode.FSImageUtil;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.FileSummary;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.INodeSection;
import org.apache.hadoop.util.LimitInputStream;
+import org.apache.hadoop.util.StringUtils;
import com.google.common.base.Preconditions;
@@ -75,11 +76,14 @@ final class FileDistributionCalculator {
private long totalSpace;
private long maxFileSize;
+ private boolean formatOutput = false;
+
FileDistributionCalculator(Configuration conf, long maxSize, int steps,
- PrintStream out) {
+ boolean formatOutput, PrintStream out) {
this.conf = conf;
this.maxSize = maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize;
this.steps = steps == 0 ? INTERVAL_DEFAULT : steps;
+ this.formatOutput = formatOutput;
this.out = out;
long numIntervals = this.maxSize / this.steps;
// avoid OutOfMemoryError when allocating an array
@@ -148,10 +152,20 @@ final class FileDistributionCalculator {
private void output() {
// write the distribution into the output file
- out.print("Size\tNumFiles\n");
+ out.print((formatOutput ? "Size Range" : "Size") + "\tNumFiles\n");
for (int i = 0; i < distribution.length; i++) {
if (distribution[i] != 0) {
- out.print(((long) i * steps) + "\t" + distribution[i]);
+ if (formatOutput) {
+ out.print((i == 0 ? "[" : "(")
+ + StringUtils.byteDesc(((long) (i == 0 ? 0 : i - 1) * steps))
+ + ", "
+ + StringUtils.byteDesc((long)
+ (i == distribution.length - 1 ? maxFileSize : i * steps))
+ + "]\t" + distribution[i]);
+ } else {
+ out.print(((long) i * steps) + "\t" + distribution[i]);
+ }
+
out.print('\n');
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
index 1cef720..7dcc299 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import java.io.IOException;
import java.util.LinkedList;
+import org.apache.hadoop.util.StringUtils;
+
/**
* File size distribution visitor.
*
@@ -67,6 +69,7 @@ class FileDistributionVisitor extends TextWriterImageVisitor {
private FileContext current;
private boolean inInode = false;
+ private boolean formatOutput = false;
/**
* File or directory information.
@@ -78,12 +81,12 @@ class FileDistributionVisitor extends TextWriterImageVisitor {
int replication;
}
- public FileDistributionVisitor(String filename,
- long maxSize,
- int step) throws IOException {
+ public FileDistributionVisitor(String filename, long maxSize, int step,
+ boolean formatOutput) throws IOException {
super(filename, false);
this.maxSize = (maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize);
this.step = (step == 0 ? INTERVAL_DEFAULT : step);
+ this.formatOutput = formatOutput;
long numIntervals = this.maxSize / this.step;
if(numIntervals >= Integer.MAX_VALUE)
throw new IOException("Too many distribution intervals " + numIntervals);
@@ -113,9 +116,22 @@ class FileDistributionVisitor extends TextWriterImageVisitor {
private void output() throws IOException {
// write the distribution into the output file
- write("Size\tNumFiles\n");
- for(int i = 0; i < distribution.length; i++)
- write(((long)i * step) + "\t" + distribution[i] + "\n");
+ write((formatOutput ? "Size Range" : "Size") + "\tNumFiles\n");
+ for (int i = 0; i < distribution.length; i++) {
+ if (distribution[i] > 0) {
+ if (formatOutput) {
+ write((i == 0 ? "[" : "(")
+ + StringUtils.byteDesc(((long) (i == 0 ? 0 : i - 1) * step))
+ + ", "
+ + StringUtils.byteDesc((long)
+ (i == distribution.length - 1 ? maxFileSize : i * step))
+ + "]\t"
+ + distribution[i] + "\n");
+ } else {
+ write(((long) i * step) + "\t" + distribution[i] + "\n");
+ }
+ }
+ }
System.out.println("totalFiles = " + totalFiles);
System.out.println("totalDirectories = " + totalDirectories);
System.out.println("totalBlocks = " + totalBlocks);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
index 770cde1..c542d90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
@@ -46,61 +46,63 @@ import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.PositionTrackingIn
public class OfflineImageViewer {
public static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
- private final static String usage =
- "Usage: bin/hdfs oiv_legacy [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" +
- "Offline Image Viewer\n" +
- "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" +
- "saving the results in OUTPUTFILE.\n" +
- "\n" +
- "The oiv utility will attempt to parse correctly formed image files\n" +
- "and will abort fail with mal-formed image files.\n" +
- "\n" +
- "The tool works offline and does not require a running cluster in\n" +
- "order to process an image file.\n" +
- "\n" +
- "The following image processors are available:\n" +
- " * Ls: The default image processor generates an lsr-style listing\n" +
- " of the files in the namespace, with the same fields in the same\n" +
- " order. Note that in order to correctly determine file sizes,\n" +
- " this formatter cannot skip blocks and will override the\n" +
- " -skipBlocks option.\n" +
- " * Indented: This processor enumerates over all of the elements in\n" +
- " the fsimage file, using levels of indentation to delineate\n" +
- " sections within the file.\n" +
- " * Delimited: Generate a text file with all of the elements common\n" +
- " to both inodes and inodes-under-construction, separated by a\n" +
- " delimiter. The default delimiter is \u0001, though this may be\n" +
- " changed via the -delimiter argument. This processor also overrides\n" +
- " the -skipBlocks option for the same reason as the Ls processor\n" +
- " * XML: This processor creates an XML document with all elements of\n" +
- " the fsimage enumerated, suitable for further analysis by XML\n" +
- " tools.\n" +
- " * FileDistribution: This processor analyzes the file size\n" +
- " distribution in the image.\n" +
- " -maxSize specifies the range [0, maxSize] of file sizes to be\n" +
- " analyzed (128GB by default).\n" +
- " -step defines the granularity of the distribution. (2MB by default)\n" +
- " * NameDistribution: This processor analyzes the file names\n" +
- " in the image and prints total number of file names and how frequently\n" +
- " file names are reused.\n" +
- "\n" +
- "Required command line arguments:\n" +
- "-i,--inputFile <arg> FSImage file to process.\n" +
- "-o,--outputFile <arg> Name of output file. If the specified\n" +
- " file exists, it will be overwritten.\n" +
- "\n" +
- "Optional command line arguments:\n" +
- "-p,--processor <arg> Select which type of processor to apply\n" +
- " against image file." +
- " (Ls|XML|Delimited|Indented|FileDistribution).\n" +
- "-h,--help Display usage information and exit\n" +
- "-printToScreen For processors that write to a file, also\n" +
- " output to screen. On large image files this\n" +
- " will dramatically increase processing time.\n" +
- "-skipBlocks Skip inodes' blocks information. May\n" +
- " significantly decrease output.\n" +
- " (default = false).\n" +
- "-delimiter <arg> Delimiting string to use with Delimited processor\n";
+ private final static String usage =
+ "Usage: bin/hdfs oiv_legacy [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n"
+ + "Offline Image Viewer\n"
+ + "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n"
+ + "saving the results in OUTPUTFILE.\n"
+ + "\n"
+ + "The oiv utility will attempt to parse correctly formed image files\n"
+ + "and will abort fail with mal-formed image files.\n"
+ + "\n"
+ + "The tool works offline and does not require a running cluster in\n"
+ + "order to process an image file.\n"
+ + "\n"
+ + "The following image processors are available:\n"
+ + " * Ls: The default image processor generates an lsr-style listing\n"
+ + " of the files in the namespace, with the same fields in the same\n"
+ + " order. Note that in order to correctly determine file sizes,\n"
+ + " this formatter cannot skip blocks and will override the\n"
+ + " -skipBlocks option.\n"
+ + " * Indented: This processor enumerates over all of the elements in\n"
+ + " the fsimage file, using levels of indentation to delineate\n"
+ + " sections within the file.\n"
+ + " * Delimited: Generate a text file with all of the elements common\n"
+ + " to both inodes and inodes-under-construction, separated by a\n"
+ + " delimiter. The default delimiter is \u0001, though this may be\n"
+ + " changed via the -delimiter argument. This processor also overrides\n"
+ + " the -skipBlocks option for the same reason as the Ls processor\n"
+ + " * XML: This processor creates an XML document with all elements of\n"
+ + " the fsimage enumerated, suitable for further analysis by XML\n"
+ + " tools.\n"
+ + " * FileDistribution: This processor analyzes the file size\n"
+ + " distribution in the image.\n"
+ + " -maxSize specifies the range [0, maxSize] of file sizes to be\n"
+ + " analyzed (128GB by default).\n"
+ + " -step defines the granularity of the distribution. (2MB by default)\n"
+ + " -format formats the output result in a human-readable fashion\n"
+ + " rather than a number of bytes. (false by default)\n"
+ + " * NameDistribution: This processor analyzes the file names\n"
+ + " in the image and prints total number of file names and how frequently\n"
+ + " file names are reused.\n"
+ + "\n"
+ + "Required command line arguments:\n"
+ + "-i,--inputFile <arg> FSImage file to process.\n"
+ + "-o,--outputFile <arg> Name of output file. If the specified\n"
+ + " file exists, it will be overwritten.\n"
+ + "\n"
+ + "Optional command line arguments:\n"
+ + "-p,--processor <arg> Select which type of processor to apply\n"
+ + " against image file."
+ + " (Ls|XML|Delimited|Indented|FileDistribution).\n"
+ + "-h,--help Display usage information and exit\n"
+ + "-printToScreen For processors that write to a file, also\n"
+ + " output to screen. On large image files this\n"
+ + " will dramatically increase processing time.\n"
+ + "-skipBlocks Skip inodes' blocks information. May\n"
+ + " significantly decrease output.\n"
+ + " (default = false).\n"
+ + "-delimiter <arg> Delimiting string to use with Delimited processor\n";
private final boolean skipBlocks;
private final String inputFile;
@@ -188,6 +190,7 @@ public class OfflineImageViewer {
options.addOption("h", "help", false, "");
options.addOption("maxSize", true, "");
options.addOption("step", true, "");
+ options.addOption("format", false, "");
options.addOption("skipBlocks", false, "");
options.addOption("printToScreen", false, "");
options.addOption("delimiter", true, "");
@@ -253,7 +256,8 @@ public class OfflineImageViewer {
} else if (processor.equals("FileDistribution")) {
long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
- v = new FileDistributionVisitor(outputFile, maxSize, step);
+ boolean formatOutput = cmd.hasOption("format");
+ v = new FileDistributionVisitor(outputFile, maxSize, step, formatOutput);
} else if (processor.equals("NameDistribution")) {
v = new NameDistributionVisitor(outputFile, printToScreen);
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
index b514b3f..c1141f3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewerPB.java
@@ -67,6 +67,8 @@ public class OfflineImageViewerPB {
+ " -maxSize specifies the range [0, maxSize] of file sizes to be\n"
+ " analyzed (128GB by default).\n"
+ " -step defines the granularity of the distribution. (2MB by default)\n"
+ + " -format formats the output result in a human-readable fashion\n"
+ + " rather than a number of bytes. (false by default)\n"
+ " * Web: Run a viewer to expose read-only WebHDFS API.\n"
+ " -addr specifies the address to listen. (localhost:5978 by default)\n"
+ " * Delimited (experimental): Generate a text file with all of the elements common\n"
@@ -111,6 +113,7 @@ public class OfflineImageViewerPB {
options.addOption("h", "help", false, "");
options.addOption("maxSize", true, "");
options.addOption("step", true, "");
+ options.addOption("format", false, "");
options.addOption("addr", true, "");
options.addOption("delimiter", true, "");
options.addOption("t", "temp", true, "");
@@ -172,43 +175,44 @@ public class OfflineImageViewerPB {
try (PrintStream out = outputFile.equals("-") ?
System.out : new PrintStream(outputFile, "UTF-8")) {
switch (processor) {
- case "FileDistribution":
- long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
- int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
- new FileDistributionCalculator(conf, maxSize, step, out).visit(
- new RandomAccessFile(inputFile, "r"));
- break;
- case "XML":
- new PBImageXmlWriter(conf, out).visit(
- new RandomAccessFile(inputFile, "r"));
- break;
- case "ReverseXML":
- try {
- OfflineImageReconstructor.run(inputFile, outputFile);
- } catch (Exception e) {
- System.err.println("OfflineImageReconstructor failed: " +
- e.getMessage());
- e.printStackTrace(System.err);
- System.exit(1);
- }
- break;
- case "Web":
- String addr = cmd.getOptionValue("addr", "localhost:5978");
- try (WebImageViewer viewer = new WebImageViewer(
- NetUtils.createSocketAddr(addr))) {
- viewer.start(inputFile);
- }
- break;
- case "Delimited":
- try (PBImageDelimitedTextWriter writer =
- new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
- writer.visit(new RandomAccessFile(inputFile, "r"));
- }
- break;
- default:
- System.err.println("Invalid processor specified : " + processor);
- printUsage();
- return -1;
+ case "FileDistribution":
+ long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
+ int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
+ boolean formatOutput = cmd.hasOption("format");
+ new FileDistributionCalculator(conf, maxSize, step, formatOutput, out)
+ .visit(new RandomAccessFile(inputFile, "r"));
+ break;
+ case "XML":
+ new PBImageXmlWriter(conf, out).visit(new RandomAccessFile(inputFile,
+ "r"));
+ break;
+ case "ReverseXML":
+ try {
+ OfflineImageReconstructor.run(inputFile, outputFile);
+ } catch (Exception e) {
+ System.err.println("OfflineImageReconstructor failed: "
+ + e.getMessage());
+ e.printStackTrace(System.err);
+ System.exit(1);
+ }
+ break;
+ case "Web":
+ String addr = cmd.getOptionValue("addr", "localhost:5978");
+ try (WebImageViewer viewer =
+ new WebImageViewer(NetUtils.createSocketAddr(addr))) {
+ viewer.start(inputFile);
+ }
+ break;
+ case "Delimited":
+ try (PBImageDelimitedTextWriter writer =
+ new PBImageDelimitedTextWriter(out, delimiter, tempPath)) {
+ writer.visit(new RandomAccessFile(inputFile, "r"));
+ }
+ break;
+ default:
+ System.err.println("Invalid processor specified : " + processor);
+ printUsage();
+ return -1;
}
return 0;
} catch (EOFException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 22886d3..4075878 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -239,6 +239,7 @@ Usage: `hdfs oiv [OPTIONS] -i INPUT_FILE`
| `-addr` *address* | Specify the address(host:port) to listen. (localhost:5978 by default). This option is used with Web processor. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
+| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
| `-delimiter` *arg* | Delimiting string to use with Delimited processor. |
| `-t`,`--temp` *temporary dir* | Use temporary dir to cache intermediate result to generate Delimited outputs. If not set, Delimited processor constructs the namespace in memory before outputting text. |
| `-h`,`--help` | Display the tool usage and help information and exit. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
index de27fc2..f991b44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsImageViewer.md
@@ -150,6 +150,7 @@ Options
| `-addr` *address* | Specify the address(host:port) to listen. (localhost:5978 by default). This option is used with Web processor. |
| `-maxSize` *size* | Specify the range [0, maxSize] of file sizes to be analyzed in bytes (128GB by default). This option is used with FileDistribution processor. |
| `-step` *size* | Specify the granularity of the distribution in bytes (2MB by default). This option is used with FileDistribution processor. |
+| `-format` | Format the output result in a human-readable fashion rather than a number of bytes. (false by default). This option is used with FileDistribution processor. |
| `-delimiter` *arg* | Delimiting string to use with Delimited processor. |
| `-t`\|`--temp` *temporary dir* | Use temporary dir to cache intermediate result to generate Delimited outputs. If not set, Delimited processor constructs the namespace in memory before outputting text. |
| `-h`\|`--help` | Display the tool usage and help information and exit. |
http://git-wip-us.apache.org/repos/asf/hadoop/blob/63f59489/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index a7c30ec..740a8ab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -237,7 +237,7 @@ public class TestOfflineImageViewer {
File truncatedFile = new File(tempDir, "truncatedFsImage");
PrintStream output = new PrintStream(NullOutputStream.NULL_OUTPUT_STREAM);
copyPartOfFile(originalFsimage, truncatedFile);
- new FileDistributionCalculator(new Configuration(), 0, 0, output)
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, output)
.visit(new RandomAccessFile(truncatedFile, "r"));
}
@@ -259,7 +259,7 @@ public class TestOfflineImageViewer {
public void testFileDistributionCalculator() throws IOException {
ByteArrayOutputStream output = new ByteArrayOutputStream();
PrintStream o = new PrintStream(output);
- new FileDistributionCalculator(new Configuration(), 0, 0, o)
+ new FileDistributionCalculator(new Configuration(), 0, 0, false, o)
.visit(new RandomAccessFile(originalFsimage, "r"));
o.close();
@@ -620,4 +620,24 @@ public class TestOfflineImageViewer {
IOUtils.closeStream(out);
}
}
+
+ @Test
+ public void testOfflineImageViewerWithFormatOption() throws Exception {
+ final ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ final PrintStream out = new PrintStream(bytes);
+ final PrintStream oldOut = System.out;
+ try {
+ System.setOut(out);
+ int status =
+ OfflineImageViewerPB.run(new String[] {"-i",
+ originalFsimage.getAbsolutePath(), "-o", "-", "-p",
+ "FileDistribution", "-maxSize", "512", "-step", "8",
+ "-format"});
+ assertEquals(0, status);
+ Assert.assertTrue(bytes.toString().contains("(0 B, 8 B]"));
+ } finally {
+ System.setOut(oldOut);
+ IOUtils.closeStream(out);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/48] hadoop git commit: HADOOP-13361. Modify hadoop_verify_user to
be consistent with hadoop_subcommand_opts (ie more granularity)
Posted by aw...@apache.org.
HADOOP-13361. Modify hadoop_verify_user to be consistent with hadoop_subcommand_opts (ie more granularity)
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/59b59ca8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/59b59ca8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/59b59ca8
Branch: refs/heads/HADOOP-13341
Commit: 59b59ca8564a09c06e4c0b2aed73ef1e94c087dc
Parents: 7f75b08
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 13:55:04 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/bin/hadoop | 4 +-
.../src/main/bin/hadoop-functions.sh | 24 +++++++--
.../hadoop-common/src/main/conf/hadoop-env.sh | 3 +-
.../src/test/scripts/hadoop_verify_user.bats | 53 ++++++++++++++++++++
.../hadoop-hdfs/src/main/bin/hdfs | 4 +-
hadoop-mapreduce-project/bin/mapred | 4 +-
hadoop-yarn-project/hadoop-yarn/bin/yarn | 4 +-
7 files changed, 82 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index 9b682e6..450543d 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -190,6 +190,8 @@ fi
HADOOP_SUBCMD=$1
shift
+hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
HADOOP_SUBCMD_ARGS=("$@")
if declare -f hadoop_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
@@ -199,8 +201,6 @@ else
hadoopcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
-hadoop_verify_user "${HADOOP_SUBCMD}"
-
hadoop_add_client_opts
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index db868bd..91546d4 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -1984,12 +1984,26 @@ function hadoop_secure_daemon_handler
## @return will exit on failure conditions
function hadoop_verify_user
{
- local command=$1
- local uservar="HADOOP_${command}_USER"
+ declare program=$1
+ declare command=$2
+ declare uprogram
+ declare ucommand
+ declare uvar
- if [[ -n ${!uservar} ]]; then
- if [[ ${!uservar} != "${USER}" ]]; then
- hadoop_error "ERROR: ${command} can only be executed by ${!uservar}."
+ if [[ -z "${BASH_VERSINFO[0]}" ]] \
+ || [[ "${BASH_VERSINFO[0]}" -lt 4 ]]; then
+ uprogram=$(echo "${program}" | tr '[:lower:]' '[:upper:]')
+ ucommand=$(echo "${command}" | tr '[:lower:]' '[:upper:]')
+ else
+ uprogram=${program^^}
+ ucommand=${command^^}
+ fi
+
+ uvar="${uprogram}_${ucommand}_USER"
+
+ if [[ -n ${!uvar} ]]; then
+ if [[ ${!uvar} != "${USER}" ]]; then
+ hadoop_error "ERROR: ${command} can only be executed by ${!uvar}."
exit 1
fi
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index 4656f4d..a78f3f6 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -417,6 +417,7 @@ esac
#
# To prevent accidents, shell commands be (superficially) locked
# to only allow certain users to execute certain subcommands.
+# It uses the format of (command)_(subcommand)_USER.
#
# For example, to limit who can execute the namenode command,
-# export HADOOP_namenode_USER=hdfs
+# export HDFS_NAMENODE_USER=hdfs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats
new file mode 100644
index 0000000..ac9fa9f
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_verify_user.bats
@@ -0,0 +1,53 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_verify_user (hadoop: no setting)" {
+ run hadoop_verify_user hadoop test
+ [ "${status}" = "0" ]
+}
+
+@test "hadoop_verify_user (yarn: no setting)" {
+ run hadoop_verify_user yarn test
+ [ "${status}" = "0" ]
+}
+
+@test "hadoop_verify_user (hadoop: allow)" {
+ HADOOP_TEST_USER=${USER}
+ run hadoop_verify_user hadoop test
+ [ "${status}" = "0" ]
+}
+
+@test "hadoop_verify_user (yarn: allow)" {
+ YARN_TEST_USER=${USER}
+ run hadoop_verify_user yarn test
+ [ "${status}" = "0" ]
+}
+
+# colon isn't a valid username, so let's use it
+# this should fail regardless of who the user is
+# that is running the test code
+@test "hadoop_verify_user (hadoop: disallow)" {
+ HADOOP_TEST_USER=:
+ run hadoop_verify_user hadoop test
+ [ "${status}" = "1" ]
+}
+
+@test "hadoop_verify_user (yarn: disallow)" {
+ YARN_TEST_USER=:
+ run hadoop_verify_user yarn test
+ [ "${status}" = "1" ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index b704b00..5ecea29 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -246,6 +246,8 @@ fi
HADOOP_SUBCMD=$1
shift
+hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
HADOOP_SUBCMD_ARGS=("$@")
if declare -f hdfs_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
@@ -255,8 +257,6 @@ else
hdfscmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
-hadoop_verify_user "${HADOOP_SUBCMD}"
-
hadoop_add_client_opts
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 3243d20..c566d5a 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -131,6 +131,8 @@ fi
HADOOP_SUBCMD=$1
shift
+hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
HADOOP_SUBCMD_ARGS=("$@")
if declare -f mapred_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
@@ -140,8 +142,6 @@ else
mapredcmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
-hadoop_verify_user "${HADOOP_SUBCMD}"
-
hadoop_add_client_opts
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
http://git-wip-us.apache.org/repos/asf/hadoop/blob/59b59ca8/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index d2a8a50..804fd1a 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -228,6 +228,8 @@ fi
HADOOP_SUBCMD=$1
shift
+hadoop_verify_user "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
HADOOP_SUBCMD_ARGS=("$@")
if declare -f yarn_subcommand_"${HADOOP_SUBCMD}" >/dev/null 2>&1; then
@@ -237,8 +239,6 @@ else
yarncmd_case "${HADOOP_SUBCMD}" "${HADOOP_SUBCMD_ARGS[@]}"
fi
-hadoop_verify_user "${HADOOP_SUBCMD}"
-
# It's unclear if YARN_CLIENT_OPTS is actually a useful
# thing to have separate from HADOOP_CLIENT_OPTS. Someone
# might use it, so let's not deprecate it and just override
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/48] hadoop git commit: HADOOP-13355. Handle HADOOP_CLIENT_OPTS in
a function (aw)
Posted by aw...@apache.org.
HADOOP-13355. Handle HADOOP_CLIENT_OPTS in a function (aw)
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0e011b79
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0e011b79
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0e011b79
Branch: refs/heads/HADOOP-13341
Commit: 0e011b79143e3fc2d9063b466bd17ecdfe7a9ac6
Parents: f462c9d
Author: Allen Wittenauer <aw...@apache.org>
Authored: Fri Aug 26 14:33:33 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/bin/hadoop | 6 +--
.../src/main/bin/hadoop-functions.sh | 21 ++++++++++
.../test/scripts/hadoop_add_client_opts.bats | 40 ++++++++++++++++++++
.../hadoop-hdfs/src/main/bin/hdfs | 14 +------
hadoop-mapreduce-project/bin/mapred | 12 +-----
.../main/shellprofile.d/hadoop-archive-logs.sh | 2 -
.../src/main/shellprofile.d/hadoop-distcp.sh | 4 --
.../src/main/shellprofile.d/hadoop-extras.sh | 2 -
.../src/main/shellprofile.d/hadoop-rumen.sh | 4 --
.../hadoop-sls/src/main/bin/rumen2sls.sh | 3 +-
hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh | 3 +-
.../src/main/shellprofile.d/hadoop-streaming.sh | 4 --
hadoop-yarn-project/hadoop-yarn/bin/yarn | 34 ++++++-----------
13 files changed, 81 insertions(+), 68 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-common-project/hadoop-common/src/main/bin/hadoop
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
index b57a4c1..4aae621 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop
@@ -161,10 +161,6 @@ function hadoopcmd_case
fi
;;
esac
-
- # Always respect HADOOP_OPTS and HADOOP_CLIENT_OPTS
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
# This script runs the hadoop core commands.
@@ -205,6 +201,8 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}"
+hadoop_add_client_opts
+
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_worker_mode_execute "${HADOOP_COMMON_HOME}/bin/hadoop" "${HADOOP_USER_PARAMS[@]}"
exit $?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 6e58dca..9003913 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -306,6 +306,13 @@ function hadoop_bootstrap
HADOOP_TOOLS_DIR=${HADOOP_TOOLS_DIR:-"share/hadoop/tools"}
HADOOP_TOOLS_LIB_JARS_DIR=${HADOOP_TOOLS_LIB_JARS_DIR:-"${HADOOP_TOOLS_DIR}/lib"}
+ # by default, whatever we are about to run doesn't support
+ # daemonization
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION=false
+
+ # shellcheck disable=SC2034
+ HADOOP_SUBCMD_SECURESERVICE=false
+
# usage output set to zero
hadoop_reset_usage
@@ -1230,6 +1237,20 @@ function hadoop_translate_cygwin_path
fi
}
+## @description Adds the HADOOP_CLIENT_OPTS variable to
+## @description HADOOP_OPTS if HADOOP_SUBCMD_SUPPORTDAEMONIZATION is false
+## @audience public
+## @stability stable
+## @replaceable yes
+function hadoop_add_client_opts
+{
+ if [[ "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" = false
+ || -z "${HADOOP_SUBCMD_SUPPORTDAEMONIZATION}" ]]; then
+ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
+ HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ fi
+}
+
## @description Finish configuring Hadoop specific system properties
## @description prior to executing Java
## @audience private
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_client_opts.bats
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_client_opts.bats b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_client_opts.bats
new file mode 100644
index 0000000..bc5051d
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/test/scripts/hadoop_add_client_opts.bats
@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+load hadoop-functions_test_helper
+
+@test "hadoop_subcommand_opts (daemonization false)" {
+ HADOOP_OPTS="1"
+ HADOOP_CLIENT_OPTS="2"
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="false"
+ hadoop_add_client_opts
+ [ "${HADOOP_OPTS}" = "1 2" ]
+}
+
+@test "hadoop_subcommand_opts (daemonization true)" {
+ HADOOP_OPTS="1"
+ HADOOP_CLIENT_OPTS="2"
+ HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
+ hadoop_add_client_opts
+ [ "${HADOOP_OPTS}" = "1" ]
+}
+
+@test "hadoop_subcommand_opts (daemonization empty)" {
+ HADOOP_OPTS="1"
+ HADOOP_CLIENT_OPTS="2"
+ unset HADOOP_SUBCMD_SUPPORTDAEMONIZATION
+ hadoop_add_client_opts
+ [ "${HADOOP_OPTS}" = "1 2" ]
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 5059528..41b3c12 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -118,18 +118,12 @@ function hdfscmd_case
;;
dfs)
HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
dfsadmin)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSAdmin
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
diskbalancer)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DiskBalancer
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
envvars)
echo "JAVA_HOME='${JAVA_HOME}'"
@@ -144,16 +138,12 @@ function hdfscmd_case
;;
erasurecode)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.erasurecode.ECCli
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
fetchdt)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
;;
fsck)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSck
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
getconf)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.GetConf
@@ -163,8 +153,6 @@ function hdfscmd_case
;;
haadmin)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.DFSHAAdmin
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
journalnode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -293,6 +281,8 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}"
+hadoop_add_client_opts
+
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_worker_mode_execute "${HADOOP_HDFS_HOME}/bin/hdfs" "${HADOOP_USER_PARAMS[@]}"
exit $?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-mapreduce-project/bin/mapred
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred
index 046d48c..3243d20 100755
--- a/hadoop-mapreduce-project/bin/mapred
+++ b/hadoop-mapreduce-project/bin/mapred
@@ -79,31 +79,21 @@ function mapredcmd_case
;;
hsadmin)
HADOOP_CLASSNAME=org.apache.hadoop.mapreduce.v2.hs.client.HSAdmin
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
job)
HADOOP_CLASSNAME=org.apache.hadoop.mapred.JobClient
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
pipes)
HADOOP_CLASSNAME=org.apache.hadoop.mapred.pipes.Submitter
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
queue)
HADOOP_CLASSNAME=org.apache.hadoop.mapred.JobQueueClient
;;
sampler)
HADOOP_CLASSNAME=org.apache.hadoop.mapred.lib.InputSampler
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
version)
HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
;;
*)
HADOOP_CLASSNAME="${subcmd}"
@@ -152,6 +142,8 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}"
+hadoop_add_client_opts
+
if [[ ${HADOOP_SLAVE_MODE} = true ]]; then
hadoop_common_slave_mode_execute "${HADOOP_MAPRED_HOME}/bin/mapred" "${HADOOP_USER_PARAMS[@]}"
exit $?
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
index ae7b6c6..c889816 100755
--- a/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
+++ b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh
@@ -32,8 +32,6 @@ function mapred_subcommand_archive-logs
# shellcheck disable=SC2034
HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs
hadoop_add_to_classpath_tools hadoop-archive-logs
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
index 0178c54..6e93ec1 100755
--- a/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
+++ b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh
@@ -32,8 +32,6 @@ function hadoop_subcommand_distcp
# shellcheck disable=SC2034
HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
hadoop_add_to_classpath_tools hadoop-distcp
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
fi
@@ -55,8 +53,6 @@ function mapred_subcommand_distcp
# shellcheck disable=SC2034
HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp
hadoop_add_to_classpath_tools hadoop-distcp
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
index 829d406..1ce9aee 100755
--- a/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
+++ b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh
@@ -32,8 +32,6 @@ function hadoop_subcommand_distch
# shellcheck disable=SC2034
HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh
hadoop_add_to_classpath_tools hadoop-extras
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh b/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
index d7d4022..77023ff 100755
--- a/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
+++ b/hadoop-tools/hadoop-rumen/src/main/shellprofile.d/hadoop-rumen.sh
@@ -30,8 +30,6 @@ function hadoop_subcommand_rumenfolder
# shellcheck disable=SC2034
HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.Folder
hadoop_add_to_classpath_tools hadoop-rumen
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
fi
@@ -51,8 +49,6 @@ function hadoop_subcommand_rumentrace
# shellcheck disable=SC2034
HADOOP_CLASSNAME=org.apache.hadoop.tools.rumen.TraceBuilder
hadoop_add_to_classpath_tools hadoop-rumen
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
}
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
index 0bd291b..565dfe6 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/rumen2sls.sh
@@ -68,8 +68,7 @@ function run_sls_generator()
hadoop_add_param args -outputJobs "-outputJobs ${outputdir}/${outputprefix}-jobs.json"
hadoop_add_param args -outputNodes "-outputNodes ${outputdir}/${outputprefix}-nodes.json"
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ hadoop_add_client_opts
hadoop_finalize
# shellcheck disable=SC2086
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
index 403c4bb..218dee4 100644
--- a/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
+++ b/hadoop-tools/hadoop-sls/src/main/bin/slsrun.sh
@@ -96,8 +96,7 @@ function run_simulation() {
hadoop_add_param args -printsimulation "-printsimulation"
fi
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
+ hadoop_add_client_opts
hadoop_finalize
# shellcheck disable=SC2086
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh b/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
index cca016d..c3010ff 100755
--- a/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
+++ b/hadoop-tools/hadoop-streaming/src/main/shellprofile.d/hadoop-streaming.sh
@@ -46,10 +46,6 @@ function mapred_subcommand_streaming
done
IFS=${oldifs}
-
- hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}"
-
}
fi
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0e011b79/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index bd91633..66a87b6 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -68,8 +68,6 @@ function yarncmd_case
case ${subcmd} in
application|applicationattempt|container)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ApplicationCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
set -- "${subcmd}" "$@"
HADOOP_SUBCMD_ARGS=("$@")
;;
@@ -78,13 +76,9 @@ function yarncmd_case
;;
cluster)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.ClusterCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto YARN_OPTS"
- YARN_OPTS="${YARN_OPTS} ${YARN_CLIENT_OPTS}"
;;
daemonlog)
HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
envvars)
echo "JAVA_HOME='${JAVA_HOME}'"
@@ -99,8 +93,6 @@ function yarncmd_case
;;
jar)
HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
historyserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -111,13 +103,9 @@ function yarncmd_case
;;
logs)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.LogsCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
node)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.NodeCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
nodemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -142,8 +130,6 @@ function yarncmd_case
;;
queue)
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.QueueCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
resourcemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -158,13 +144,9 @@ function yarncmd_case
;;
rmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.cli.RMAdminCLI'
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
scmadmin)
HADOOP_CLASSNAME='org.apache.hadoop.yarn.client.SCMAdmin'
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
@@ -191,8 +173,6 @@ function yarncmd_case
;;
version)
HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
;;
top)
doNotSetCols=0
@@ -222,8 +202,6 @@ function yarncmd_case
fi
fi
HADOOP_CLASSNAME=org.apache.hadoop.yarn.client.cli.TopCLI
- hadoop_debug "Append YARN_CLIENT_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_CLIENT_OPTS}"
HADOOP_SUBCMD_ARGS=("$@")
;;
*)
@@ -273,6 +251,18 @@ fi
hadoop_verify_user "${HADOOP_SUBCMD}"
+# It's unclear if YARN_CLIENT_OPTS is actually a useful
+# thing to have separate from HADOOP_CLIENT_OPTS. Someone
+# might use it, so let's not deprecate it and just override
+# HADOOP_CLIENT_OPTS instead before we (potentially) add it
+# to the command line
+if [[ -n "${YARN_CLIENT_OPTS}" ]]; then
+ # shellcheck disable=SC2034
+ HADOOP_CLIENT_OPTS=${YARN_CLIENT_OPTS}
+fi
+
+hadoop_add_client_opts
+
if [[ ${HADOOP_WORKER_MODE} = true ]]; then
hadoop_common_worker_mode_execute "${HADOOP_YARN_HOME}/bin/yarn" "${HADOOP_USER_PARAMS[@]}"
exit $?
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/48] hadoop git commit: HADOOP-13558. UserGroupInformation created
from a Subject incorrectly tries to renew the Kerberos ticket. Contributed by
Xiao Chen.
Posted by aw...@apache.org.
HADOOP-13558. UserGroupInformation created from a Subject incorrectly tries to renew the Kerberos ticket. Contributed by Xiao Chen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/680be58a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/680be58a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/680be58a
Branch: refs/heads/HADOOP-13341
Commit: 680be58aac03a9ffab6b07c8fde9602ddb9dc858
Parents: 5f23abf
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 20:25:26 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 20:54:17 2016 -0700
----------------------------------------------------------------------
.../hadoop/security/UserGroupInformation.java | 22 +++++++++++++++---
.../security/TestUserGroupInformation.java | 24 ++++++++++++++++++++
2 files changed, 43 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/680be58a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
index fe6fbe4..ed3a9d0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/UserGroupInformation.java
@@ -632,9 +632,24 @@ public class UserGroupInformation {
* @param subject the user's subject
*/
UserGroupInformation(Subject subject) {
+ this(subject, false);
+ }
+
+ /**
+ * Create a UGI from the given subject.
+ * @param subject the subject
+ * @param externalKeyTab if the subject's keytab is managed by the user.
+ * Setting this to true will prevent UGI from attempting
+ * to login the keytab, or to renew it.
+ */
+ private UserGroupInformation(Subject subject, final boolean externalKeyTab) {
this.subject = subject;
this.user = subject.getPrincipals(User.class).iterator().next();
- this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject);
+ if (externalKeyTab) {
+ this.isKeytab = false;
+ } else {
+ this.isKeytab = KerberosUtil.hasKerberosKeyTab(subject);
+ }
this.isKrbTkt = KerberosUtil.hasKerberosTicket(subject);
}
@@ -850,10 +865,11 @@ public class UserGroupInformation {
newLoginContext(authenticationMethod.getLoginAppName(),
subject, new HadoopConfiguration());
login.login();
- UserGroupInformation realUser = new UserGroupInformation(subject);
+ LOG.debug("Assuming keytab is managed externally since logged in from"
+ + " subject.");
+ UserGroupInformation realUser = new UserGroupInformation(subject, true);
realUser.setLogin(login);
realUser.setAuthenticationMethod(authenticationMethod);
- realUser = new UserGroupInformation(login.getSubject());
// If the HADOOP_PROXY_USER environment variable or property
// is specified, create a proxy user as the logged in user.
String proxyUser = System.getenv(HADOOP_PROXY_USER);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/680be58a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
index a306d35..e45d70d 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestUserGroupInformation.java
@@ -37,6 +37,7 @@ import org.junit.Test;
import javax.security.auth.Subject;
import javax.security.auth.kerberos.KerberosPrincipal;
+import javax.security.auth.kerberos.KeyTab;
import javax.security.auth.login.AppConfigurationEntry;
import javax.security.auth.login.LoginContext;
@@ -1030,4 +1031,27 @@ public class TestUserGroupInformation {
assertTrue(credsugiTokens.contains(token1));
assertTrue(credsugiTokens.contains(token2));
}
+
+ @Test
+ public void testCheckTGTAfterLoginFromSubject() throws Exception {
+ // security on, default is remove default realm
+ SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ UserGroupInformation.setConfiguration(conf);
+
+ // Login from a pre-set subject with a keytab
+ final Subject subject = new Subject();
+ KeyTab keytab = KeyTab.getInstance();
+ subject.getPrivateCredentials().add(keytab);
+ UserGroupInformation ugi = UserGroupInformation.getCurrentUser();
+ ugi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws IOException {
+ UserGroupInformation.loginUserFromSubject(subject);
+ // this should not throw.
+ UserGroupInformation.getLoginUser().checkTGTAndReloginFromKeytab();
+ return null;
+ }
+ });
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/48] hadoop git commit: HDFS-10822. Log DataNodes in the write
pipeline. John Zhuge via Lei Xu
Posted by aw...@apache.org.
HDFS-10822. Log DataNodes in the write pipeline. John Zhuge via Lei Xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a8c5064
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a8c5064
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a8c5064
Branch: refs/heads/HADOOP-13341
Commit: 5a8c5064d1a1d596b1f5c385299a86ec6ab9ad1e
Parents: 23abb09
Author: Lei Xu <le...@apache.org>
Authored: Fri Sep 2 11:01:13 2016 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Fri Sep 2 11:01:13 2016 -0700
----------------------------------------------------------------------
.../src/main/java/org/apache/hadoop/hdfs/DataStreamer.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a8c5064/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
index 062a416..5166c8c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DataStreamer.java
@@ -508,6 +508,12 @@ class DataStreamer extends Daemon {
private void initDataStreaming() {
this.setName("DataStreamer for file " + src +
" block " + block);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("nodes {} storageTypes {} storageIDs {}",
+ Arrays.toString(nodes),
+ Arrays.toString(storageTypes),
+ Arrays.toString(storageIDs));
+ }
response = new ResponseProcessor(nodes);
response.start();
stage = BlockConstructionStage.DATA_STREAMING;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/48] hadoop git commit: FsPermission string constructor does not
recognize sticky bit. Contributed by Atul Sikaria
Posted by aw...@apache.org.
FsPermission string constructor does not recognize sticky bit. Contributed by Atul Sikaria
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3069df75
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3069df75
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3069df75
Branch: refs/heads/HADOOP-13341
Commit: 3069df75513180a5b56130a7e05c6e339ab0a66a
Parents: dcd21d0
Author: Chris Douglas <cd...@apache.org>
Authored: Thu Sep 1 11:32:26 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Thu Sep 1 11:32:26 2016 -0700
----------------------------------------------------------------------
.../hadoop/fs/permission/FsPermission.java | 2 +-
.../apache/hadoop/fs/permission/RawParser.java | 44 +
.../hadoop/fs/permission/TestFsPermission.java | 1061 +++++++++---------
3 files changed, 585 insertions(+), 522 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3069df75/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index b535fd6..78255bbd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -103,7 +103,7 @@ public class FsPermission implements Writable {
* @throws IllegalArgumentException if <code>mode</code> is invalid
*/
public FsPermission(String mode) {
- this(new UmaskParser(mode).getUMask());
+ this(new RawParser(mode).getPermission());
}
/** Return user {@link FsAction}. */
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3069df75/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java
new file mode 100644
index 0000000..3bbe9cb
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/RawParser.java
@@ -0,0 +1,44 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class RawParser extends PermissionParser {
+ private static Pattern rawOctalPattern =
+ Pattern.compile("^\\s*([01]?)([0-7]{3})\\s*$");
+ private static Pattern rawNormalPattern =
+ Pattern.compile("\\G\\s*([ugoa]*)([+=-]+)([rwxt]*)([,\\s]*)\\s*");
+
+ private short permission;
+
+ public RawParser(String modeStr) throws IllegalArgumentException {
+ super(modeStr, rawNormalPattern, rawOctalPattern);
+ permission = (short)combineModes(0, false);
+ }
+
+ public short getPermission() {
+ return permission;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3069df75/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
index 04dbe01..6368a57 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/permission/TestFsPermission.java
@@ -56,6 +56,8 @@ public class TestFsPermission extends TestCase {
public void testConvertingPermissions() {
for(short s = 0; s <= 01777; s++) {
assertEquals(s, new FsPermission(s).toShort());
+ // check string formats
+ assertEquals(s, new FsPermission(String.format("%03o", s)).toShort());
}
short s = 0;
@@ -75,6 +77,7 @@ public class TestFsPermission extends TestCase {
}
}
}
+ assertEquals(02000, s);
}
public void testSpecialBitsToString() {
@@ -129,6 +132,20 @@ public class TestFsPermission extends TestCase {
}
}
+ public void testSymbolicPermission() {
+ for (int i = 0; i < SYMBOLIC.length; ++i) {
+ short val = 0777;
+ val &= ~Short.valueOf(SYMBOLIC[i][1], 8);
+ assertEquals(val, new FsPermission(SYMBOLIC[i][0]).toShort());
+ }
+ // add sticky bit to "other" when exec enabled
+ for (int i = 1; i < SYMBOLIC.length; i += 2) {
+ short val = 01777;
+ val &= ~Short.valueOf(SYMBOLIC[i][1], 8);
+ assertEquals(val, new FsPermission(SYMBOLIC[i][0] + "t").toShort());
+ }
+ }
+
public void testUMaskParser() throws IOException {
Configuration conf = new Configuration();
@@ -150,529 +167,13 @@ public class TestFsPermission extends TestCase {
Configuration conf = new Configuration();
// Test some symbolic to octal settings
- // Symbolic umask list is generated in linux shell using by the command:
- // umask 0; umask <octal number>; umask -S
- String [][] symbolic = new String [][] {
- {"a+rw", "111",},
- {"u=rwx,g=rwx,o=rwx", "0",},
- {"u=rwx,g=rwx,o=rw", "1",},
- {"u=rwx,g=rwx,o=rx", "2",},
- {"u=rwx,g=rwx,o=r", "3",},
- {"u=rwx,g=rwx,o=wx", "4",},
- {"u=rwx,g=rwx,o=w", "5",},
- {"u=rwx,g=rwx,o=x", "6",},
- {"u=rwx,g=rwx,o=", "7",},
- {"u=rwx,g=rw,o=rwx", "10",},
- {"u=rwx,g=rw,o=rw", "11",},
- {"u=rwx,g=rw,o=rx", "12",},
- {"u=rwx,g=rw,o=r", "13",},
- {"u=rwx,g=rw,o=wx", "14",},
- {"u=rwx,g=rw,o=w", "15",},
- {"u=rwx,g=rw,o=x", "16",},
- {"u=rwx,g=rw,o=", "17",},
- {"u=rwx,g=rx,o=rwx", "20",},
- {"u=rwx,g=rx,o=rw", "21",},
- {"u=rwx,g=rx,o=rx", "22",},
- {"u=rwx,g=rx,o=r", "23",},
- {"u=rwx,g=rx,o=wx", "24",},
- {"u=rwx,g=rx,o=w", "25",},
- {"u=rwx,g=rx,o=x", "26",},
- {"u=rwx,g=rx,o=", "27",},
- {"u=rwx,g=r,o=rwx", "30",},
- {"u=rwx,g=r,o=rw", "31",},
- {"u=rwx,g=r,o=rx", "32",},
- {"u=rwx,g=r,o=r", "33",},
- {"u=rwx,g=r,o=wx", "34",},
- {"u=rwx,g=r,o=w", "35",},
- {"u=rwx,g=r,o=x", "36",},
- {"u=rwx,g=r,o=", "37",},
- {"u=rwx,g=wx,o=rwx", "40",},
- {"u=rwx,g=wx,o=rw", "41",},
- {"u=rwx,g=wx,o=rx", "42",},
- {"u=rwx,g=wx,o=r", "43",},
- {"u=rwx,g=wx,o=wx", "44",},
- {"u=rwx,g=wx,o=w", "45",},
- {"u=rwx,g=wx,o=x", "46",},
- {"u=rwx,g=wx,o=", "47",},
- {"u=rwx,g=w,o=rwx", "50",},
- {"u=rwx,g=w,o=rw", "51",},
- {"u=rwx,g=w,o=rx", "52",},
- {"u=rwx,g=w,o=r", "53",},
- {"u=rwx,g=w,o=wx", "54",},
- {"u=rwx,g=w,o=w", "55",},
- {"u=rwx,g=w,o=x", "56",},
- {"u=rwx,g=w,o=", "57",},
- {"u=rwx,g=x,o=rwx", "60",},
- {"u=rwx,g=x,o=rw", "61",},
- {"u=rwx,g=x,o=rx", "62",},
- {"u=rwx,g=x,o=r", "63",},
- {"u=rwx,g=x,o=wx", "64",},
- {"u=rwx,g=x,o=w", "65",},
- {"u=rwx,g=x,o=x", "66",},
- {"u=rwx,g=x,o=", "67",},
- {"u=rwx,g=,o=rwx", "70",},
- {"u=rwx,g=,o=rw", "71",},
- {"u=rwx,g=,o=rx", "72",},
- {"u=rwx,g=,o=r", "73",},
- {"u=rwx,g=,o=wx", "74",},
- {"u=rwx,g=,o=w", "75",},
- {"u=rwx,g=,o=x", "76",},
- {"u=rwx,g=,o=", "77",},
- {"u=rw,g=rwx,o=rwx", "100",},
- {"u=rw,g=rwx,o=rw", "101",},
- {"u=rw,g=rwx,o=rx", "102",},
- {"u=rw,g=rwx,o=r", "103",},
- {"u=rw,g=rwx,o=wx", "104",},
- {"u=rw,g=rwx,o=w", "105",},
- {"u=rw,g=rwx,o=x", "106",},
- {"u=rw,g=rwx,o=", "107",},
- {"u=rw,g=rw,o=rwx", "110",},
- {"u=rw,g=rw,o=rw", "111",},
- {"u=rw,g=rw,o=rx", "112",},
- {"u=rw,g=rw,o=r", "113",},
- {"u=rw,g=rw,o=wx", "114",},
- {"u=rw,g=rw,o=w", "115",},
- {"u=rw,g=rw,o=x", "116",},
- {"u=rw,g=rw,o=", "117",},
- {"u=rw,g=rx,o=rwx", "120",},
- {"u=rw,g=rx,o=rw", "121",},
- {"u=rw,g=rx,o=rx", "122",},
- {"u=rw,g=rx,o=r", "123",},
- {"u=rw,g=rx,o=wx", "124",},
- {"u=rw,g=rx,o=w", "125",},
- {"u=rw,g=rx,o=x", "126",},
- {"u=rw,g=rx,o=", "127",},
- {"u=rw,g=r,o=rwx", "130",},
- {"u=rw,g=r,o=rw", "131",},
- {"u=rw,g=r,o=rx", "132",},
- {"u=rw,g=r,o=r", "133",},
- {"u=rw,g=r,o=wx", "134",},
- {"u=rw,g=r,o=w", "135",},
- {"u=rw,g=r,o=x", "136",},
- {"u=rw,g=r,o=", "137",},
- {"u=rw,g=wx,o=rwx", "140",},
- {"u=rw,g=wx,o=rw", "141",},
- {"u=rw,g=wx,o=rx", "142",},
- {"u=rw,g=wx,o=r", "143",},
- {"u=rw,g=wx,o=wx", "144",},
- {"u=rw,g=wx,o=w", "145",},
- {"u=rw,g=wx,o=x", "146",},
- {"u=rw,g=wx,o=", "147",},
- {"u=rw,g=w,o=rwx", "150",},
- {"u=rw,g=w,o=rw", "151",},
- {"u=rw,g=w,o=rx", "152",},
- {"u=rw,g=w,o=r", "153",},
- {"u=rw,g=w,o=wx", "154",},
- {"u=rw,g=w,o=w", "155",},
- {"u=rw,g=w,o=x", "156",},
- {"u=rw,g=w,o=", "157",},
- {"u=rw,g=x,o=rwx", "160",},
- {"u=rw,g=x,o=rw", "161",},
- {"u=rw,g=x,o=rx", "162",},
- {"u=rw,g=x,o=r", "163",},
- {"u=rw,g=x,o=wx", "164",},
- {"u=rw,g=x,o=w", "165",},
- {"u=rw,g=x,o=x", "166",},
- {"u=rw,g=x,o=", "167",},
- {"u=rw,g=,o=rwx", "170",},
- {"u=rw,g=,o=rw", "171",},
- {"u=rw,g=,o=rx", "172",},
- {"u=rw,g=,o=r", "173",},
- {"u=rw,g=,o=wx", "174",},
- {"u=rw,g=,o=w", "175",},
- {"u=rw,g=,o=x", "176",},
- {"u=rw,g=,o=", "177",},
- {"u=rx,g=rwx,o=rwx", "200",},
- {"u=rx,g=rwx,o=rw", "201",},
- {"u=rx,g=rwx,o=rx", "202",},
- {"u=rx,g=rwx,o=r", "203",},
- {"u=rx,g=rwx,o=wx", "204",},
- {"u=rx,g=rwx,o=w", "205",},
- {"u=rx,g=rwx,o=x", "206",},
- {"u=rx,g=rwx,o=", "207",},
- {"u=rx,g=rw,o=rwx", "210",},
- {"u=rx,g=rw,o=rw", "211",},
- {"u=rx,g=rw,o=rx", "212",},
- {"u=rx,g=rw,o=r", "213",},
- {"u=rx,g=rw,o=wx", "214",},
- {"u=rx,g=rw,o=w", "215",},
- {"u=rx,g=rw,o=x", "216",},
- {"u=rx,g=rw,o=", "217",},
- {"u=rx,g=rx,o=rwx", "220",},
- {"u=rx,g=rx,o=rw", "221",},
- {"u=rx,g=rx,o=rx", "222",},
- {"u=rx,g=rx,o=r", "223",},
- {"u=rx,g=rx,o=wx", "224",},
- {"u=rx,g=rx,o=w", "225",},
- {"u=rx,g=rx,o=x", "226",},
- {"u=rx,g=rx,o=", "227",},
- {"u=rx,g=r,o=rwx", "230",},
- {"u=rx,g=r,o=rw", "231",},
- {"u=rx,g=r,o=rx", "232",},
- {"u=rx,g=r,o=r", "233",},
- {"u=rx,g=r,o=wx", "234",},
- {"u=rx,g=r,o=w", "235",},
- {"u=rx,g=r,o=x", "236",},
- {"u=rx,g=r,o=", "237",},
- {"u=rx,g=wx,o=rwx", "240",},
- {"u=rx,g=wx,o=rw", "241",},
- {"u=rx,g=wx,o=rx", "242",},
- {"u=rx,g=wx,o=r", "243",},
- {"u=rx,g=wx,o=wx", "244",},
- {"u=rx,g=wx,o=w", "245",},
- {"u=rx,g=wx,o=x", "246",},
- {"u=rx,g=wx,o=", "247",},
- {"u=rx,g=w,o=rwx", "250",},
- {"u=rx,g=w,o=rw", "251",},
- {"u=rx,g=w,o=rx", "252",},
- {"u=rx,g=w,o=r", "253",},
- {"u=rx,g=w,o=wx", "254",},
- {"u=rx,g=w,o=w", "255",},
- {"u=rx,g=w,o=x", "256",},
- {"u=rx,g=w,o=", "257",},
- {"u=rx,g=x,o=rwx", "260",},
- {"u=rx,g=x,o=rw", "261",},
- {"u=rx,g=x,o=rx", "262",},
- {"u=rx,g=x,o=r", "263",},
- {"u=rx,g=x,o=wx", "264",},
- {"u=rx,g=x,o=w", "265",},
- {"u=rx,g=x,o=x", "266",},
- {"u=rx,g=x,o=", "267",},
- {"u=rx,g=,o=rwx", "270",},
- {"u=rx,g=,o=rw", "271",},
- {"u=rx,g=,o=rx", "272",},
- {"u=rx,g=,o=r", "273",},
- {"u=rx,g=,o=wx", "274",},
- {"u=rx,g=,o=w", "275",},
- {"u=rx,g=,o=x", "276",},
- {"u=rx,g=,o=", "277",},
- {"u=r,g=rwx,o=rwx", "300",},
- {"u=r,g=rwx,o=rw", "301",},
- {"u=r,g=rwx,o=rx", "302",},
- {"u=r,g=rwx,o=r", "303",},
- {"u=r,g=rwx,o=wx", "304",},
- {"u=r,g=rwx,o=w", "305",},
- {"u=r,g=rwx,o=x", "306",},
- {"u=r,g=rwx,o=", "307",},
- {"u=r,g=rw,o=rwx", "310",},
- {"u=r,g=rw,o=rw", "311",},
- {"u=r,g=rw,o=rx", "312",},
- {"u=r,g=rw,o=r", "313",},
- {"u=r,g=rw,o=wx", "314",},
- {"u=r,g=rw,o=w", "315",},
- {"u=r,g=rw,o=x", "316",},
- {"u=r,g=rw,o=", "317",},
- {"u=r,g=rx,o=rwx", "320",},
- {"u=r,g=rx,o=rw", "321",},
- {"u=r,g=rx,o=rx", "322",},
- {"u=r,g=rx,o=r", "323",},
- {"u=r,g=rx,o=wx", "324",},
- {"u=r,g=rx,o=w", "325",},
- {"u=r,g=rx,o=x", "326",},
- {"u=r,g=rx,o=", "327",},
- {"u=r,g=r,o=rwx", "330",},
- {"u=r,g=r,o=rw", "331",},
- {"u=r,g=r,o=rx", "332",},
- {"u=r,g=r,o=r", "333",},
- {"u=r,g=r,o=wx", "334",},
- {"u=r,g=r,o=w", "335",},
- {"u=r,g=r,o=x", "336",},
- {"u=r,g=r,o=", "337",},
- {"u=r,g=wx,o=rwx", "340",},
- {"u=r,g=wx,o=rw", "341",},
- {"u=r,g=wx,o=rx", "342",},
- {"u=r,g=wx,o=r", "343",},
- {"u=r,g=wx,o=wx", "344",},
- {"u=r,g=wx,o=w", "345",},
- {"u=r,g=wx,o=x", "346",},
- {"u=r,g=wx,o=", "347",},
- {"u=r,g=w,o=rwx", "350",},
- {"u=r,g=w,o=rw", "351",},
- {"u=r,g=w,o=rx", "352",},
- {"u=r,g=w,o=r", "353",},
- {"u=r,g=w,o=wx", "354",},
- {"u=r,g=w,o=w", "355",},
- {"u=r,g=w,o=x", "356",},
- {"u=r,g=w,o=", "357",},
- {"u=r,g=x,o=rwx", "360",},
- {"u=r,g=x,o=rw", "361",},
- {"u=r,g=x,o=rx", "362",},
- {"u=r,g=x,o=r", "363",},
- {"u=r,g=x,o=wx", "364",},
- {"u=r,g=x,o=w", "365",},
- {"u=r,g=x,o=x", "366",},
- {"u=r,g=x,o=", "367",},
- {"u=r,g=,o=rwx", "370",},
- {"u=r,g=,o=rw", "371",},
- {"u=r,g=,o=rx", "372",},
- {"u=r,g=,o=r", "373",},
- {"u=r,g=,o=wx", "374",},
- {"u=r,g=,o=w", "375",},
- {"u=r,g=,o=x", "376",},
- {"u=r,g=,o=", "377",},
- {"u=wx,g=rwx,o=rwx", "400",},
- {"u=wx,g=rwx,o=rw", "401",},
- {"u=wx,g=rwx,o=rx", "402",},
- {"u=wx,g=rwx,o=r", "403",},
- {"u=wx,g=rwx,o=wx", "404",},
- {"u=wx,g=rwx,o=w", "405",},
- {"u=wx,g=rwx,o=x", "406",},
- {"u=wx,g=rwx,o=", "407",},
- {"u=wx,g=rw,o=rwx", "410",},
- {"u=wx,g=rw,o=rw", "411",},
- {"u=wx,g=rw,o=rx", "412",},
- {"u=wx,g=rw,o=r", "413",},
- {"u=wx,g=rw,o=wx", "414",},
- {"u=wx,g=rw,o=w", "415",},
- {"u=wx,g=rw,o=x", "416",},
- {"u=wx,g=rw,o=", "417",},
- {"u=wx,g=rx,o=rwx", "420",},
- {"u=wx,g=rx,o=rw", "421",},
- {"u=wx,g=rx,o=rx", "422",},
- {"u=wx,g=rx,o=r", "423",},
- {"u=wx,g=rx,o=wx", "424",},
- {"u=wx,g=rx,o=w", "425",},
- {"u=wx,g=rx,o=x", "426",},
- {"u=wx,g=rx,o=", "427",},
- {"u=wx,g=r,o=rwx", "430",},
- {"u=wx,g=r,o=rw", "431",},
- {"u=wx,g=r,o=rx", "432",},
- {"u=wx,g=r,o=r", "433",},
- {"u=wx,g=r,o=wx", "434",},
- {"u=wx,g=r,o=w", "435",},
- {"u=wx,g=r,o=x", "436",},
- {"u=wx,g=r,o=", "437",},
- {"u=wx,g=wx,o=rwx", "440",},
- {"u=wx,g=wx,o=rw", "441",},
- {"u=wx,g=wx,o=rx", "442",},
- {"u=wx,g=wx,o=r", "443",},
- {"u=wx,g=wx,o=wx", "444",},
- {"u=wx,g=wx,o=w", "445",},
- {"u=wx,g=wx,o=x", "446",},
- {"u=wx,g=wx,o=", "447",},
- {"u=wx,g=w,o=rwx", "450",},
- {"u=wx,g=w,o=rw", "451",},
- {"u=wx,g=w,o=rx", "452",},
- {"u=wx,g=w,o=r", "453",},
- {"u=wx,g=w,o=wx", "454",},
- {"u=wx,g=w,o=w", "455",},
- {"u=wx,g=w,o=x", "456",},
- {"u=wx,g=w,o=", "457",},
- {"u=wx,g=x,o=rwx", "460",},
- {"u=wx,g=x,o=rw", "461",},
- {"u=wx,g=x,o=rx", "462",},
- {"u=wx,g=x,o=r", "463",},
- {"u=wx,g=x,o=wx", "464",},
- {"u=wx,g=x,o=w", "465",},
- {"u=wx,g=x,o=x", "466",},
- {"u=wx,g=x,o=", "467",},
- {"u=wx,g=,o=rwx", "470",},
- {"u=wx,g=,o=rw", "471",},
- {"u=wx,g=,o=rx", "472",},
- {"u=wx,g=,o=r", "473",},
- {"u=wx,g=,o=wx", "474",},
- {"u=wx,g=,o=w", "475",},
- {"u=wx,g=,o=x", "476",},
- {"u=wx,g=,o=", "477",},
- {"u=w,g=rwx,o=rwx", "500",},
- {"u=w,g=rwx,o=rw", "501",},
- {"u=w,g=rwx,o=rx", "502",},
- {"u=w,g=rwx,o=r", "503",},
- {"u=w,g=rwx,o=wx", "504",},
- {"u=w,g=rwx,o=w", "505",},
- {"u=w,g=rwx,o=x", "506",},
- {"u=w,g=rwx,o=", "507",},
- {"u=w,g=rw,o=rwx", "510",},
- {"u=w,g=rw,o=rw", "511",},
- {"u=w,g=rw,o=rx", "512",},
- {"u=w,g=rw,o=r", "513",},
- {"u=w,g=rw,o=wx", "514",},
- {"u=w,g=rw,o=w", "515",},
- {"u=w,g=rw,o=x", "516",},
- {"u=w,g=rw,o=", "517",},
- {"u=w,g=rx,o=rwx", "520",},
- {"u=w,g=rx,o=rw", "521",},
- {"u=w,g=rx,o=rx", "522",},
- {"u=w,g=rx,o=r", "523",},
- {"u=w,g=rx,o=wx", "524",},
- {"u=w,g=rx,o=w", "525",},
- {"u=w,g=rx,o=x", "526",},
- {"u=w,g=rx,o=", "527",},
- {"u=w,g=r,o=rwx", "530",},
- {"u=w,g=r,o=rw", "531",},
- {"u=w,g=r,o=rx", "532",},
- {"u=w,g=r,o=r", "533",},
- {"u=w,g=r,o=wx", "534",},
- {"u=w,g=r,o=w", "535",},
- {"u=w,g=r,o=x", "536",},
- {"u=w,g=r,o=", "537",},
- {"u=w,g=wx,o=rwx", "540",},
- {"u=w,g=wx,o=rw", "541",},
- {"u=w,g=wx,o=rx", "542",},
- {"u=w,g=wx,o=r", "543",},
- {"u=w,g=wx,o=wx", "544",},
- {"u=w,g=wx,o=w", "545",},
- {"u=w,g=wx,o=x", "546",},
- {"u=w,g=wx,o=", "547",},
- {"u=w,g=w,o=rwx", "550",},
- {"u=w,g=w,o=rw", "551",},
- {"u=w,g=w,o=rx", "552",},
- {"u=w,g=w,o=r", "553",},
- {"u=w,g=w,o=wx", "554",},
- {"u=w,g=w,o=w", "555",},
- {"u=w,g=w,o=x", "556",},
- {"u=w,g=w,o=", "557",},
- {"u=w,g=x,o=rwx", "560",},
- {"u=w,g=x,o=rw", "561",},
- {"u=w,g=x,o=rx", "562",},
- {"u=w,g=x,o=r", "563",},
- {"u=w,g=x,o=wx", "564",},
- {"u=w,g=x,o=w", "565",},
- {"u=w,g=x,o=x", "566",},
- {"u=w,g=x,o=", "567",},
- {"u=w,g=,o=rwx", "570",},
- {"u=w,g=,o=rw", "571",},
- {"u=w,g=,o=rx", "572",},
- {"u=w,g=,o=r", "573",},
- {"u=w,g=,o=wx", "574",},
- {"u=w,g=,o=w", "575",},
- {"u=w,g=,o=x", "576",},
- {"u=w,g=,o=", "577",},
- {"u=x,g=rwx,o=rwx", "600",},
- {"u=x,g=rwx,o=rw", "601",},
- {"u=x,g=rwx,o=rx", "602",},
- {"u=x,g=rwx,o=r", "603",},
- {"u=x,g=rwx,o=wx", "604",},
- {"u=x,g=rwx,o=w", "605",},
- {"u=x,g=rwx,o=x", "606",},
- {"u=x,g=rwx,o=", "607",},
- {"u=x,g=rw,o=rwx", "610",},
- {"u=x,g=rw,o=rw", "611",},
- {"u=x,g=rw,o=rx", "612",},
- {"u=x,g=rw,o=r", "613",},
- {"u=x,g=rw,o=wx", "614",},
- {"u=x,g=rw,o=w", "615",},
- {"u=x,g=rw,o=x", "616",},
- {"u=x,g=rw,o=", "617",},
- {"u=x,g=rx,o=rwx", "620",},
- {"u=x,g=rx,o=rw", "621",},
- {"u=x,g=rx,o=rx", "622",},
- {"u=x,g=rx,o=r", "623",},
- {"u=x,g=rx,o=wx", "624",},
- {"u=x,g=rx,o=w", "625",},
- {"u=x,g=rx,o=x", "626",},
- {"u=x,g=rx,o=", "627",},
- {"u=x,g=r,o=rwx", "630",},
- {"u=x,g=r,o=rw", "631",},
- {"u=x,g=r,o=rx", "632",},
- {"u=x,g=r,o=r", "633",},
- {"u=x,g=r,o=wx", "634",},
- {"u=x,g=r,o=w", "635",},
- {"u=x,g=r,o=x", "636",},
- {"u=x,g=r,o=", "637",},
- {"u=x,g=wx,o=rwx", "640",},
- {"u=x,g=wx,o=rw", "641",},
- {"u=x,g=wx,o=rx", "642",},
- {"u=x,g=wx,o=r", "643",},
- {"u=x,g=wx,o=wx", "644",},
- {"u=x,g=wx,o=w", "645",},
- {"u=x,g=wx,o=x", "646",},
- {"u=x,g=wx,o=", "647",},
- {"u=x,g=w,o=rwx", "650",},
- {"u=x,g=w,o=rw", "651",},
- {"u=x,g=w,o=rx", "652",},
- {"u=x,g=w,o=r", "653",},
- {"u=x,g=w,o=wx", "654",},
- {"u=x,g=w,o=w", "655",},
- {"u=x,g=w,o=x", "656",},
- {"u=x,g=w,o=", "657",},
- {"u=x,g=x,o=rwx", "660",},
- {"u=x,g=x,o=rw", "661",},
- {"u=x,g=x,o=rx", "662",},
- {"u=x,g=x,o=r", "663",},
- {"u=x,g=x,o=wx", "664",},
- {"u=x,g=x,o=w", "665",},
- {"u=x,g=x,o=x", "666",},
- {"u=x,g=x,o=", "667",},
- {"u=x,g=,o=rwx", "670",},
- {"u=x,g=,o=rw", "671",},
- {"u=x,g=,o=rx", "672",},
- {"u=x,g=,o=r", "673",},
- {"u=x,g=,o=wx", "674",},
- {"u=x,g=,o=w", "675",},
- {"u=x,g=,o=x", "676",},
- {"u=x,g=,o=", "677",},
- {"u=,g=rwx,o=rwx", "700",},
- {"u=,g=rwx,o=rw", "701",},
- {"u=,g=rwx,o=rx", "702",},
- {"u=,g=rwx,o=r", "703",},
- {"u=,g=rwx,o=wx", "704",},
- {"u=,g=rwx,o=w", "705",},
- {"u=,g=rwx,o=x", "706",},
- {"u=,g=rwx,o=", "707",},
- {"u=,g=rw,o=rwx", "710",},
- {"u=,g=rw,o=rw", "711",},
- {"u=,g=rw,o=rx", "712",},
- {"u=,g=rw,o=r", "713",},
- {"u=,g=rw,o=wx", "714",},
- {"u=,g=rw,o=w", "715",},
- {"u=,g=rw,o=x", "716",},
- {"u=,g=rw,o=", "717",},
- {"u=,g=rx,o=rwx", "720",},
- {"u=,g=rx,o=rw", "721",},
- {"u=,g=rx,o=rx", "722",},
- {"u=,g=rx,o=r", "723",},
- {"u=,g=rx,o=wx", "724",},
- {"u=,g=rx,o=w", "725",},
- {"u=,g=rx,o=x", "726",},
- {"u=,g=rx,o=", "727",},
- {"u=,g=r,o=rwx", "730",},
- {"u=,g=r,o=rw", "731",},
- {"u=,g=r,o=rx", "732",},
- {"u=,g=r,o=r", "733",},
- {"u=,g=r,o=wx", "734",},
- {"u=,g=r,o=w", "735",},
- {"u=,g=r,o=x", "736",},
- {"u=,g=r,o=", "737",},
- {"u=,g=wx,o=rwx", "740",},
- {"u=,g=wx,o=rw", "741",},
- {"u=,g=wx,o=rx", "742",},
- {"u=,g=wx,o=r", "743",},
- {"u=,g=wx,o=wx", "744",},
- {"u=,g=wx,o=w", "745",},
- {"u=,g=wx,o=x", "746",},
- {"u=,g=wx,o=", "747",},
- {"u=,g=w,o=rwx", "750",},
- {"u=,g=w,o=rw", "751",},
- {"u=,g=w,o=rx", "752",},
- {"u=,g=w,o=r", "753",},
- {"u=,g=w,o=wx", "754",},
- {"u=,g=w,o=w", "755",},
- {"u=,g=w,o=x", "756",},
- {"u=,g=w,o=", "757",},
- {"u=,g=x,o=rwx", "760",},
- {"u=,g=x,o=rw", "761",},
- {"u=,g=x,o=rx", "762",},
- {"u=,g=x,o=r", "763",},
- {"u=,g=x,o=wx", "764",},
- {"u=,g=x,o=w", "765",},
- {"u=,g=x,o=x", "766",},
- {"u=,g=x,o=", "767",},
- {"u=,g=,o=rwx", "770",},
- {"u=,g=,o=rw", "771",},
- {"u=,g=,o=rx", "772",},
- {"u=,g=,o=r", "773",},
- {"u=,g=,o=wx", "774",},
- {"u=,g=,o=w", "775",},
- {"u=,g=,o=x", "776",},
- {"u=,g=,o=", "777"}
- };
-
- for(int i = 0; i < symbolic.length; i += 2) {
- conf.set(FsPermission.UMASK_LABEL, symbolic[i][0]);
- short val = Short.valueOf(symbolic[i][1], 8);
+ for (int i = 0; i < SYMBOLIC.length; ++i) {
+ conf.set(FsPermission.UMASK_LABEL, SYMBOLIC[i][0]);
+ short val = Short.valueOf(SYMBOLIC[i][1], 8);
assertEquals(val, FsPermission.getUMask(conf).toShort());
}
+ conf.set(FsPermission.UMASK_LABEL, "a+rw");
+ assertEquals(0111, FsPermission.getUMask(conf).toShort());
}
public void testBadUmasks() {
@@ -695,4 +196,522 @@ public class TestFsPermission extends TestCase {
msg.contains(umask) &&
msg.contains("octal or symbolic");
}
+
+ // Symbolic umask list is generated in linux shell using by the command:
+ // umask 0; umask <octal number>; umask -S
+ static final String[][] SYMBOLIC = new String[][] {
+ {"u=rwx,g=rwx,o=rwx", "0"},
+ {"u=rwx,g=rwx,o=rw", "1"},
+ {"u=rwx,g=rwx,o=rx", "2"},
+ {"u=rwx,g=rwx,o=r", "3"},
+ {"u=rwx,g=rwx,o=wx", "4"},
+ {"u=rwx,g=rwx,o=w", "5"},
+ {"u=rwx,g=rwx,o=x", "6"},
+ {"u=rwx,g=rwx,o=", "7"},
+ {"u=rwx,g=rw,o=rwx", "10"},
+ {"u=rwx,g=rw,o=rw", "11"},
+ {"u=rwx,g=rw,o=rx", "12"},
+ {"u=rwx,g=rw,o=r", "13"},
+ {"u=rwx,g=rw,o=wx", "14"},
+ {"u=rwx,g=rw,o=w", "15"},
+ {"u=rwx,g=rw,o=x", "16"},
+ {"u=rwx,g=rw,o=", "17"},
+ {"u=rwx,g=rx,o=rwx", "20"},
+ {"u=rwx,g=rx,o=rw", "21"},
+ {"u=rwx,g=rx,o=rx", "22"},
+ {"u=rwx,g=rx,o=r", "23"},
+ {"u=rwx,g=rx,o=wx", "24"},
+ {"u=rwx,g=rx,o=w", "25"},
+ {"u=rwx,g=rx,o=x", "26"},
+ {"u=rwx,g=rx,o=", "27"},
+ {"u=rwx,g=r,o=rwx", "30"},
+ {"u=rwx,g=r,o=rw", "31"},
+ {"u=rwx,g=r,o=rx", "32"},
+ {"u=rwx,g=r,o=r", "33"},
+ {"u=rwx,g=r,o=wx", "34"},
+ {"u=rwx,g=r,o=w", "35"},
+ {"u=rwx,g=r,o=x", "36"},
+ {"u=rwx,g=r,o=", "37"},
+ {"u=rwx,g=wx,o=rwx", "40"},
+ {"u=rwx,g=wx,o=rw", "41"},
+ {"u=rwx,g=wx,o=rx", "42"},
+ {"u=rwx,g=wx,o=r", "43"},
+ {"u=rwx,g=wx,o=wx", "44"},
+ {"u=rwx,g=wx,o=w", "45"},
+ {"u=rwx,g=wx,o=x", "46"},
+ {"u=rwx,g=wx,o=", "47"},
+ {"u=rwx,g=w,o=rwx", "50"},
+ {"u=rwx,g=w,o=rw", "51"},
+ {"u=rwx,g=w,o=rx", "52"},
+ {"u=rwx,g=w,o=r", "53"},
+ {"u=rwx,g=w,o=wx", "54"},
+ {"u=rwx,g=w,o=w", "55"},
+ {"u=rwx,g=w,o=x", "56"},
+ {"u=rwx,g=w,o=", "57"},
+ {"u=rwx,g=x,o=rwx", "60"},
+ {"u=rwx,g=x,o=rw", "61"},
+ {"u=rwx,g=x,o=rx", "62"},
+ {"u=rwx,g=x,o=r", "63"},
+ {"u=rwx,g=x,o=wx", "64"},
+ {"u=rwx,g=x,o=w", "65"},
+ {"u=rwx,g=x,o=x", "66"},
+ {"u=rwx,g=x,o=", "67"},
+ {"u=rwx,g=,o=rwx", "70"},
+ {"u=rwx,g=,o=rw", "71"},
+ {"u=rwx,g=,o=rx", "72"},
+ {"u=rwx,g=,o=r", "73"},
+ {"u=rwx,g=,o=wx", "74"},
+ {"u=rwx,g=,o=w", "75"},
+ {"u=rwx,g=,o=x", "76"},
+ {"u=rwx,g=,o=", "77"},
+ {"u=rw,g=rwx,o=rwx", "100"},
+ {"u=rw,g=rwx,o=rw", "101"},
+ {"u=rw,g=rwx,o=rx", "102"},
+ {"u=rw,g=rwx,o=r", "103"},
+ {"u=rw,g=rwx,o=wx", "104"},
+ {"u=rw,g=rwx,o=w", "105"},
+ {"u=rw,g=rwx,o=x", "106"},
+ {"u=rw,g=rwx,o=", "107"},
+ {"u=rw,g=rw,o=rwx", "110"},
+ {"u=rw,g=rw,o=rw", "111"},
+ {"u=rw,g=rw,o=rx", "112"},
+ {"u=rw,g=rw,o=r", "113"},
+ {"u=rw,g=rw,o=wx", "114"},
+ {"u=rw,g=rw,o=w", "115"},
+ {"u=rw,g=rw,o=x", "116"},
+ {"u=rw,g=rw,o=", "117"},
+ {"u=rw,g=rx,o=rwx", "120"},
+ {"u=rw,g=rx,o=rw", "121"},
+ {"u=rw,g=rx,o=rx", "122"},
+ {"u=rw,g=rx,o=r", "123"},
+ {"u=rw,g=rx,o=wx", "124"},
+ {"u=rw,g=rx,o=w", "125"},
+ {"u=rw,g=rx,o=x", "126"},
+ {"u=rw,g=rx,o=", "127"},
+ {"u=rw,g=r,o=rwx", "130"},
+ {"u=rw,g=r,o=rw", "131"},
+ {"u=rw,g=r,o=rx", "132"},
+ {"u=rw,g=r,o=r", "133"},
+ {"u=rw,g=r,o=wx", "134"},
+ {"u=rw,g=r,o=w", "135"},
+ {"u=rw,g=r,o=x", "136"},
+ {"u=rw,g=r,o=", "137"},
+ {"u=rw,g=wx,o=rwx", "140"},
+ {"u=rw,g=wx,o=rw", "141"},
+ {"u=rw,g=wx,o=rx", "142"},
+ {"u=rw,g=wx,o=r", "143"},
+ {"u=rw,g=wx,o=wx", "144"},
+ {"u=rw,g=wx,o=w", "145"},
+ {"u=rw,g=wx,o=x", "146"},
+ {"u=rw,g=wx,o=", "147"},
+ {"u=rw,g=w,o=rwx", "150"},
+ {"u=rw,g=w,o=rw", "151"},
+ {"u=rw,g=w,o=rx", "152"},
+ {"u=rw,g=w,o=r", "153"},
+ {"u=rw,g=w,o=wx", "154"},
+ {"u=rw,g=w,o=w", "155"},
+ {"u=rw,g=w,o=x", "156"},
+ {"u=rw,g=w,o=", "157"},
+ {"u=rw,g=x,o=rwx", "160"},
+ {"u=rw,g=x,o=rw", "161"},
+ {"u=rw,g=x,o=rx", "162"},
+ {"u=rw,g=x,o=r", "163"},
+ {"u=rw,g=x,o=wx", "164"},
+ {"u=rw,g=x,o=w", "165"},
+ {"u=rw,g=x,o=x", "166"},
+ {"u=rw,g=x,o=", "167"},
+ {"u=rw,g=,o=rwx", "170"},
+ {"u=rw,g=,o=rw", "171"},
+ {"u=rw,g=,o=rx", "172"},
+ {"u=rw,g=,o=r", "173"},
+ {"u=rw,g=,o=wx", "174"},
+ {"u=rw,g=,o=w", "175"},
+ {"u=rw,g=,o=x", "176"},
+ {"u=rw,g=,o=", "177"},
+ {"u=rx,g=rwx,o=rwx", "200"},
+ {"u=rx,g=rwx,o=rw", "201"},
+ {"u=rx,g=rwx,o=rx", "202"},
+ {"u=rx,g=rwx,o=r", "203"},
+ {"u=rx,g=rwx,o=wx", "204"},
+ {"u=rx,g=rwx,o=w", "205"},
+ {"u=rx,g=rwx,o=x", "206"},
+ {"u=rx,g=rwx,o=", "207"},
+ {"u=rx,g=rw,o=rwx", "210"},
+ {"u=rx,g=rw,o=rw", "211"},
+ {"u=rx,g=rw,o=rx", "212"},
+ {"u=rx,g=rw,o=r", "213"},
+ {"u=rx,g=rw,o=wx", "214"},
+ {"u=rx,g=rw,o=w", "215"},
+ {"u=rx,g=rw,o=x", "216"},
+ {"u=rx,g=rw,o=", "217"},
+ {"u=rx,g=rx,o=rwx", "220"},
+ {"u=rx,g=rx,o=rw", "221"},
+ {"u=rx,g=rx,o=rx", "222"},
+ {"u=rx,g=rx,o=r", "223"},
+ {"u=rx,g=rx,o=wx", "224"},
+ {"u=rx,g=rx,o=w", "225"},
+ {"u=rx,g=rx,o=x", "226"},
+ {"u=rx,g=rx,o=", "227"},
+ {"u=rx,g=r,o=rwx", "230"},
+ {"u=rx,g=r,o=rw", "231"},
+ {"u=rx,g=r,o=rx", "232"},
+ {"u=rx,g=r,o=r", "233"},
+ {"u=rx,g=r,o=wx", "234"},
+ {"u=rx,g=r,o=w", "235"},
+ {"u=rx,g=r,o=x", "236"},
+ {"u=rx,g=r,o=", "237"},
+ {"u=rx,g=wx,o=rwx", "240"},
+ {"u=rx,g=wx,o=rw", "241"},
+ {"u=rx,g=wx,o=rx", "242"},
+ {"u=rx,g=wx,o=r", "243"},
+ {"u=rx,g=wx,o=wx", "244"},
+ {"u=rx,g=wx,o=w", "245"},
+ {"u=rx,g=wx,o=x", "246"},
+ {"u=rx,g=wx,o=", "247"},
+ {"u=rx,g=w,o=rwx", "250"},
+ {"u=rx,g=w,o=rw", "251"},
+ {"u=rx,g=w,o=rx", "252"},
+ {"u=rx,g=w,o=r", "253"},
+ {"u=rx,g=w,o=wx", "254"},
+ {"u=rx,g=w,o=w", "255"},
+ {"u=rx,g=w,o=x", "256"},
+ {"u=rx,g=w,o=", "257"},
+ {"u=rx,g=x,o=rwx", "260"},
+ {"u=rx,g=x,o=rw", "261"},
+ {"u=rx,g=x,o=rx", "262"},
+ {"u=rx,g=x,o=r", "263"},
+ {"u=rx,g=x,o=wx", "264"},
+ {"u=rx,g=x,o=w", "265"},
+ {"u=rx,g=x,o=x", "266"},
+ {"u=rx,g=x,o=", "267"},
+ {"u=rx,g=,o=rwx", "270"},
+ {"u=rx,g=,o=rw", "271"},
+ {"u=rx,g=,o=rx", "272"},
+ {"u=rx,g=,o=r", "273"},
+ {"u=rx,g=,o=wx", "274"},
+ {"u=rx,g=,o=w", "275"},
+ {"u=rx,g=,o=x", "276"},
+ {"u=rx,g=,o=", "277"},
+ {"u=r,g=rwx,o=rwx", "300"},
+ {"u=r,g=rwx,o=rw", "301"},
+ {"u=r,g=rwx,o=rx", "302"},
+ {"u=r,g=rwx,o=r", "303"},
+ {"u=r,g=rwx,o=wx", "304"},
+ {"u=r,g=rwx,o=w", "305"},
+ {"u=r,g=rwx,o=x", "306"},
+ {"u=r,g=rwx,o=", "307"},
+ {"u=r,g=rw,o=rwx", "310"},
+ {"u=r,g=rw,o=rw", "311"},
+ {"u=r,g=rw,o=rx", "312"},
+ {"u=r,g=rw,o=r", "313"},
+ {"u=r,g=rw,o=wx", "314"},
+ {"u=r,g=rw,o=w", "315"},
+ {"u=r,g=rw,o=x", "316"},
+ {"u=r,g=rw,o=", "317"},
+ {"u=r,g=rx,o=rwx", "320"},
+ {"u=r,g=rx,o=rw", "321"},
+ {"u=r,g=rx,o=rx", "322"},
+ {"u=r,g=rx,o=r", "323"},
+ {"u=r,g=rx,o=wx", "324"},
+ {"u=r,g=rx,o=w", "325"},
+ {"u=r,g=rx,o=x", "326"},
+ {"u=r,g=rx,o=", "327"},
+ {"u=r,g=r,o=rwx", "330"},
+ {"u=r,g=r,o=rw", "331"},
+ {"u=r,g=r,o=rx", "332"},
+ {"u=r,g=r,o=r", "333"},
+ {"u=r,g=r,o=wx", "334"},
+ {"u=r,g=r,o=w", "335"},
+ {"u=r,g=r,o=x", "336"},
+ {"u=r,g=r,o=", "337"},
+ {"u=r,g=wx,o=rwx", "340"},
+ {"u=r,g=wx,o=rw", "341"},
+ {"u=r,g=wx,o=rx", "342"},
+ {"u=r,g=wx,o=r", "343"},
+ {"u=r,g=wx,o=wx", "344"},
+ {"u=r,g=wx,o=w", "345"},
+ {"u=r,g=wx,o=x", "346"},
+ {"u=r,g=wx,o=", "347"},
+ {"u=r,g=w,o=rwx", "350"},
+ {"u=r,g=w,o=rw", "351"},
+ {"u=r,g=w,o=rx", "352"},
+ {"u=r,g=w,o=r", "353"},
+ {"u=r,g=w,o=wx", "354"},
+ {"u=r,g=w,o=w", "355"},
+ {"u=r,g=w,o=x", "356"},
+ {"u=r,g=w,o=", "357"},
+ {"u=r,g=x,o=rwx", "360"},
+ {"u=r,g=x,o=rw", "361"},
+ {"u=r,g=x,o=rx", "362"},
+ {"u=r,g=x,o=r", "363"},
+ {"u=r,g=x,o=wx", "364"},
+ {"u=r,g=x,o=w", "365"},
+ {"u=r,g=x,o=x", "366"},
+ {"u=r,g=x,o=", "367"},
+ {"u=r,g=,o=rwx", "370"},
+ {"u=r,g=,o=rw", "371"},
+ {"u=r,g=,o=rx", "372"},
+ {"u=r,g=,o=r", "373"},
+ {"u=r,g=,o=wx", "374"},
+ {"u=r,g=,o=w", "375"},
+ {"u=r,g=,o=x", "376"},
+ {"u=r,g=,o=", "377"},
+ {"u=wx,g=rwx,o=rwx", "400"},
+ {"u=wx,g=rwx,o=rw", "401"},
+ {"u=wx,g=rwx,o=rx", "402"},
+ {"u=wx,g=rwx,o=r", "403"},
+ {"u=wx,g=rwx,o=wx", "404"},
+ {"u=wx,g=rwx,o=w", "405"},
+ {"u=wx,g=rwx,o=x", "406"},
+ {"u=wx,g=rwx,o=", "407"},
+ {"u=wx,g=rw,o=rwx", "410"},
+ {"u=wx,g=rw,o=rw", "411"},
+ {"u=wx,g=rw,o=rx", "412"},
+ {"u=wx,g=rw,o=r", "413"},
+ {"u=wx,g=rw,o=wx", "414"},
+ {"u=wx,g=rw,o=w", "415"},
+ {"u=wx,g=rw,o=x", "416"},
+ {"u=wx,g=rw,o=", "417"},
+ {"u=wx,g=rx,o=rwx", "420"},
+ {"u=wx,g=rx,o=rw", "421"},
+ {"u=wx,g=rx,o=rx", "422"},
+ {"u=wx,g=rx,o=r", "423"},
+ {"u=wx,g=rx,o=wx", "424"},
+ {"u=wx,g=rx,o=w", "425"},
+ {"u=wx,g=rx,o=x", "426"},
+ {"u=wx,g=rx,o=", "427"},
+ {"u=wx,g=r,o=rwx", "430"},
+ {"u=wx,g=r,o=rw", "431"},
+ {"u=wx,g=r,o=rx", "432"},
+ {"u=wx,g=r,o=r", "433"},
+ {"u=wx,g=r,o=wx", "434"},
+ {"u=wx,g=r,o=w", "435"},
+ {"u=wx,g=r,o=x", "436"},
+ {"u=wx,g=r,o=", "437"},
+ {"u=wx,g=wx,o=rwx", "440"},
+ {"u=wx,g=wx,o=rw", "441"},
+ {"u=wx,g=wx,o=rx", "442"},
+ {"u=wx,g=wx,o=r", "443"},
+ {"u=wx,g=wx,o=wx", "444"},
+ {"u=wx,g=wx,o=w", "445"},
+ {"u=wx,g=wx,o=x", "446"},
+ {"u=wx,g=wx,o=", "447"},
+ {"u=wx,g=w,o=rwx", "450"},
+ {"u=wx,g=w,o=rw", "451"},
+ {"u=wx,g=w,o=rx", "452"},
+ {"u=wx,g=w,o=r", "453"},
+ {"u=wx,g=w,o=wx", "454"},
+ {"u=wx,g=w,o=w", "455"},
+ {"u=wx,g=w,o=x", "456"},
+ {"u=wx,g=w,o=", "457"},
+ {"u=wx,g=x,o=rwx", "460"},
+ {"u=wx,g=x,o=rw", "461"},
+ {"u=wx,g=x,o=rx", "462"},
+ {"u=wx,g=x,o=r", "463"},
+ {"u=wx,g=x,o=wx", "464"},
+ {"u=wx,g=x,o=w", "465"},
+ {"u=wx,g=x,o=x", "466"},
+ {"u=wx,g=x,o=", "467"},
+ {"u=wx,g=,o=rwx", "470"},
+ {"u=wx,g=,o=rw", "471"},
+ {"u=wx,g=,o=rx", "472"},
+ {"u=wx,g=,o=r", "473"},
+ {"u=wx,g=,o=wx", "474"},
+ {"u=wx,g=,o=w", "475"},
+ {"u=wx,g=,o=x", "476"},
+ {"u=wx,g=,o=", "477"},
+ {"u=w,g=rwx,o=rwx", "500"},
+ {"u=w,g=rwx,o=rw", "501"},
+ {"u=w,g=rwx,o=rx", "502"},
+ {"u=w,g=rwx,o=r", "503"},
+ {"u=w,g=rwx,o=wx", "504"},
+ {"u=w,g=rwx,o=w", "505"},
+ {"u=w,g=rwx,o=x", "506"},
+ {"u=w,g=rwx,o=", "507"},
+ {"u=w,g=rw,o=rwx", "510"},
+ {"u=w,g=rw,o=rw", "511"},
+ {"u=w,g=rw,o=rx", "512"},
+ {"u=w,g=rw,o=r", "513"},
+ {"u=w,g=rw,o=wx", "514"},
+ {"u=w,g=rw,o=w", "515"},
+ {"u=w,g=rw,o=x", "516"},
+ {"u=w,g=rw,o=", "517"},
+ {"u=w,g=rx,o=rwx", "520"},
+ {"u=w,g=rx,o=rw", "521"},
+ {"u=w,g=rx,o=rx", "522"},
+ {"u=w,g=rx,o=r", "523"},
+ {"u=w,g=rx,o=wx", "524"},
+ {"u=w,g=rx,o=w", "525"},
+ {"u=w,g=rx,o=x", "526"},
+ {"u=w,g=rx,o=", "527"},
+ {"u=w,g=r,o=rwx", "530"},
+ {"u=w,g=r,o=rw", "531"},
+ {"u=w,g=r,o=rx", "532"},
+ {"u=w,g=r,o=r", "533"},
+ {"u=w,g=r,o=wx", "534"},
+ {"u=w,g=r,o=w", "535"},
+ {"u=w,g=r,o=x", "536"},
+ {"u=w,g=r,o=", "537"},
+ {"u=w,g=wx,o=rwx", "540"},
+ {"u=w,g=wx,o=rw", "541"},
+ {"u=w,g=wx,o=rx", "542"},
+ {"u=w,g=wx,o=r", "543"},
+ {"u=w,g=wx,o=wx", "544"},
+ {"u=w,g=wx,o=w", "545"},
+ {"u=w,g=wx,o=x", "546"},
+ {"u=w,g=wx,o=", "547"},
+ {"u=w,g=w,o=rwx", "550"},
+ {"u=w,g=w,o=rw", "551"},
+ {"u=w,g=w,o=rx", "552"},
+ {"u=w,g=w,o=r", "553"},
+ {"u=w,g=w,o=wx", "554"},
+ {"u=w,g=w,o=w", "555"},
+ {"u=w,g=w,o=x", "556"},
+ {"u=w,g=w,o=", "557"},
+ {"u=w,g=x,o=rwx", "560"},
+ {"u=w,g=x,o=rw", "561"},
+ {"u=w,g=x,o=rx", "562"},
+ {"u=w,g=x,o=r", "563"},
+ {"u=w,g=x,o=wx", "564"},
+ {"u=w,g=x,o=w", "565"},
+ {"u=w,g=x,o=x", "566"},
+ {"u=w,g=x,o=", "567"},
+ {"u=w,g=,o=rwx", "570"},
+ {"u=w,g=,o=rw", "571"},
+ {"u=w,g=,o=rx", "572"},
+ {"u=w,g=,o=r", "573"},
+ {"u=w,g=,o=wx", "574"},
+ {"u=w,g=,o=w", "575"},
+ {"u=w,g=,o=x", "576"},
+ {"u=w,g=,o=", "577"},
+ {"u=x,g=rwx,o=rwx", "600"},
+ {"u=x,g=rwx,o=rw", "601"},
+ {"u=x,g=rwx,o=rx", "602"},
+ {"u=x,g=rwx,o=r", "603"},
+ {"u=x,g=rwx,o=wx", "604"},
+ {"u=x,g=rwx,o=w", "605"},
+ {"u=x,g=rwx,o=x", "606"},
+ {"u=x,g=rwx,o=", "607"},
+ {"u=x,g=rw,o=rwx", "610"},
+ {"u=x,g=rw,o=rw", "611"},
+ {"u=x,g=rw,o=rx", "612"},
+ {"u=x,g=rw,o=r", "613"},
+ {"u=x,g=rw,o=wx", "614"},
+ {"u=x,g=rw,o=w", "615"},
+ {"u=x,g=rw,o=x", "616"},
+ {"u=x,g=rw,o=", "617"},
+ {"u=x,g=rx,o=rwx", "620"},
+ {"u=x,g=rx,o=rw", "621"},
+ {"u=x,g=rx,o=rx", "622"},
+ {"u=x,g=rx,o=r", "623"},
+ {"u=x,g=rx,o=wx", "624"},
+ {"u=x,g=rx,o=w", "625"},
+ {"u=x,g=rx,o=x", "626"},
+ {"u=x,g=rx,o=", "627"},
+ {"u=x,g=r,o=rwx", "630"},
+ {"u=x,g=r,o=rw", "631"},
+ {"u=x,g=r,o=rx", "632"},
+ {"u=x,g=r,o=r", "633"},
+ {"u=x,g=r,o=wx", "634"},
+ {"u=x,g=r,o=w", "635"},
+ {"u=x,g=r,o=x", "636"},
+ {"u=x,g=r,o=", "637"},
+ {"u=x,g=wx,o=rwx", "640"},
+ {"u=x,g=wx,o=rw", "641"},
+ {"u=x,g=wx,o=rx", "642"},
+ {"u=x,g=wx,o=r", "643"},
+ {"u=x,g=wx,o=wx", "644"},
+ {"u=x,g=wx,o=w", "645"},
+ {"u=x,g=wx,o=x", "646"},
+ {"u=x,g=wx,o=", "647"},
+ {"u=x,g=w,o=rwx", "650"},
+ {"u=x,g=w,o=rw", "651"},
+ {"u=x,g=w,o=rx", "652"},
+ {"u=x,g=w,o=r", "653"},
+ {"u=x,g=w,o=wx", "654"},
+ {"u=x,g=w,o=w", "655"},
+ {"u=x,g=w,o=x", "656"},
+ {"u=x,g=w,o=", "657"},
+ {"u=x,g=x,o=rwx", "660"},
+ {"u=x,g=x,o=rw", "661"},
+ {"u=x,g=x,o=rx", "662"},
+ {"u=x,g=x,o=r", "663"},
+ {"u=x,g=x,o=wx", "664"},
+ {"u=x,g=x,o=w", "665"},
+ {"u=x,g=x,o=x", "666"},
+ {"u=x,g=x,o=", "667"},
+ {"u=x,g=,o=rwx", "670"},
+ {"u=x,g=,o=rw", "671"},
+ {"u=x,g=,o=rx", "672"},
+ {"u=x,g=,o=r", "673"},
+ {"u=x,g=,o=wx", "674"},
+ {"u=x,g=,o=w", "675"},
+ {"u=x,g=,o=x", "676"},
+ {"u=x,g=,o=", "677"},
+ {"u=,g=rwx,o=rwx", "700"},
+ {"u=,g=rwx,o=rw", "701"},
+ {"u=,g=rwx,o=rx", "702"},
+ {"u=,g=rwx,o=r", "703"},
+ {"u=,g=rwx,o=wx", "704"},
+ {"u=,g=rwx,o=w", "705"},
+ {"u=,g=rwx,o=x", "706"},
+ {"u=,g=rwx,o=", "707"},
+ {"u=,g=rw,o=rwx", "710"},
+ {"u=,g=rw,o=rw", "711"},
+ {"u=,g=rw,o=rx", "712"},
+ {"u=,g=rw,o=r", "713"},
+ {"u=,g=rw,o=wx", "714"},
+ {"u=,g=rw,o=w", "715"},
+ {"u=,g=rw,o=x", "716"},
+ {"u=,g=rw,o=", "717"},
+ {"u=,g=rx,o=rwx", "720"},
+ {"u=,g=rx,o=rw", "721"},
+ {"u=,g=rx,o=rx", "722"},
+ {"u=,g=rx,o=r", "723"},
+ {"u=,g=rx,o=wx", "724"},
+ {"u=,g=rx,o=w", "725"},
+ {"u=,g=rx,o=x", "726"},
+ {"u=,g=rx,o=", "727"},
+ {"u=,g=r,o=rwx", "730"},
+ {"u=,g=r,o=rw", "731"},
+ {"u=,g=r,o=rx", "732"},
+ {"u=,g=r,o=r", "733"},
+ {"u=,g=r,o=wx", "734"},
+ {"u=,g=r,o=w", "735"},
+ {"u=,g=r,o=x", "736"},
+ {"u=,g=r,o=", "737"},
+ {"u=,g=wx,o=rwx", "740"},
+ {"u=,g=wx,o=rw", "741"},
+ {"u=,g=wx,o=rx", "742"},
+ {"u=,g=wx,o=r", "743"},
+ {"u=,g=wx,o=wx", "744"},
+ {"u=,g=wx,o=w", "745"},
+ {"u=,g=wx,o=x", "746"},
+ {"u=,g=wx,o=", "747"},
+ {"u=,g=w,o=rwx", "750"},
+ {"u=,g=w,o=rw", "751"},
+ {"u=,g=w,o=rx", "752"},
+ {"u=,g=w,o=r", "753"},
+ {"u=,g=w,o=wx", "754"},
+ {"u=,g=w,o=w", "755"},
+ {"u=,g=w,o=x", "756"},
+ {"u=,g=w,o=", "757"},
+ {"u=,g=x,o=rwx", "760"},
+ {"u=,g=x,o=rw", "761"},
+ {"u=,g=x,o=rx", "762"},
+ {"u=,g=x,o=r", "763"},
+ {"u=,g=x,o=wx", "764"},
+ {"u=,g=x,o=w", "765"},
+ {"u=,g=x,o=x", "766"},
+ {"u=,g=x,o=", "767"},
+ {"u=,g=,o=rwx", "770"},
+ {"u=,g=,o=rw", "771"},
+ {"u=,g=,o=rx", "772"},
+ {"u=,g=,o=r", "773"},
+ {"u=,g=,o=wx", "774"},
+ {"u=,g=,o=w", "775"},
+ {"u=,g=,o=x", "776"},
+ {"u=,g=,o=", "777"}
+ };
+
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/48] hadoop git commit: HDFS-9847. HDFS configuration should
accept time units. Contributed by Yiqun Lin
Posted by aw...@apache.org.
HDFS-9847. HDFS configuration should accept time units. Contributed by Yiqun Lin
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d37dc5d1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d37dc5d1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d37dc5d1
Branch: refs/heads/HADOOP-13341
Commit: d37dc5d1b8e022a7085118a2e7066623483c293f
Parents: f388a20
Author: Chris Douglas <cd...@apache.org>
Authored: Tue Sep 6 10:37:54 2016 -0700
Committer: Chris Douglas <cd...@apache.org>
Committed: Tue Sep 6 10:38:04 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 47 ++++++++++++++------
.../apache/hadoop/conf/TestConfiguration.java | 46 +++++++++++++++++++
.../hadoop/hdfs/client/impl/DfsClientConf.java | 6 ++-
.../hadoop/hdfs/server/balancer/Balancer.java | 12 +++--
.../server/blockmanagement/BlockManager.java | 6 ++-
.../BlockPlacementPolicyDefault.java | 5 ++-
.../server/blockmanagement/DatanodeManager.java | 9 ++--
.../blockmanagement/DecommissionManager.java | 7 +--
.../hadoop/hdfs/server/datanode/DNConf.java | 18 ++++----
.../hadoop/hdfs/server/datanode/DataNode.java | 4 +-
.../hdfs/server/datanode/DirectoryScanner.java | 6 ++-
.../apache/hadoop/hdfs/server/mover/Mover.java | 12 +++--
.../hdfs/server/namenode/CheckpointConf.java | 10 +++--
.../hadoop/hdfs/server/namenode/FSImage.java | 7 +--
.../hdfs/server/namenode/ha/EditLogTailer.java | 10 +++--
.../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 6 ++-
.../src/main/resources/hdfs-default.xml | 42 ++++++++++++++---
17 files changed, 187 insertions(+), 66 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index 29733eb..66734b6 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -630,10 +630,15 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
ArrayList<String > names = new ArrayList<String>();
if (isDeprecated(name)) {
DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
- warnOnceIfDeprecated(deprecations, name);
- for (String newKey : keyInfo.newKeys) {
- if(newKey != null) {
- names.add(newKey);
+ if (keyInfo != null) {
+ if (!keyInfo.getAndSetAccessed()) {
+ logDeprecation(keyInfo.getWarningMessage(name));
+ }
+
+ for (String newKey : keyInfo.newKeys) {
+ if (newKey != null) {
+ names.add(newKey);
+ }
}
}
}
@@ -1232,11 +1237,9 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
}
- private void warnOnceIfDeprecated(DeprecationContext deprecations, String name) {
- DeprecatedKeyInfo keyInfo = deprecations.getDeprecatedKeyMap().get(name);
- if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
- LOG_DEPRECATION.info(keyInfo.getWarningMessage(name));
- }
+ @VisibleForTesting
+ void logDeprecation(String message) {
+ LOG_DEPRECATION.info(message);
}
/**
@@ -1625,20 +1628,38 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
String vStr = get(name);
if (null == vStr) {
return defaultValue;
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
+ }
+ }
+
+ public long getTimeDuration(String name, String defaultValue, TimeUnit unit) {
+ String vStr = get(name);
+ if (null == vStr) {
+ return getTimeDurationHelper(name, defaultValue, unit);
+ } else {
+ return getTimeDurationHelper(name, vStr, unit);
}
- vStr = vStr.trim();
- return getTimeDurationHelper(name, vStr, unit);
}
private long getTimeDurationHelper(String name, String vStr, TimeUnit unit) {
+ vStr = vStr.trim();
+ vStr = StringUtils.toLowerCase(vStr);
ParsedTimeDuration vUnit = ParsedTimeDuration.unitFor(vStr);
if (null == vUnit) {
- LOG.warn("No unit for " + name + "(" + vStr + ") assuming " + unit);
+ logDeprecation("No unit for " + name + "(" + vStr + ") assuming " + unit);
vUnit = ParsedTimeDuration.unitFor(unit);
} else {
vStr = vStr.substring(0, vStr.lastIndexOf(vUnit.suffix()));
}
- return unit.convert(Long.parseLong(vStr), vUnit.unit());
+
+ long raw = Long.parseLong(vStr);
+ long converted = unit.convert(raw, vUnit.unit());
+ if (vUnit.unit().convert(converted, unit) < raw) {
+ logDeprecation("Possible loss of precision converting " + vStr
+ + vUnit.suffix() + " to " + unit + " for " + name);
+ }
+ return converted;
}
public long[] getTimeDurations(String name, TimeUnit unit) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index aa3b239..e7ebea1 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -880,6 +880,15 @@ public class TestConfiguration extends TestCase {
assertEquals(30L, conf.getTimeDuration("test.time.X", 30, SECONDS));
conf.set("test.time.X", "30");
assertEquals(30L, conf.getTimeDuration("test.time.X", 40, SECONDS));
+ assertEquals(10L, conf.getTimeDuration("test.time.c", "10", SECONDS));
+ assertEquals(30L, conf.getTimeDuration("test.time.c", "30s", SECONDS));
+ assertEquals(120L, conf.getTimeDuration("test.time.c", "2m", SECONDS));
+ conf.set("test.time.c", "30");
+ assertEquals(30L, conf.getTimeDuration("test.time.c", "40s", SECONDS));
+
+ // check suffix insensitive
+ conf.set("test.time.d", "30S");
+ assertEquals(30L, conf.getTimeDuration("test.time.d", 40, SECONDS));
for (Configuration.ParsedTimeDuration ptd :
Configuration.ParsedTimeDuration.values()) {
@@ -889,6 +898,43 @@ public class TestConfiguration extends TestCase {
}
}
+ public void testTimeDurationWarning() {
+ // check warn for possible loss of precision
+ final String warnFormat = "Possible loss of precision converting %s" +
+ " to %s for test.time.warn";
+ final ArrayList<String> warnchk = new ArrayList<>();
+ Configuration wconf = new Configuration(false) {
+ @Override
+ void logDeprecation(String message) {
+ warnchk.add(message);
+ }
+ };
+ String[] convDAYS = new String[]{"23h", "30m", "40s", "10us", "40000ms"};
+ for (String s : convDAYS) {
+ wconf.set("test.time.warn", s);
+ assertEquals(0, wconf.getTimeDuration("test.time.warn", 1, DAYS));
+ }
+ for (int i = 0; i < convDAYS.length; ++i) {
+ String wchk = String.format(warnFormat, convDAYS[i], "DAYS");
+ assertEquals(wchk, warnchk.get(i));
+ }
+
+ warnchk.clear();
+ wconf.setTimeDuration("test.time.warn", 1441, MINUTES);
+ assertEquals(1, wconf.getTimeDuration("test.time.warn", 0, DAYS));
+ assertEquals(24, wconf.getTimeDuration("test.time.warn", 0, HOURS));
+ String dchk = String.format(warnFormat, "1441m", "DAYS");
+ assertEquals(dchk, warnchk.get(0));
+ String hchk = String.format(warnFormat, "1441m", "HOURS");
+ assertEquals(hchk, warnchk.get(1));
+ assertEquals(1441, wconf.getTimeDuration("test.time.warn", 0, MINUTES));
+ // no warning
+ assertEquals(2, warnchk.size());
+ assertEquals(86460, wconf.getTimeDuration("test.time.warn", 0, SECONDS));
+ // no warning
+ assertEquals(2, warnchk.size());
+ }
+
public void testPattern() throws IOException {
out = new BufferedWriter(new FileWriter(CONFIG));
startConfig();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
index ded989c..4f4c7b2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/impl/DfsClientConf.java
@@ -85,6 +85,7 @@ import static org.apache.hadoop.hdfs.client.HdfsClientConfigKeys.Write;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.TimeUnit;
/**
* DFSClient configuration.
@@ -233,9 +234,10 @@ public class DfsClientConf {
connectToDnViaHostname = conf.getBoolean(DFS_CLIENT_USE_DN_HOSTNAME,
DFS_CLIENT_USE_DN_HOSTNAME_DEFAULT);
- datanodeRestartTimeout = conf.getLong(
+ datanodeRestartTimeout = conf.getTimeDuration(
DFS_CLIENT_DATANODE_RESTART_TIMEOUT_KEY,
- DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT) * 1000;
+ DFS_CLIENT_DATANODE_RESTART_TIMEOUT_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
slowIoWarningThresholdMs = conf.getLong(
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_KEY,
DFS_CLIENT_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
index 8d94ed5..2037d01 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Balancer.java
@@ -33,6 +33,7 @@ import java.util.Iterator;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -662,10 +663,13 @@ public class Balancer {
static int run(Collection<URI> namenodes, final BalancerParameters p,
Configuration conf) throws IOException, InterruptedException {
final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 2000 +
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
LOG.info("namenodes = " + namenodes);
LOG.info("parameters = " + p);
LOG.info("included nodes = " + p.getIncludedNodes());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 9e3b935..1362c0b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -42,6 +42,7 @@ import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.FutureTask;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicLong;
import javax.management.ObjectName;
@@ -405,8 +406,9 @@ public class BlockManager implements BlockStatsMXBean {
this.blocksReplWorkMultiplier = DFSUtil.getReplWorkMultiplier(conf);
this.replicationRecheckInterval =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000L;
+ conf.getTimeDuration(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000L;
this.storageInfoDefragmentInterval =
conf.getLong(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 19f4dde..abfa782 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.blockmanagement;
import static org.apache.hadoop.util.Time.monotonicNow;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Preconditions;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -89,9 +90,9 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
this.stats = stats;
this.clusterMap = clusterMap;
this.host2datanodeMap = host2datanodeMap;
- this.heartbeatInterval = conf.getLong(
+ this.heartbeatInterval = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000;
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000;
this.tolerateHeartbeatMultiplier = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_KEY,
DFSConfigKeys.DFS_NAMENODE_TOLERATE_HEARTBEAT_MULTIPLIER_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index fffe29c..2d6547f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -58,6 +58,7 @@ import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.*;
import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
/**
* Manage datanodes, include decommission and other activities.
@@ -234,9 +235,9 @@ public class DatanodeManager {
dnsToSwitchMapping.resolve(locations);
}
- heartbeatIntervalSeconds = conf.getLong(
+ heartbeatIntervalSeconds = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
heartbeatRecheckInterval = conf.getInt(
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY,
DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_DEFAULT); // 5 minutes
@@ -289,9 +290,9 @@ public class DatanodeManager {
" = '" + staleInterval + "' is invalid. " +
"It should be a positive non-zero value.");
- final long heartbeatIntervalSeconds = conf.getLong(
+ final long heartbeatIntervalSeconds = conf.getTimeDuration(
DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT);
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS);
// The stale interval value cannot be smaller than
// 3 times of heartbeat interval
final long minStaleInterval = conf.getInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
index c456aba..6436fab 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DecommissionManager.java
@@ -134,9 +134,10 @@ public class DecommissionManager {
* @param conf
*/
void activate(Configuration conf) {
- final int intervalSecs =
- conf.getInt(DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT);
+ final int intervalSecs = (int) conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
checkArgument(intervalSecs >= 0, "Cannot set a negative " +
"value for " + DFSConfigKeys.DFS_NAMENODE_DECOMMISSION_INTERVAL_KEY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
index 11197e6..757d7f5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DNConf.java
@@ -64,6 +64,8 @@ import org.apache.hadoop.hdfs.protocol.datatransfer.TrustedChannelResolver;
import org.apache.hadoop.hdfs.protocol.datatransfer.sasl.DataTransferSaslUtil;
import org.apache.hadoop.security.SaslPropertiesResolver;
+import java.util.concurrent.TimeUnit;
+
/**
* Simple class encapsulating all of the configuration that the DataNode
* loads at startup time.
@@ -184,9 +186,9 @@ public class DNConf {
DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_KEY,
DFSConfigKeys.DFS_DATANODE_SLOW_IO_WARNING_THRESHOLD_DEFAULT);
- long initBRDelay = conf.getLong(
+ long initBRDelay = conf.getTimeDuration(
DFS_BLOCKREPORT_INITIAL_DELAY_KEY,
- DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT) * 1000L;
+ DFS_BLOCKREPORT_INITIAL_DELAY_DEFAULT, TimeUnit.SECONDS) * 1000L;
if (initBRDelay >= blockReportInterval) {
initBRDelay = 0;
DataNode.LOG.info("dfs.blockreport.initialDelay is "
@@ -195,12 +197,12 @@ public class DNConf {
}
initialBlockReportDelayMs = initBRDelay;
- heartBeatInterval = conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT) * 1000L;
+ heartBeatInterval = conf.getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS) * 1000L;
long confLifelineIntervalMs =
conf.getLong(DFS_DATANODE_LIFELINE_INTERVAL_SECONDS_KEY,
- 3 * conf.getLong(DFS_HEARTBEAT_INTERVAL_KEY,
- DFS_HEARTBEAT_INTERVAL_DEFAULT)) * 1000L;
+ 3 * conf.getTimeDuration(DFS_HEARTBEAT_INTERVAL_KEY,
+ DFS_HEARTBEAT_INTERVAL_DEFAULT, TimeUnit.SECONDS)) * 1000L;
if (confLifelineIntervalMs <= heartBeatInterval) {
confLifelineIntervalMs = 3 * heartBeatInterval;
DataNode.LOG.warn(
@@ -245,9 +247,9 @@ public class DNConf {
DFS_DATANODE_NON_LOCAL_LAZY_PERSIST,
DFS_DATANODE_NON_LOCAL_LAZY_PERSIST_DEFAULT);
- this.bpReadyTimeout = conf.getLong(
+ this.bpReadyTimeout = conf.getTimeDuration(
DFS_DATANODE_BP_READY_TIMEOUT_KEY,
- DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT);
+ DFS_DATANODE_BP_READY_TIMEOUT_DEFAULT, TimeUnit.SECONDS);
this.volFailuresTolerated =
conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 08a1fc0..0025040 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -1024,8 +1024,8 @@ public class DataNode extends ReconfigurableBase
return;
}
String reason = null;
- if (conf.getInt(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
- DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT) < 0) {
+ if (conf.getTimeDuration(DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
+ DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT, TimeUnit.SECONDS) < 0) {
reason = "verification is turned off by configuration";
} else if ("SimulatedFSDataset".equals(data.getClass().getSimpleName())) {
reason = "verifcation is not supported by SimulatedFSDataset";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
index 7c8985e..f9ebab9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DirectoryScanner.java
@@ -414,8 +414,10 @@ public class DirectoryScanner implements Runnable {
DirectoryScanner(DataNode datanode, FsDatasetSpi<?> dataset, Configuration conf) {
this.datanode = datanode;
this.dataset = dataset;
- int interval = conf.getInt(DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
- DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT);
+ int interval = (int) conf.getTimeDuration(
+ DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_KEY,
+ DFSConfigKeys.DFS_DATANODE_DIRECTORYSCAN_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS);
scanPeriodMsecs = interval * MILLIS_PER_SECOND; //msec
int throttle =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
index fdb6cfa..0c85374 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java
@@ -64,6 +64,7 @@ import java.net.InetSocketAddress;
import java.net.URI;
import java.text.DateFormat;
import java.util.*;
+import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
@InterfaceAudience.Private
@@ -602,10 +603,13 @@ public class Mover {
static int run(Map<URI, List<Path>> namenodes, Configuration conf)
throws IOException, InterruptedException {
final long sleeptime =
- conf.getLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
- DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT) * 2000 +
- conf.getLong(DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
- DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT) * 1000;
+ conf.getTimeDuration(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY,
+ DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 2000 +
+ conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_KEY,
+ DFSConfigKeys.DFS_NAMENODE_REPLICATION_INTERVAL_DEFAULT,
+ TimeUnit.SECONDS) * 1000;
AtomicInteger retryCount = new AtomicInteger(0);
LOG.info("namenodes = " + namenodes);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
index c30730b..aeee87d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/CheckpointConf.java
@@ -26,6 +26,8 @@ import org.apache.hadoop.conf.Configuration;
import com.google.common.collect.ImmutableList;
+import java.util.concurrent.TimeUnit;
+
@InterfaceAudience.Private
public class CheckpointConf {
private static final Log LOG = LogFactory.getLog(CheckpointConf.class);
@@ -52,12 +54,12 @@ public class CheckpointConf {
private double quietMultiplier;
public CheckpointConf(Configuration conf) {
- checkpointCheckPeriod = conf.getLong(
+ checkpointCheckPeriod = conf.getTimeDuration(
DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_KEY,
- DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT);
+ DFS_NAMENODE_CHECKPOINT_CHECK_PERIOD_DEFAULT, TimeUnit.SECONDS);
- checkpointPeriod = conf.getLong(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ checkpointPeriod = conf.getTimeDuration(DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
+ DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
checkpointTxnCount = conf.getLong(DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
maxRetriesOnMergeError = conf.getInt(DFS_NAMENODE_CHECKPOINT_MAX_RETRIES_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
index b637105..d02a6ca 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImage.java
@@ -34,6 +34,7 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.TimeUnit;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -798,9 +799,9 @@ public class FSImage implements Closeable {
*/
private boolean needsResaveBasedOnStaleCheckpoint(
File imageFile, long numEditsLoaded) {
- final long checkpointPeriod = conf.getLong(
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ final long checkpointPeriod = conf.getTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT, TimeUnit.SECONDS);
final long checkpointTxnCount = conf.getLong(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
index 95c3d58..f57cb4b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
@@ -146,8 +146,9 @@ public class EditLogTailer {
lastLoadTimeMs = monotonicNow();
- logRollPeriodMs = conf.getInt(DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT) * 1000;
+ logRollPeriodMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY,
+ DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
List<RemoteNameNodeInfo> nns = Collections.emptyList();
if (logRollPeriodMs >= 0) {
try {
@@ -172,8 +173,9 @@ public class EditLogTailer {
DFSConfigKeys.DFS_HA_LOGROLL_PERIOD_KEY + " is negative.");
}
- sleepTimeMs = conf.getInt(DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
- DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT) * 1000;
+ sleepTimeMs = conf.getTimeDuration(
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_KEY,
+ DFSConfigKeys.DFS_HA_TAILEDITS_PERIOD_DEFAULT, TimeUnit.SECONDS) * 1000;
rollEditsTimeoutMs = conf.getInt(
DFSConfigKeys.DFS_HA_TAILEDITS_ROLLEDITS_TIMEOUT_KEY,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
index ba09e8b..e6e0fbb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSAdmin.java
@@ -34,6 +34,7 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
+import java.util.concurrent.TimeUnit;
import com.google.common.base.Joiner;
import com.google.common.base.Optional;
@@ -742,9 +743,10 @@ public class DFSAdmin extends FsShell {
long timeWindow = 0;
long txGap = 0;
if (argv.length > 1 && "-beforeShutdown".equals(argv[1])) {
- final long checkpointPeriod = dfsConf.getLong(
+ final long checkpointPeriod = dfsConf.getTimeDuration(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_KEY,
- DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT);
+ DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_PERIOD_DEFAULT,
+ TimeUnit.SECONDS);
final long checkpointTxnCount = dfsConf.getLong(
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_KEY,
DFSConfigKeys.DFS_NAMENODE_CHECKPOINT_TXNS_DEFAULT);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d37dc5d1/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 0c2c743..4eca80c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -637,8 +637,12 @@
</property>
<property>
- <name>dfs.blockreport.initialDelay</name> <value>0</value>
- <description>Delay for first block report in seconds.</description>
+ <name>dfs.blockreport.initialDelay</name>
+ <value>0</value>
+ <description>
+ Delay for first block report in seconds. Support multiple time unit
+ suffix(case insensitive), as described in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -681,6 +685,8 @@
<value>21600</value>
<description>Interval in seconds for Datanode to scan data directories and
reconcile the difference between blocks in memory and on the disk.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -715,7 +721,13 @@
<property>
<name>dfs.heartbeat.interval</name>
<value>3</value>
- <description>Determines datanode heartbeat interval in seconds.</description>
+ <description>
+ Determines datanode heartbeat interval in seconds.
+ Can use the following suffix (case insensitive):
+ ms(millis), s(sec), m(min), h(hour), d(day)
+ to specify the time (such as 2s, 2m, 1h, etc.).
+ Or provide complete number in seconds (such as 30 for 30 seconds).
+ </description>
</property>
<property>
@@ -920,7 +932,9 @@
<name>dfs.namenode.decommission.interval</name>
<value>30</value>
<description>Namenode periodicity in seconds to check if decommission is
- complete.</description>
+ complete. Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -949,7 +963,9 @@
<name>dfs.namenode.replication.interval</name>
<value>3</value>
<description>The periodicity in seconds with which the namenode computes
- replication work for datanodes. </description>
+ replication work for datanodes. Support multiple time unit suffix(case insensitive),
+ as described in dfs.heartbeat.interval.
+ </description>
</property>
<property>
@@ -1044,7 +1060,10 @@
<property>
<name>dfs.namenode.checkpoint.period</name>
<value>3600</value>
- <description>The number of seconds between two periodic checkpoints.
+ <description>
+ The number of seconds between two periodic checkpoints.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1062,7 +1081,8 @@
<value>60</value>
<description>The SecondaryNameNode and CheckpointNode will poll the NameNode
every 'dfs.namenode.checkpoint.check.period' seconds to query the number
- of uncheckpointed transactions.
+ of uncheckpointed transactions. Support multiple time unit suffix(case insensitive),
+ as described in dfs.heartbeat.interval.
</description>
</property>
@@ -1408,6 +1428,8 @@
the datanode dead and invoking the normal recovery mechanisms.
The notification is sent by a datanode when it is being shutdown
using the shutdownDatanode admin command with the upgrade option.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1475,6 +1497,8 @@
log segments, the StandbyNode will only be as up-to-date as how
often the logs are rolled. Note that failover triggers a log roll
so the StandbyNode will be up to date before it becomes active.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -1484,6 +1508,8 @@
<description>
How often, in seconds, the StandbyNode should check for new
finalized log segments in the shared edits log.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
@@ -2918,6 +2944,8 @@
received request. Setting this to 0 fails requests right away if the
datanode is not yet registered with the namenode. This wait time
reduces initial request failures after datanode restart.
+ Support multiple time unit suffix(case insensitive), as described
+ in dfs.heartbeat.interval.
</description>
</property>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/48] hadoop git commit: HDFS-10817. Add Logging for Long-held NN
Read Locks. Contributed by Erik Krogen.
Posted by aw...@apache.org.
HDFS-10817. Add Logging for Long-held NN Read Locks. Contributed by Erik Krogen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f4b0d33
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f4b0d33
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f4b0d33
Branch: refs/heads/HADOOP-13341
Commit: 6f4b0d33ca339e3724623a1d23c101f8cfd3cdd5
Parents: 85bab5f
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Aug 31 15:40:01 2016 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Aug 31 15:40:09 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 32 +++++
.../src/main/resources/hdfs-default.xml | 9 ++
.../hdfs/server/namenode/TestFSNamesystem.java | 144 ++++++++++++++++++-
4 files changed, 188 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f4b0d33/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index 2eff3b0..f2ab321 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -407,10 +407,14 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final long
DFS_NAMENODE_MAX_LOCK_HOLD_TO_RELEASE_LEASE_MS_DEFAULT = 25;
- // Threshold for how long a write lock must be held for the event to be logged
+ // Threshold for how long namenode locks must be held for the
+ // event to be logged
public static final String DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY =
"dfs.namenode.write-lock-reporting-threshold-ms";
public static final long DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 1000L;
+ public static final String DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY =
+ "dfs.namenode.read-lock-reporting-threshold-ms";
+ public static final long DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT = 5000L;
public static final String DFS_UPGRADE_DOMAIN_FACTOR = "dfs.namenode.upgrade.domain.factor";
public static final int DFS_UPGRADE_DOMAIN_FACTOR_DEFAULT = DFS_REPLICATION_DEFAULT;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f4b0d33/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index f4b742e..3b14eec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -73,6 +73,8 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_I
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RESOURCE_CHECK_INTERVAL_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_EXPIRYTIME_MILLIS_KEY;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_RETRY_CACHE_HEAP_PERCENT_DEFAULT;
@@ -824,6 +826,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
this.writeLockReportingThreshold = conf.getLong(
DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY,
DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_DEFAULT);
+ this.readLockReportingThreshold = conf.getLong(
+ DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY,
+ DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_DEFAULT);
// For testing purposes, allow the DT secret manager to be started regardless
// of whether security is enabled.
@@ -1507,14 +1512,41 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
private long writeLockReportingThreshold;
/** Last time stamp for write lock. Keep the longest one for multi-entrance.*/
private long writeLockHeldTimeStamp;
+ /** Threshold (ms) for long holding read lock report. */
+ private long readLockReportingThreshold;
+ /**
+ * Last time stamp for read lock. Keep the longest one for
+ * multi-entrance. This is ThreadLocal since there could be
+ * many read locks held simultaneously.
+ */
+ private static ThreadLocal<Long> readLockHeldTimeStamp =
+ new ThreadLocal<Long>() {
+ @Override
+ public Long initialValue() {
+ return Long.MAX_VALUE;
+ }
+ };
@Override
public void readLock() {
this.fsLock.readLock().lock();
+ if (this.fsLock.getReadHoldCount() == 1) {
+ readLockHeldTimeStamp.set(monotonicNow());
+ }
}
@Override
public void readUnlock() {
+ final boolean needReport = this.fsLock.getReadHoldCount() == 1;
+ final long readLockInterval = monotonicNow() - readLockHeldTimeStamp.get();
this.fsLock.readLock().unlock();
+
+ if (needReport) {
+ readLockHeldTimeStamp.remove();
+ if (readLockInterval > this.readLockReportingThreshold) {
+ LOG.info("FSNamesystem read lock held for " + readLockInterval +
+ " ms via\n" + StringUtils.getStackTrace(Thread.currentThread()));
+ }
+ }
}
@Override
public void writeLock() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f4b0d33/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index b9a0812..0c2c743 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2624,6 +2624,15 @@
</property>
<property>
+ <name>dfs.namenode.read-lock-reporting-threshold-ms</name>
+ <value>5000</value>
+ <description>When a read lock is held on the namenode for a long time,
+ this will be logged as the lock is released. This sets how long the
+ lock must be held for logging to occur.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.startup.delay.block.deletion.sec</name>
<value>0</value>
<description>The delay in seconds at which we will pause the blocks deletion
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f4b0d33/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index df9001d..ccc8a6e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -28,6 +28,7 @@ import java.io.File;
import java.io.IOException;
import java.net.InetAddress;
import java.net.URI;
+import java.util.ArrayList;
import java.util.Collection;
import com.google.common.base.Supplier;
@@ -58,6 +59,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeoutException;
+import java.util.regex.Pattern;
public class TestFSNamesystem {
@@ -286,10 +288,11 @@ public class TestFSNamesystem {
}
/**
- * Test when FSNamesystem lock is held for a long time, logger will report it.
+ * Test when FSNamesystem write lock is held for a long time,
+ * logger will report it.
*/
@Test(timeout=45000)
- public void testFSLockLongHoldingReport() throws Exception {
+ public void testFSWriteLockLongHoldingReport() throws Exception {
final long writeLockReportingThreshold = 100L;
Configuration conf = new Configuration();
conf.setLong(DFSConfigKeys.DFS_NAMENODE_WRITE_LOCK_REPORTING_THRESHOLD_MS_KEY,
@@ -341,6 +344,143 @@ public class TestFSNamesystem {
assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName()));
}
+ /**
+ * Test when FSNamesystem read lock is held for a long time,
+ * logger will report it.
+ */
+ @Test(timeout=45000)
+ public void testFSReadLockLongHoldingReport() throws Exception {
+ final long readLockReportingThreshold = 100L;
+ final String readLockLogStmt = "FSNamesystem read lock held for ";
+ Configuration conf = new Configuration();
+ conf.setLong(
+ DFSConfigKeys.DFS_NAMENODE_READ_LOCK_REPORTING_THRESHOLD_MS_KEY,
+ readLockReportingThreshold);
+ FSImage fsImage = Mockito.mock(FSImage.class);
+ FSEditLog fsEditLog = Mockito.mock(FSEditLog.class);
+ Mockito.when(fsImage.getEditLog()).thenReturn(fsEditLog);
+ FSNamesystem fsn = new FSNamesystem(conf, fsImage);
+
+ LogCapturer logs = LogCapturer.captureLogs(FSNamesystem.LOG);
+ GenericTestUtils.setLogLevel(FSNamesystem.LOG, Level.INFO);
+
+ // Don't report if the read lock is held for a short time
+ fsn.readLock();
+ Thread.sleep(readLockReportingThreshold / 2);
+ fsn.readUnlock();
+ assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) &&
+ logs.getOutput().contains(readLockLogStmt));
+
+ // Report if the read lock is held for a long time
+ fsn.readLock();
+ Thread.sleep(readLockReportingThreshold + 10);
+ logs.clearOutput();
+ fsn.readUnlock();
+ assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName())
+ && logs.getOutput().contains(readLockLogStmt));
+
+ // Report if it's held for a long time when re-entering read lock
+ fsn.readLock();
+ Thread.sleep(readLockReportingThreshold / 2 + 1);
+ fsn.readLock();
+ Thread.sleep(readLockReportingThreshold / 2 + 1);
+ logs.clearOutput();
+ fsn.readUnlock();
+ assertFalse(logs.getOutput().contains(GenericTestUtils.getMethodName()) ||
+ logs.getOutput().contains(readLockLogStmt));
+ logs.clearOutput();
+ fsn.readUnlock();
+ assertTrue(logs.getOutput().contains(GenericTestUtils.getMethodName()) &&
+ logs.getOutput().contains(readLockLogStmt));
+
+ // Report if it's held for a long time while another thread also has the
+ // read lock. Let one thread hold the lock long enough to activate an
+ // alert, then have another thread grab the read lock to ensure that this
+ // doesn't reset the timing.
+ logs.clearOutput();
+ CountDownLatch barrier = new CountDownLatch(1);
+ CountDownLatch barrier2 = new CountDownLatch(1);
+ Thread t1 = new Thread() {
+ @Override
+ public void run() {
+ try {
+ fsn.readLock();
+ Thread.sleep(readLockReportingThreshold + 1);
+ barrier.countDown(); // Allow for t2 to acquire the read lock
+ barrier2.await(); // Wait until t2 has the read lock
+ fsn.readUnlock();
+ } catch (InterruptedException e) {
+ fail("Interrupted during testing");
+ }
+ }
+ };
+ Thread t2 = new Thread() {
+ @Override
+ public void run() {
+ try {
+ barrier.await(); // Wait until t1 finishes sleeping
+ fsn.readLock();
+ barrier2.countDown(); // Allow for t1 to unlock
+ fsn.readUnlock();
+ } catch (InterruptedException e) {
+ fail("Interrupted during testing");
+ }
+ }
+ };
+ t1.start();
+ t2.start();
+ t1.join();
+ t2.join();
+ Pattern t1Pattern = Pattern.compile(
+ String.format("\\Q%s\\E.+%s", t1.getName(), readLockLogStmt));
+ assertTrue(t1Pattern.matcher(logs.getOutput()).find());
+ Pattern t2Pattern = Pattern.compile(
+ String.format("\\Q%s\\E.+%s", t2.getName(), readLockLogStmt));
+ assertFalse(t2Pattern.matcher(logs.getOutput()).find());
+
+ // Spin up a bunch of threads all grabbing the lock at once; assign some
+ // to go over threshold and some under. Check that they all log correctly.
+ logs.clearOutput();
+ final int threadCount = 50;
+ List<Thread> threads = new ArrayList<>(threadCount);
+ for (int i = 0; i < threadCount; i++) {
+ threads.add(new Thread() {
+ @Override
+ public void run() {
+ try {
+ long sleepTime;
+ if (this.getName().hashCode() % 2 == 0) {
+ sleepTime = readLockReportingThreshold + 10;
+ } else {
+ sleepTime = readLockReportingThreshold / 2;
+ }
+ fsn.readLock();
+ Thread.sleep(sleepTime);
+ fsn.readUnlock();
+ } catch (InterruptedException e) {
+ fail("Interrupted during testing");
+ }
+ }
+ });
+ }
+ for (Thread t : threads) {
+ t.start();
+ }
+ for (Thread t : threads) {
+ t.join();
+ }
+ for (Thread t : threads) {
+ Pattern p = Pattern.compile(
+ String.format("\\Q%s\\E.+%s", t.getName(), readLockLogStmt));
+ boolean foundLog = p.matcher(logs.getOutput()).find();
+ if (t.getName().hashCode() % 2 == 0) {
+ assertTrue(foundLog);
+ } else {
+ assertFalse(foundLog);
+ }
+ }
+ }
+
@Test
public void testSafemodeReplicationConf() throws IOException {
Configuration conf = new Configuration();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/48] hadoop git commit: HADOOP-13375.
o.a.h.security.TestGroupsCaching.testBackgroundRefreshCounters seems flaky.
(Contributed by Weiwei Yang)
Posted by aw...@apache.org.
HADOOP-13375. o.a.h.security.TestGroupsCaching.testBackgroundRefreshCounters seems flaky. (Contributed by Weiwei Yang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dcd21d08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dcd21d08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dcd21d08
Branch: refs/heads/HADOOP-13341
Commit: dcd21d083ab2a66fc3ca3bfda03887461698b7b1
Parents: 08f55cc
Author: Mingliang Liu <li...@apache.org>
Authored: Thu Sep 1 11:03:06 2016 -0700
Committer: Mingliang Liu <li...@apache.org>
Committed: Thu Sep 1 11:03:06 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/security/Groups.java | 33 +++----
.../hadoop/security/TestGroupsCaching.java | 91 +++++++++++++++++---
2 files changed, 95 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd21d08/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
index 4891907..32660c2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/Groups.java
@@ -43,6 +43,8 @@ import com.google.common.cache.CacheBuilder;
import com.google.common.cache.Cache;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
+import com.google.common.util.concurrent.FutureCallback;
+import com.google.common.util.concurrent.Futures;
import com.google.common.util.concurrent.ListenableFuture;
import com.google.common.util.concurrent.ListeningExecutorService;
import com.google.common.util.concurrent.MoreExecutors;
@@ -355,23 +357,24 @@ public class Groups {
executorService.submit(new Callable<List<String>>() {
@Override
public List<String> call() throws Exception {
- boolean success = false;
- try {
- backgroundRefreshQueued.decrementAndGet();
- backgroundRefreshRunning.incrementAndGet();
- List<String> results = load(key);
- success = true;
- return results;
- } finally {
- backgroundRefreshRunning.decrementAndGet();
- if (success) {
- backgroundRefreshSuccess.incrementAndGet();
- } else {
- backgroundRefreshException.incrementAndGet();
- }
- }
+ backgroundRefreshQueued.decrementAndGet();
+ backgroundRefreshRunning.incrementAndGet();
+ List<String> results = load(key);
+ return results;
}
});
+ Futures.addCallback(listenableFuture, new FutureCallback<List<String>>() {
+ @Override
+ public void onSuccess(List<String> result) {
+ backgroundRefreshSuccess.incrementAndGet();
+ backgroundRefreshRunning.decrementAndGet();
+ }
+ @Override
+ public void onFailure(Throwable t) {
+ backgroundRefreshException.incrementAndGet();
+ backgroundRefreshRunning.decrementAndGet();
+ }
+ });
return listenableFuture;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dcd21d08/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
index 5a5596e..58c2d1a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestGroupsCaching.java
@@ -25,11 +25,16 @@ import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeoutException;
import org.apache.hadoop.test.GenericTestUtils;
import org.apache.hadoop.util.FakeTimer;
import org.junit.Before;
import org.junit.Test;
+
+import com.google.common.base.Supplier;
+
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.assertFalse;
@@ -67,6 +72,7 @@ public class TestGroupsCaching {
private static int requestCount = 0;
private static long getGroupsDelayMs = 0;
private static boolean throwException;
+ private static volatile CountDownLatch latch = null;
@Override
public List<String> getGroups(String user) throws IOException {
@@ -85,7 +91,19 @@ public class TestGroupsCaching {
return new LinkedList<String>(allGroups);
}
+ /**
+ * Delay returning on a latch or a specific amount of time.
+ */
private void delayIfNecessary() {
+ // cause current method to pause
+ // resume until get notified
+ if (latch != null) {
+ try {
+ latch.await();
+ return;
+ } catch (InterruptedException e) {}
+ }
+
if (getGroupsDelayMs > 0) {
try {
Thread.sleep(getGroupsDelayMs);
@@ -114,6 +132,7 @@ public class TestGroupsCaching {
requestCount = 0;
getGroupsDelayMs = 0;
throwException = false;
+ latch = null;
}
@Override
@@ -142,6 +161,31 @@ public class TestGroupsCaching {
public static void setThrowException(boolean throwIfTrue) {
throwException = throwIfTrue;
}
+
+ /**
+ * Hold on returning the group names unless being notified,
+ * ensure this method is called before {@link #getGroups(String)}.
+ * Call {@link #resume()} will resume the process.
+ */
+ public static void pause() {
+ // Set a static latch, multiple background refresh threads
+ // share this instance. So when await is called, all the
+ // threads will pause until the it decreases the count of
+ // the latch.
+ latch = new CountDownLatch(1);
+ }
+
+ /**
+ * Resume the background refresh thread and return the value
+ * of group names.
+ */
+ public static void resume() {
+ // if latch is null, it means pause was not called and it is
+ // safe to ignore.
+ if (latch != null) {
+ latch.countDown();
+ }
+ }
}
public static class ExceptionalGroupMapping extends ShellBasedUnixGroupsMapping {
@@ -610,22 +654,18 @@ public class TestGroupsCaching {
// expire the cache
timer.advance(2*1000);
- FakeGroupMapping.setGetGroupsDelayMs(40);
+ FakeGroupMapping.pause();
// Request all groups again, as there are 2 threads to process them
// 3 should get queued and 2 should be running
for (String g: grps) {
groups.getGroups(g);
}
- Thread.sleep(20);
- assertEquals(groups.getBackgroundRefreshQueued(), 3);
- assertEquals(groups.getBackgroundRefreshRunning(), 2);
+ waitForGroupCounters(groups, 3, 2, 0, 0);
+ FakeGroupMapping.resume();
- // After 120ms all should have completed running
- Thread.sleep(120);
- assertEquals(groups.getBackgroundRefreshQueued(), 0);
- assertEquals(groups.getBackgroundRefreshRunning(), 0);
- assertEquals(groups.getBackgroundRefreshSuccess(), 5);
+ // Once resumed, all results should be returned immediately
+ waitForGroupCounters(groups, 0, 0, 5, 0);
// Now run again, this time throwing exceptions but no delay
timer.advance(2*1000);
@@ -634,11 +674,34 @@ public class TestGroupsCaching {
for (String g: grps) {
groups.getGroups(g);
}
- Thread.sleep(20);
- assertEquals(groups.getBackgroundRefreshQueued(), 0);
- assertEquals(groups.getBackgroundRefreshRunning(), 0);
- assertEquals(groups.getBackgroundRefreshSuccess(), 5);
- assertEquals(groups.getBackgroundRefreshException(), 5);
+ waitForGroupCounters(groups, 0, 0, 5, 5);
+ }
+
+ private void waitForGroupCounters(Groups groups, long expectedQueued,
+ long expectedRunning, long expectedSuccess, long expectedExpection)
+ throws InterruptedException {
+ long[] expected = {expectedQueued, expectedRunning,
+ expectedSuccess, expectedExpection};
+ long[] actual = new long[expected.length];
+ // wait for a certain time until the counters reach
+ // to expected values. Check values in 20 ms interval.
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ actual[0] = groups.getBackgroundRefreshQueued();
+ actual[1] = groups.getBackgroundRefreshRunning();
+ actual[2] = groups.getBackgroundRefreshSuccess();
+ actual[3] = groups.getBackgroundRefreshException();
+ return Arrays.equals(actual, expected);
+ }
+ }, 20, 1000);
+ } catch (TimeoutException e) {
+ fail("Excepted group counter values are not reached in given time,"
+ + " expecting (Queued, Running, Success, Exception) : "
+ + Arrays.toString(expected) + " but actual : "
+ + Arrays.toString(actual));
+ }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/48] hadoop git commit: HDFS-10841. Remove duplicate or unused
variable in appendFile(). Contributed by Kihwal Lee.
Posted by aw...@apache.org.
HDFS-10841. Remove duplicate or unused variable in appendFile(). Contributed by Kihwal Lee.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f6c0b754
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f6c0b754
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f6c0b754
Branch: refs/heads/HADOOP-13341
Commit: f6c0b7543f612de756ff0c03e9a2c6e33b496a36
Parents: 7c213c3
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Sep 6 11:48:35 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Sep 6 11:48:35 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f6c0b754/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
index 5192352..6f898ef 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirAppendOp.java
@@ -84,12 +84,10 @@ final class FSDirAppendOp {
final LocatedBlock lb;
final FSDirectory fsd = fsn.getFSDirectory();
- final String src;
final INodesInPath iip;
fsd.writeLock();
try {
iip = fsd.resolvePathForWrite(pc, srcArg);
- src = iip.getPath();
// Verify that the destination does not exist as a directory already
final INode inode = iip.getLastINode();
final String path = iip.getPath();
@@ -111,7 +109,7 @@ final class FSDirAppendOp {
// not support appending file with striped blocks
if (file.isStriped()) {
throw new UnsupportedOperationException(
- "Cannot append to files with striped block " + src);
+ "Cannot append to files with striped block " + path);
}
BlockManager blockManager = fsd.getBlockManager();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/48] hadoop git commit: HADOOP-13549. Eliminate intermediate
buffer for server-side PB encoding. Contributed by Daryn Sharp.
Posted by aw...@apache.org.
HADOOP-13549. Eliminate intermediate buffer for server-side PB encoding. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/39d1b1d7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/39d1b1d7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/39d1b1d7
Branch: refs/heads/HADOOP-13341
Commit: 39d1b1d747b1e325792b897b3264272f32b756a9
Parents: e6fcfe2
Author: Kihwal Lee <ki...@apache.org>
Authored: Tue Sep 6 10:01:01 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Tue Sep 6 10:02:42 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 52 ++++++++++++++++++--
1 file changed, 49 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/39d1b1d7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index ea4f533..531d574 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -119,6 +119,7 @@ import org.apache.htrace.core.Tracer;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
+import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
/** An abstract IPC service. IPC calls take a single {@link Writable} as a
@@ -2752,24 +2753,69 @@ public abstract class Server {
private void setupResponse(RpcCall call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
+ final byte[] response;
+ if (rv == null || (rv instanceof RpcWritable.ProtobufWrapper)) {
+ response = setupResponseForProtobuf(header, rv);
+ } else {
+ response = setupResponseForWritable(header, rv);
+ }
+ if (response.length > maxRespSize) {
+ LOG.warn("Large response size " + response.length + " for call "
+ + call.toString());
+ }
+ call.setResponse(ByteBuffer.wrap(response));
+ }
+
+ private byte[] setupResponseForWritable(
+ RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
RpcWritable.wrap(header).writeTo(buf);
if (rv != null) {
RpcWritable.wrap(rv).writeTo(buf);
}
- call.setResponse(ByteBuffer.wrap(buf.toByteArray()));
+ return buf.toByteArray();
} finally {
// Discard a large buf and reset it back to smaller size
// to free up heap.
if (buf.capacity() > maxRespSize) {
- LOG.warn("Large response size " + buf.size() + " for call "
- + call.toString());
buf.setCapacity(INITIAL_RESP_BUF_SIZE);
}
}
}
+
+ // writing to a pre-allocated array is the most efficient way to construct
+ // a protobuf response.
+ private byte[] setupResponseForProtobuf(
+ RpcResponseHeaderProto header, Writable rv) throws IOException {
+ Message payload = (rv != null)
+ ? ((RpcWritable.ProtobufWrapper)rv).getMessage() : null;
+ int length = getDelimitedLength(header);
+ if (payload != null) {
+ length += getDelimitedLength(payload);
+ }
+ byte[] buf = new byte[length + 4];
+ CodedOutputStream cos = CodedOutputStream.newInstance(buf);
+ // the stream only supports little endian ints
+ cos.writeRawByte((byte)((length >>> 24) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 16) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 8) & 0xFF));
+ cos.writeRawByte((byte)((length >>> 0) & 0xFF));
+ cos.writeRawVarint32(header.getSerializedSize());
+ header.writeTo(cos);
+ if (payload != null) {
+ cos.writeRawVarint32(payload.getSerializedSize());
+ payload.writeTo(cos);
+ }
+ return buf;
+ }
+
+ private static int getDelimitedLength(Message message) {
+ int length = message.getSerializedSize();
+ return length + CodedOutputStream.computeRawVarint32Size(length);
+ }
+
/**
* Setup response for the IPC Call on Fatal Error from a
* client that is using old version of Hadoop.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/48] hadoop git commit: HADOOP-13358. Modify HDFS to use
hadoop_subcommand_opts
Posted by aw...@apache.org.
HADOOP-13358. Modify HDFS to use hadoop_subcommand_opts
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ff987861
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ff987861
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ff987861
Branch: refs/heads/HADOOP-13341
Commit: ff98786144ebc1d21127a13850bb943a76ceac01
Parents: d03c6f8
Author: Allen Wittenauer <aw...@apache.org>
Authored: Sun Aug 28 10:45:00 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../src/main/bin/hadoop-functions.sh | 1 +
.../hadoop-common/src/main/conf/hadoop-env.sh | 28 +++++++++----------
.../hadoop-hdfs/src/main/bin/hdfs | 29 ++++----------------
.../hadoop-hdfs/src/main/bin/hdfs-config.sh | 28 ++++++++++++-------
4 files changed, 38 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff987861/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 695fd41..db868bd 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -2048,6 +2048,7 @@ function hadoop_subcommand_opts
return 0
fi
}
+
## @description Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS.
## @description This *does not* handle the pre-3.x deprecated cases
## @audience public
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff987861/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
index f4493f1..4656f4d 100644
--- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
+++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh
@@ -294,16 +294,16 @@ esac
# and therefore may override any similar flags set in HADOOP_OPTS
#
# a) Set JMX options
-# export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
+# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026"
#
# b) Set garbage collection logs
-# export HADOOP_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
#
# c) ... or set them directly
-# export HADOOP_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
+# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')"
# this is the default:
-# export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
+# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
###
# SecondaryNameNode specific parameters
@@ -313,7 +313,7 @@ esac
# and therefore may override any similar flags set in HADOOP_OPTS
#
# This is the default:
-# export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
+# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS"
###
# DataNode specific parameters
@@ -323,7 +323,7 @@ esac
# and therefore may override any similar flags set in HADOOP_OPTS
#
# This is the default:
-# export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
+# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS"
# On secure datanodes, user to run the datanode as after dropping privileges.
# This **MUST** be uncommented to enable secure HDFS if using privileged ports
@@ -336,7 +336,7 @@ esac
# Supplemental options for secure datanodes
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
-# export HADOOP_DN_SECURE_EXTRA_OPTS="-jvm server"
+# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server"
# Where datanode log files are stored in the secure data environment.
# This will replace the hadoop.log.dir Java property in secure mode.
@@ -352,18 +352,18 @@ esac
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
-# export HADOOP_NFS3_OPTS=""
+# export HDFS_NFS3_OPTS=""
# Specify the JVM options to be used when starting the Hadoop portmapper.
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
-# export HADOOP_PORTMAP_OPTS="-Xmx512m"
+# export HDFS_PORTMAP_OPTS="-Xmx512m"
# Supplemental options for priviliged gateways
# By default, Hadoop uses jsvc which needs to know to launch a
# server jvm.
-# export HADOOP_NFS3_SECURE_EXTRA_OPTS="-jvm server"
+# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server"
# On privileged gateways, user to run the gateway as after dropping privileges
# This will replace the hadoop.id.str Java property in secure mode.
@@ -376,7 +376,7 @@ esac
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
-# export HADOOP_ZKFC_OPTS=""
+# export HDFS_ZKFC_OPTS=""
###
# QuorumJournalNode specific parameters
@@ -385,7 +385,7 @@ esac
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
-# export HADOOP_JOURNALNODE_OPTS=""
+# export HDFS_JOURNALNODE_OPTS=""
###
# HDFS Balancer specific parameters
@@ -394,7 +394,7 @@ esac
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
-# export HADOOP_BALANCER_OPTS=""
+# export HDFS_BALANCER_OPTS=""
###
# HDFS Mover specific parameters
@@ -403,7 +403,7 @@ esac
# These options will be appended to the options specified as HADOOP_OPTS
# and therefore may override any similar flags set in HADOOP_OPTS
#
-# export HADOOP_MOVER_OPTS=""
+# export HDFS_MOVER_OPTS=""
###
# Advanced Users Only!
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff987861/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index 41b3c12..b704b00 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -79,8 +79,6 @@ function hdfscmd_case
balancer)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.balancer.Balancer
- hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}"
;;
cacheadmin)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin
@@ -103,13 +101,8 @@ function hdfscmd_case
HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}"
HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}"
- hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
- hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}"
HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter"
else
- hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode'
fi
;;
@@ -157,8 +150,6 @@ function hdfscmd_case
journalnode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode'
- hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}"
;;
jmxget)
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet
@@ -169,14 +160,10 @@ function hdfscmd_case
mover)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover
- hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}"
;;
namenode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode'
- hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}"
hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
;;
nfs3)
@@ -189,13 +176,8 @@ function hdfscmd_case
HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}"
HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}"
- hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
- hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}"
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter
else
- hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}"
HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3
fi
;;
@@ -211,14 +193,10 @@ function hdfscmd_case
portmap)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap
- hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}"
;;
secondarynamenode)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode'
- hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}"
hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}"
;;
snapshotDiff)
@@ -233,8 +211,6 @@ function hdfscmd_case
zkfc)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DFSZKFailoverController'
- hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}"
;;
*)
HADOOP_CLASSNAME="${subcmd}"
@@ -288,8 +264,13 @@ if [[ ${HADOOP_WORKER_MODE} = true ]]; then
exit $?
fi
+hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+
+ hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
hadoop_verify_secure_prereq
hadoop_setup_secure_service
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ff987861/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
old mode 100644
new mode 100755
index d440210..cba37a4
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh
@@ -26,7 +26,7 @@ function hadoop_subproject_init
export HADOOP_HDFS_ENV_PROCESSED=true
fi
fi
-
+
# at some point in time, someone thought it would be a good idea to
# create separate vars for every subproject. *sigh*
# let's perform some overrides and setup some defaults for bw compat
@@ -42,23 +42,31 @@ function hadoop_subproject_init
hadoop_deprecate_envvar HADOOP_HDFS_NICENESS HADOOP_NICENESS
hadoop_deprecate_envvar HADOOP_HDFS_STOP_TIMEOUT HADOOP_STOP_TIMEOUT
-
+
hadoop_deprecate_envvar HADOOP_HDFS_PID_DIR HADOOP_PID_DIR
hadoop_deprecate_envvar HADOOP_HDFS_ROOT_LOGGER HADOOP_ROOT_LOGGER
hadoop_deprecate_envvar HADOOP_HDFS_IDENT_STRING HADOOP_IDENT_STRING
-
+
+ hadoop_deprecate_envvar HADOOP_DN_SECURE_EXTRA_OPTS HDFS_DATANODE_SECURE_EXTRA_OPTS
+
+ hadoop_deprecate_envvar HADOOP_NFS3_SECURE_EXTRA_OPTS HDFS_NFS3_SECURE_EXTRA_OPTS
+
+
HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME}"
-
+
# turn on the defaults
export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-INFO,NullAppender}
- export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
- export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
- export HADOOP_DATANODE_OPTS=${HADOOP_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"}
- export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"}
- export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"}
- export HADOOP_PORTMAP_OPTS=${HADOOP_PORTMAP_OPTS:-"-Xmx512m"}
+ export HDFS_NAMENODE_OPTS=${HDFS_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
+ export HDFS_SECONDARYNAMENODE_OPTS=${HDFS_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"}
+ export HDFS_DATANODE_OPTS=${HDFS_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"}
+ export HDFS_PORTMAP_OPTS=${HDFS_PORTMAP_OPTS:-"-Xmx512m"}
+
+ # depending upon what is being used to start Java, these may need to be
+ # set empty. (thus no colon)
+ export HDFS_DATANODE_SECURE_EXTRA_OPTS=${HDFS_DATANODE_SECURE_EXTRA_OPTS-"-jvm server"}
+ export HDFS_NFS3_SECURE_EXTRA_OPTS=${HDFS_NFS3_SECURE_EXTRA_OPTS-"-jvm server"}
}
if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/48] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by aw...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f388a209/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
new file mode 100644
index 0000000..e539a80
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/release/3.0.0-alpha1/CHANGES.3.0.0-alpha1.md
@@ -0,0 +1,3712 @@
+
+<!---
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+-->
+# "Apache Hadoop" Changelog
+
+## Release 3.0.0-alpha1 - 2016-08-30
+
+### INCOMPATIBLE CHANGES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [HADOOP-8124](https://issues.apache.org/jira/browse/HADOOP-8124) | Remove the deprecated Syncable.sync() method | Major | fs | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-3034](https://issues.apache.org/jira/browse/HDFS-3034) | Remove the deprecated Syncable.sync() method | Major | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-5079](https://issues.apache.org/jira/browse/HDFS-5079) | Cleaning up NNHAStatusHeartbeat.State DatanodeProtocolProtos. | Major | datanode, namenode | Konstantin Shvachko | Tao Luo |
+| [HDFS-5570](https://issues.apache.org/jira/browse/HDFS-5570) | Deprecate hftp / hsftp and replace them with webhdfs / swebhdfs | Major | . | Haohui Mai | Haohui Mai |
+| [HADOOP-10485](https://issues.apache.org/jira/browse/HADOOP-10485) | Remove dead classes in hadoop-streaming | Major | tools | Haohui Mai | Haohui Mai |
+| [HDFS-6246](https://issues.apache.org/jira/browse/HDFS-6246) | Remove 'dfs.support.append' flag from trunk code | Minor | . | Uma Maheswara Rao G | Uma Maheswara Rao G |
+| [HADOOP-10474](https://issues.apache.org/jira/browse/HADOOP-10474) | Move o.a.h.record to hadoop-streaming | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-2538](https://issues.apache.org/jira/browse/HDFS-2538) | option to disable fsck dots | Minor | . | Allen Wittenauer | Mohammad Kamrul Islam |
+| [HADOOP-9902](https://issues.apache.org/jira/browse/HADOOP-9902) | Shell script rewrite | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11041](https://issues.apache.org/jira/browse/HADOOP-11041) | VersionInfo output specifies subversion | Minor | build | Guo Ruijing | Tsuyoshi Ozawa |
+| [MAPREDUCE-5972](https://issues.apache.org/jira/browse/MAPREDUCE-5972) | Fix typo 'programatically' in job.xml (and a few other places) | Trivial | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-11356](https://issues.apache.org/jira/browse/HADOOP-11356) | Removed deprecated o.a.h.fs.permission.AccessControlException | Major | . | Haohui Mai | Li Lu |
+| [HADOOP-10950](https://issues.apache.org/jira/browse/HADOOP-10950) | rework heap management vars | Major | scripts | Allen Wittenauer | John Smith |
+| [MAPREDUCE-5785](https://issues.apache.org/jira/browse/MAPREDUCE-5785) | Derive heap size or mapreduce.\*.memory.mb automatically | Major | mr-am, task | Gera Shegalov | Gera Shegalov |
+| [YARN-2428](https://issues.apache.org/jira/browse/YARN-2428) | LCE default banned user list should have yarn | Trivial | nodemanager | Allen Wittenauer | Varun Saxena |
+| [MAPREDUCE-5653](https://issues.apache.org/jira/browse/MAPREDUCE-5653) | DistCp does not honour config-overrides for mapreduce.[map,reduce].memory.mb | Major | distcp | Mithun Radhakrishnan | Ratandeep Ratti |
+| [HADOOP-11657](https://issues.apache.org/jira/browse/HADOOP-11657) | Align the output of `hadoop fs -du` to be more Unix-like | Minor | . | Nick Dimiduk | Akira Ajisaka |
+| [HDFS-7302](https://issues.apache.org/jira/browse/HDFS-7302) | namenode -rollingUpgrade downgrade may finalize a rolling upgrade | Major | namenode | Tsz Wo Nicholas Sze | Kai Sasaki |
+| [HADOOP-6857](https://issues.apache.org/jira/browse/HADOOP-6857) | FsShell should report raw disk usage including replication factor | Major | fs | Alex Kozlov | Byron Wong |
+| [HADOOP-10115](https://issues.apache.org/jira/browse/HADOOP-10115) | Exclude duplicate jars in hadoop package under different component's lib | Major | build | Vinayakumar B | Vinayakumar B |
+| [YARN-3154](https://issues.apache.org/jira/browse/YARN-3154) | Should not upload partial logs for MR jobs or other "short-running' applications | Blocker | nodemanager, resourcemanager | Xuan Gong | Xuan Gong |
+| [MAPREDUCE-4424](https://issues.apache.org/jira/browse/MAPREDUCE-4424) | 'mapred job -list' command should show the job name as well | Minor | mrv2 | Harsh J | Avinash Kujur |
+| [YARN-3241](https://issues.apache.org/jira/browse/YARN-3241) | FairScheduler handles "invalid" queue names inconsistently | Major | fairscheduler | zhihai xu | zhihai xu |
+| [HDFS-7985](https://issues.apache.org/jira/browse/HDFS-7985) | WebHDFS should be always enabled | Major | . | Haohui Mai | Li Lu |
+| [HDFS-6353](https://issues.apache.org/jira/browse/HDFS-6353) | Check and make checkpoint before stopping the NameNode | Major | namenode | Suresh Srinivas | Jing Zhao |
+| [HADOOP-11553](https://issues.apache.org/jira/browse/HADOOP-11553) | Formalize the shell API | Blocker | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11731](https://issues.apache.org/jira/browse/HADOOP-11731) | Rework the changelog and releasenotes | Major | documentation | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11746](https://issues.apache.org/jira/browse/HADOOP-11746) | rewrite test-patch.sh | Major | build, test | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11627](https://issues.apache.org/jira/browse/HADOOP-11627) | Remove io.native.lib.available | Major | native | Akira Ajisaka | Brahma Reddy Battula |
+| [HDFS-7281](https://issues.apache.org/jira/browse/HDFS-7281) | Missing block is marked as corrupted block | Major | namenode | Ming Ma | Ming Ma |
+| [HDFS-8226](https://issues.apache.org/jira/browse/HDFS-8226) | Non-HA rollback compatibility broken | Blocker | . | J.Andreina | J.Andreina |
+| [MAPREDUCE-2632](https://issues.apache.org/jira/browse/MAPREDUCE-2632) | Avoid calling the partitioner when the numReduceTasks is 1. | Major | . | Ravi Teja Ch N V | Sunil G |
+| [HDFS-8241](https://issues.apache.org/jira/browse/HDFS-8241) | Remove unused NameNode startup option -finalize | Minor | namenode | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8332](https://issues.apache.org/jira/browse/HDFS-8332) | DFS client API calls should check filesystem closed | Major | . | Rakesh R | Rakesh R |
+| [HADOOP-11698](https://issues.apache.org/jira/browse/HADOOP-11698) | Remove DistCpV1 and Logalyzer | Major | tools/distcp | Allen Wittenauer | Brahma Reddy Battula |
+| [HADOOP-11772](https://issues.apache.org/jira/browse/HADOOP-11772) | RPC Invoker relies on static ClientCache which has synchronized(this) blocks | Major | ipc, performance | Gopal V | Haohui Mai |
+| [YARN-2336](https://issues.apache.org/jira/browse/YARN-2336) | Fair scheduler REST api returns a missing '[' bracket JSON for deep queue tree | Major | fairscheduler | Kenji Kikushima | Akira Ajisaka |
+| [HDFS-8135](https://issues.apache.org/jira/browse/HDFS-8135) | Remove the deprecated FSConstants class | Major | . | Haohui Mai | Li Lu |
+| [MAPREDUCE-6336](https://issues.apache.org/jira/browse/MAPREDUCE-6336) | Enable v2 FileOutputCommitter by default | Major | mrv2 | Gera Shegalov | Siqi Li |
+| [YARN-41](https://issues.apache.org/jira/browse/YARN-41) | The RM should handle the graceful shutdown of the NM. | Major | nodemanager, resourcemanager | Ravi Teja Ch N V | Devaraj K |
+| [HDFS-8591](https://issues.apache.org/jira/browse/HDFS-8591) | Remove support for deprecated configuration key dfs.namenode.decommission.nodes.per.interval | Minor | namenode | Andrew Wang | Andrew Wang |
+| [HDFS-6564](https://issues.apache.org/jira/browse/HDFS-6564) | Use slf4j instead of common-logging in hdfs-client | Major | build | Haohui Mai | Rakesh R |
+| [MAPREDUCE-6427](https://issues.apache.org/jira/browse/MAPREDUCE-6427) | Fix typo in JobHistoryEventHandler | Minor | . | Brahma Reddy Battula | Ray Chiang |
+| [HADOOP-12269](https://issues.apache.org/jira/browse/HADOOP-12269) | Update aws-sdk dependency to 1.10.6; move to aws-sdk-s3 | Major | fs/s3 | Thomas Demoor | Thomas Demoor |
+| [HDFS-8895](https://issues.apache.org/jira/browse/HDFS-8895) | Remove deprecated BlockStorageLocation APIs | Major | . | Andrew Wang | Andrew Wang |
+| [HDFS-8900](https://issues.apache.org/jira/browse/HDFS-8900) | Compact XAttrs to optimize memory footprint. | Major | namenode | Yi Liu | Yi Liu |
+| [HDFS-8981](https://issues.apache.org/jira/browse/HDFS-8981) | Adding revision to data node jmx getVersion() method | Minor | . | Siqi Li | Siqi Li |
+| [YARN-4087](https://issues.apache.org/jira/browse/YARN-4087) | Followup fixes after YARN-2019 regarding RM behavior when state-store error occurs | Major | . | Jian He | Jian He |
+| [HADOOP-12416](https://issues.apache.org/jira/browse/HADOOP-12416) | Trash messages should be handled by Logger instead of being delivered on System.out | Major | trash | Ashutosh Chauhan | Mingliang Liu |
+| [HDFS-9085](https://issues.apache.org/jira/browse/HDFS-9085) | Show renewer information in DelegationTokenIdentifier#toString | Trivial | security | zhihai xu | zhihai xu |
+| [HADOOP-12495](https://issues.apache.org/jira/browse/HADOOP-12495) | Fix posix\_spawn error on OS X | Major | util | Allen Wittenauer | Allen Wittenauer |
+| [HDFS-9070](https://issues.apache.org/jira/browse/HDFS-9070) | Allow fsck display pending replica location information for being-written blocks | Major | . | Rui Gao | Rui Gao |
+| [HDFS-9278](https://issues.apache.org/jira/browse/HDFS-9278) | Fix preferredBlockSize typo in OIV XML output | Trivial | tools | Nicole Pazmany | Nicole Pazmany |
+| [HADOOP-10787](https://issues.apache.org/jira/browse/HADOOP-10787) | Rename/remove non-HADOOP\_\*, etc from the shell scripts | Blocker | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12294](https://issues.apache.org/jira/browse/HADOOP-12294) | Throw an Exception when fs.permissions.umask-mode is misconfigured | Major | conf | Chang Li | Chang Li |
+| [HADOOP-10465](https://issues.apache.org/jira/browse/HADOOP-10465) | Fix use of generics within SortedMapWritable | Minor | . | Bertrand Dechoux | Bertrand Dechoux |
+| [HDFS-5165](https://issues.apache.org/jira/browse/HDFS-5165) | Remove the TotalFiles metrics | Minor | . | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-9047](https://issues.apache.org/jira/browse/HDFS-9047) | Retire libwebhdfs | Major | webhdfs | Allen Wittenauer | Haohui Mai |
+| [HADOOP-12651](https://issues.apache.org/jira/browse/HADOOP-12651) | Replace dev-support with wrappers to Yetus | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6613](https://issues.apache.org/jira/browse/MAPREDUCE-6613) | Change mapreduce.jobhistory.jhist.format default from json to binary | Minor | . | Ray Chiang | Ray Chiang |
+| [HDFS-9395](https://issues.apache.org/jira/browse/HDFS-9395) | Make HDFS audit logging consistant | Major | . | Kihwal Lee | Kuhu Shukla |
+| [HADOOP-12552](https://issues.apache.org/jira/browse/HADOOP-12552) | Fix undeclared/unused dependency to httpclient | Minor | build | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-12850](https://issues.apache.org/jira/browse/HADOOP-12850) | pull shell code out of hadoop-dist | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11792](https://issues.apache.org/jira/browse/HADOOP-11792) | Remove all of the CHANGES.txt files | Major | build | Allen Wittenauer | Andrew Wang |
+| [HADOOP-12857](https://issues.apache.org/jira/browse/HADOOP-12857) | Rework hadoop-tools | Major | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12967](https://issues.apache.org/jira/browse/HADOOP-12967) | Remove FileUtil#copyMerge | Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HADOOP-12811](https://issues.apache.org/jira/browse/HADOOP-12811) | Change kms server port number which conflicts with HMaster port number | Critical | kms | Yufeng Jiang | Xiao Chen |
+| [YARN-4784](https://issues.apache.org/jira/browse/YARN-4784) | Fairscheduler: defaultQueueSchedulingPolicy should not accept FIFO | Major | fairscheduler | Yufei Gu | Yufei Gu |
+| [HDFS-9427](https://issues.apache.org/jira/browse/HDFS-9427) | HDFS should not default to ephemeral ports | Critical | datanode, hdfs-client, namenode | Arpit Agarwal | Xiaobing Zhou |
+| [HADOOP-12563](https://issues.apache.org/jira/browse/HADOOP-12563) | Updated utility to create/modify token files | Major | . | Allen Wittenauer | Matthew Paduano |
+| [HADOOP-13045](https://issues.apache.org/jira/browse/HADOOP-13045) | hadoop\_add\_classpath is not working in .hadooprc | Major | . | Akira Ajisaka | Allen Wittenauer |
+| [MAPREDUCE-6526](https://issues.apache.org/jira/browse/MAPREDUCE-6526) | Remove usage of metrics v1 from hadoop-mapreduce | Blocker | . | Akira Ajisaka | Akira Ajisaka |
+| [HADOOP-12504](https://issues.apache.org/jira/browse/HADOOP-12504) | Remove metrics v1 | Blocker | metrics | Akira Ajisaka | Akira Ajisaka |
+| [HDFS-10337](https://issues.apache.org/jira/browse/HDFS-10337) | OfflineEditsViewer stats option should print 0 instead of null for the count of operations | Minor | . | Akira Ajisaka | Yiqun Lin |
+| [HADOOP-11858](https://issues.apache.org/jira/browse/HADOOP-11858) | [JDK8] Set minimum version of Hadoop 3 to JDK 8 | Blocker | build | Robert Kanter | Robert Kanter |
+| [YARN-5035](https://issues.apache.org/jira/browse/YARN-5035) | FairScheduler: Adjust maxAssign dynamically when assignMultiple is turned on | Major | fairscheduler | Karthik Kambatla | Karthik Kambatla |
+| [HADOOP-13175](https://issues.apache.org/jira/browse/HADOOP-13175) | Remove hadoop-ant from hadoop-tools | Major | . | Chris Douglas | Chris Douglas |
+| [HADOOP-12892](https://issues.apache.org/jira/browse/HADOOP-12892) | fix/rewrite create-release | Blocker | build | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9613](https://issues.apache.org/jira/browse/HADOOP-9613) | [JDK8] Update jersey version to latest 1.x release | Major | build | Timothy St. Clair | Tsuyoshi Ozawa |
+| [HADOOP-13209](https://issues.apache.org/jira/browse/HADOOP-13209) | replace slaves with workers | Major | scripts | John Smith | John Smith |
+| [HDFS-6434](https://issues.apache.org/jira/browse/HDFS-6434) | Default permission for creating file should be 644 for WebHdfs/HttpFS | Minor | webhdfs | Juan Yu | Wellington Chevreuil |
+| [HADOOP-12864](https://issues.apache.org/jira/browse/HADOOP-12864) | Remove bin/rcc script | Blocker | tools | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12709](https://issues.apache.org/jira/browse/HADOOP-12709) | Cut s3:// from trunk | Major | fs/s3 | Steve Loughran | Mingliang Liu |
+| [HADOOP-12064](https://issues.apache.org/jira/browse/HADOOP-12064) | [JDK8] Update guice version to 4.0 | Blocker | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [HDFS-10548](https://issues.apache.org/jira/browse/HDFS-10548) | Remove the long deprecated BlockReaderRemote | Major | hdfs | Kai Zheng | Kai Zheng |
+| [HADOOP-13382](https://issues.apache.org/jira/browse/HADOOP-13382) | remove unneeded commons-httpclient dependencies from POM files in Hadoop and sub-projects | Major | build | Matt Foley | Matt Foley |
+| [HDFS-10689](https://issues.apache.org/jira/browse/HDFS-10689) | Hdfs dfs chmod should reset sticky bit permission when the bit is omitted in the octal mode | Minor | fs | Manoj Govindassamy | Manoj Govindassamy |
+| [HDFS-7933](https://issues.apache.org/jira/browse/HDFS-7933) | fsck should also report decommissioning replicas. | Major | namenode | Jitendra Nath Pandey | Xiaoyu Yao |
+| [HADOOP-12726](https://issues.apache.org/jira/browse/HADOOP-12726) | Unsupported FS operations should throw UnsupportedOperationException | Major | fs | Daniel Templeton | Daniel Templeton |
+
+
+### IMPORTANT ISSUES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [YARN-4785](https://issues.apache.org/jira/browse/YARN-4785) | inconsistent value type of the "type" field for LeafQueueInfo in response of RM REST API - cluster/scheduler | Major | webapp | Jayesh | Varun Vasudev |
+| [HADOOP-11393](https://issues.apache.org/jira/browse/HADOOP-11393) | Revert HADOOP\_PREFIX, go back to HADOOP\_HOME | Major | scripts, tracing | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12930](https://issues.apache.org/jira/browse/HADOOP-12930) | [Umbrella] Dynamic subcommands for hadoop shell scripts | Critical | scripts | Allen Wittenauer | Allen Wittenauer |
+
+
+### NEW FEATURES:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-4887](https://issues.apache.org/jira/browse/MAPREDUCE-4887) | Rehashing partitioner for better distribution | Major | . | Radim Kolar | Radim Kolar |
+| [HADOOP-11013](https://issues.apache.org/jira/browse/HADOOP-11013) | CLASSPATH handling should be consolidated, debuggable | Major | . | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-6590](https://issues.apache.org/jira/browse/HADOOP-6590) | Add a username check for hadoop sub-commands | Minor | scripts | Scott Chen | John Smith |
+| [HADOOP-11353](https://issues.apache.org/jira/browse/HADOOP-11353) | Add support for .hadooprc | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9044](https://issues.apache.org/jira/browse/HADOOP-9044) | add FindClass main class to provide classpath checking of installations | Minor | util | Steve Loughran | Steve Loughran |
+| [HADOOP-11485](https://issues.apache.org/jira/browse/HADOOP-11485) | Pluggable shell integration | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-8934](https://issues.apache.org/jira/browse/HADOOP-8934) | Shell command ls should include sort options | Minor | fs | Jonathan Allen | Jonathan Allen |
+| [HADOOP-11554](https://issues.apache.org/jira/browse/HADOOP-11554) | Expose HadoopKerberosName as a hadoop subcommand | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11565](https://issues.apache.org/jira/browse/HADOOP-11565) | Add --slaves shell option | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-9477](https://issues.apache.org/jira/browse/HADOOP-9477) | Add posixGroups support for LDAP groups mapping service | Major | . | Kai Zheng | Dapeng Sun |
+| [HDFS-8009](https://issues.apache.org/jira/browse/HDFS-8009) | Signal congestion on the DataNode | Major | datanode | Haohui Mai | Haohui Mai |
+| [YARN-2901](https://issues.apache.org/jira/browse/YARN-2901) | Add errors and warning metrics page to RM, NM web UI | Major | nodemanager, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [HADOOP-9642](https://issues.apache.org/jira/browse/HADOOP-9642) | Configuration to resolve environment variables via ${env.VARIABLE} references | Minor | conf, scripts | Steve Loughran | Kengo Seki |
+| [HDFS-7891](https://issues.apache.org/jira/browse/HDFS-7891) | A block placement policy with best rack failure tolerance | Minor | namenode | Walter Su | Walter Su |
+| [HADOOP-11843](https://issues.apache.org/jira/browse/HADOOP-11843) | Make setting up the build environment easier | Major | build | Niels Basjes | Niels Basjes |
+| [MAPREDUCE-6284](https://issues.apache.org/jira/browse/MAPREDUCE-6284) | Add Task Attempt State API to MapReduce Application Master REST API | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-10971](https://issues.apache.org/jira/browse/HADOOP-10971) | Add -C flag to make `hadoop fs -ls` print filenames only | Major | fs | Ryan Williams | Kengo Seki |
+| [MAPREDUCE-6364](https://issues.apache.org/jira/browse/MAPREDUCE-6364) | Add a "Kill" link to Task Attempts page | Minor | applicationmaster | Ryu Kobayashi | Ryu Kobayashi |
+| [MAPREDUCE-6304](https://issues.apache.org/jira/browse/MAPREDUCE-6304) | Specifying node labels when submitting MR jobs | Major | . | Jian Fang | Naganarasimha G R |
+| [HADOOP-7947](https://issues.apache.org/jira/browse/HADOOP-7947) | Validate XMLs if a relevant tool is available, when using scripts | Major | scripts | Harsh J | Kengo Seki |
+| [HDFS-8487](https://issues.apache.org/jira/browse/HDFS-8487) | Generalize BlockInfo in preparation of merging HDFS-7285 into trunk and branch-2 | Major | namenode | Zhe Zhang | Zhe Zhang |
+| [HDFS-8608](https://issues.apache.org/jira/browse/HDFS-8608) | Merge HDFS-7912 to trunk and branch-2 (track BlockInfo instead of Block in UnderReplicatedBlocks and PendingReplicationBlocks) | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-6440](https://issues.apache.org/jira/browse/HDFS-6440) | Support more than 2 NameNodes | Major | auto-failover, ha, namenode | Jesse Yates | Jesse Yates |
+| [HADOOP-5732](https://issues.apache.org/jira/browse/HADOOP-5732) | Add SFTP FileSystem | Minor | fs | ��igo Goiri | ramtin |
+| [HDFS-8622](https://issues.apache.org/jira/browse/HDFS-8622) | Implement GETCONTENTSUMMARY operation for WebImageViewer | Major | . | Jagadesh Kiran N | Jagadesh Kiran N |
+| [HDFS-8155](https://issues.apache.org/jira/browse/HDFS-8155) | Support OAuth2 in WebHDFS | Major | webhdfs | Jakob Homan | Jakob Homan |
+| [MAPREDUCE-6415](https://issues.apache.org/jira/browse/MAPREDUCE-6415) | Create a tool to combine aggregated logs into HAR files | Major | . | Robert Kanter | Robert Kanter |
+| [HADOOP-12360](https://issues.apache.org/jira/browse/HADOOP-12360) | Create StatsD metrics2 sink | Minor | metrics | Dave Marion | Dave Marion |
+| [HDFS-7285](https://issues.apache.org/jira/browse/HDFS-7285) | Erasure Coding Support inside HDFS | Major | . | Weihua Jiang | Zhe Zhang |
+| [HADOOP-10949](https://issues.apache.org/jira/browse/HADOOP-10949) | metrics2 sink plugin for Apache Kafka | Major | metrics | Babak Behzad | Babak Behzad |
+| [YARN-261](https://issues.apache.org/jira/browse/YARN-261) | Ability to fail AM attempts | Major | api | Jason Lowe | Rohith Sharma K S |
+| [HDFS-9184](https://issues.apache.org/jira/browse/HDFS-9184) | Logging HDFS operation's caller context into audit logs | Major | . | Mingliang Liu | Mingliang Liu |
+| [HDFS-9057](https://issues.apache.org/jira/browse/HDFS-9057) | allow/disallow snapshots via webhdfs | Major | webhdfs | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-4349](https://issues.apache.org/jira/browse/YARN-4349) | Support CallerContext in YARN | Major | . | Wangda Tan | Wangda Tan |
+| [HADOOP-12366](https://issues.apache.org/jira/browse/HADOOP-12366) | expose calculated paths | Major | . | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12321](https://issues.apache.org/jira/browse/HADOOP-12321) | Make JvmPauseMonitor an AbstractService | Major | . | Steve Loughran | Sunil G |
+| [YARN-3623](https://issues.apache.org/jira/browse/YARN-3623) | We should have a config to indicate the Timeline Service version | Major | timelineserver | Zhijie Shen | Xuan Gong |
+| [HADOOP-12657](https://issues.apache.org/jira/browse/HADOOP-12657) | Add a option to skip newline on empty files with getMerge -nl | Minor | . | Jan Filipiak | Kanaka Kumar Avvaru |
+| [YARN-3458](https://issues.apache.org/jira/browse/YARN-3458) | CPU resource monitoring in Windows | Minor | nodemanager | Inigo Goiri | Inigo Goiri |
+| [HADOOP-12691](https://issues.apache.org/jira/browse/HADOOP-12691) | Add CSRF Filter for REST APIs to Hadoop Common | Major | security | Larry McCay | Larry McCay |
+| [HADOOP-12635](https://issues.apache.org/jira/browse/HADOOP-12635) | Adding Append API support for WASB | Major | azure | Dushyanth | Dushyanth |
+| [HDFS-9525](https://issues.apache.org/jira/browse/HDFS-9525) | hadoop utilities need to support provided delegation tokens | Blocker | security | Allen Wittenauer | HeeSoo Kim |
+| [HADOOP-12702](https://issues.apache.org/jira/browse/HADOOP-12702) | Add an HDFS metrics sink | Major | metrics | Daniel Templeton | Daniel Templeton |
+| [HADOOP-12426](https://issues.apache.org/jira/browse/HADOOP-12426) | Add Entry point for Kerberos health check | Minor | security | Steve Loughran | Steve Loughran |
+| [HDFS-9244](https://issues.apache.org/jira/browse/HDFS-9244) | Support nested encryption zones | Major | encryption | Xiaoyu Yao | Zhe Zhang |
+| [HADOOP-12548](https://issues.apache.org/jira/browse/HADOOP-12548) | Read s3a creds from a Credential Provider | Major | fs/s3 | Allen Wittenauer | Larry McCay |
+| [HDFS-9711](https://issues.apache.org/jira/browse/HDFS-9711) | Integrate CSRF prevention filter in WebHDFS. | Major | datanode, namenode, webhdfs | Chris Nauroth | Chris Nauroth |
+| [HDFS-9835](https://issues.apache.org/jira/browse/HDFS-9835) | OIV: add ReverseXML processor which reconstructs an fsimage from an XML file | Major | tools | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-9239](https://issues.apache.org/jira/browse/HDFS-9239) | DataNode Lifeline Protocol: an alternative protocol for reporting DataNode liveness | Major | datanode, namenode | Chris Nauroth | Chris Nauroth |
+| [HADOOP-12909](https://issues.apache.org/jira/browse/HADOOP-12909) | Change ipc.Client to support asynchronous calls | Major | ipc | Tsz Wo Nicholas Sze | Xiaobing Zhou |
+| [HDFS-9945](https://issues.apache.org/jira/browse/HDFS-9945) | Datanode command for evicting writers | Major | datanode | Kihwal Lee | Kihwal Lee |
+| [HADOOP-13008](https://issues.apache.org/jira/browse/HADOOP-13008) | Add XFS Filter for UIs to Hadoop Common | Major | security | Larry McCay | Larry McCay |
+| [HADOOP-13065](https://issues.apache.org/jira/browse/HADOOP-13065) | Add a new interface for retrieving FS and FC Statistics | Major | fs | Ram Venkatesh | Mingliang Liu |
+| [HADOOP-12723](https://issues.apache.org/jira/browse/HADOOP-12723) | S3A: Add ability to plug in any AWSCredentialsProvider | Major | fs/s3 | Steven K. Wong | Steven K. Wong |
+| [HADOOP-12847](https://issues.apache.org/jira/browse/HADOOP-12847) | hadoop daemonlog should support https and SPNEGO for Kerberized cluster | Major | security | Wei-Chiu Chuang | Wei-Chiu Chuang |
+| [HADOOP-13226](https://issues.apache.org/jira/browse/HADOOP-13226) | Support async call retry and failover | Major | io, ipc | Xiaobing Zhou | Tsz Wo Nicholas Sze |
+| [HADOOP-12537](https://issues.apache.org/jira/browse/HADOOP-12537) | S3A to support Amazon STS temporary credentials | Minor | fs/s3 | Sean Mackrory | Sean Mackrory |
+| [HADOOP-12666](https://issues.apache.org/jira/browse/HADOOP-12666) | Support Microsoft Azure Data Lake - as a file system in Hadoop | Major | fs, fs/azure, tools | Vishwajeet Dusane | Vishwajeet Dusane |
+| [HDFS-1312](https://issues.apache.org/jira/browse/HDFS-1312) | Re-balance disks within a Datanode | Major | datanode | Travis Crawford | Anu Engineer |
+| [YARN-2928](https://issues.apache.org/jira/browse/YARN-2928) | YARN Timeline Service v.2: alpha 1 | Critical | timelineserver | Sangjin Lee | Sangjin Lee |
+| [MAPREDUCE-6331](https://issues.apache.org/jira/browse/MAPREDUCE-6331) | [Umbrella] Make MapReduce work with Timeline Service Nextgen (YARN-2928) | Major | . | Vinod Kumar Vavilapalli | Sangjin Lee |
+| [HADOOP-12747](https://issues.apache.org/jira/browse/HADOOP-12747) | support wildcard in libjars argument | Major | util | Sangjin Lee | Sangjin Lee |
+| [HDFS-9804](https://issues.apache.org/jira/browse/HDFS-9804) | Allow long-running Balancer to login with keytab | Major | balancer & mover, security | Xiao Chen | Xiao Chen |
+| [MAPREDUCE-6690](https://issues.apache.org/jira/browse/MAPREDUCE-6690) | Limit the number of resources a single map reduce job can submit for localization | Major | . | Chris Trezzo | Chris Trezzo |
+| [MAPREDUCE-6578](https://issues.apache.org/jira/browse/MAPREDUCE-6578) | Add support for HDFS heterogeneous storage testing to TestDFSIO | Major | . | Wei Zhou | Wei Zhou |
+| [HADOOP-13396](https://issues.apache.org/jira/browse/HADOOP-13396) | Allow pluggable audit loggers in KMS | Major | kms | Xiao Chen | Xiao Chen |
+
+
+### IMPROVEMENTS:
+
+| JIRA | Summary | Priority | Component | Reporter | Contributor |
+|:---- |:---- | :--- |:---- |:---- |:---- |
+| [MAPREDUCE-2944](https://issues.apache.org/jira/browse/MAPREDUCE-2944) | Improve checking of input for Api displayTasks() | Trivial | client | XieXianshan | XieXianshan |
+| [HADOOP-8017](https://issues.apache.org/jira/browse/HADOOP-8017) | Configure hadoop-main pom to get rid of M2E plugin execution not covered | Major | build | Eric Charles | Eric Charles |
+| [HADOOP-8078](https://issues.apache.org/jira/browse/HADOOP-8078) | Add capability to turn on security in unit tests. | Major | test | Jitendra Nath Pandey | Jaimin D Jetly |
+| [MAPREDUCE-3956](https://issues.apache.org/jira/browse/MAPREDUCE-3956) | Remove the use of the deprecated Syncable.sync() method | Minor | examples | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HADOOP-8147](https://issues.apache.org/jira/browse/HADOOP-8147) | test-patch should run tests with -fn to avoid masking test failures | Major | test | Robert Joseph Evans | Robert Joseph Evans |
+| [HADOOP-8290](https://issues.apache.org/jira/browse/HADOOP-8290) | Remove two remaining references to "hadoop.native.lib" oldprop | Trivial | test | Harsh J | Harsh J |
+| [HDFS-2743](https://issues.apache.org/jira/browse/HDFS-2743) | Streamline usage of bookkeeper journal manager | Major | ha | Ivan Kelly | Ivan Kelly |
+| [HADOOP-8312](https://issues.apache.org/jira/browse/HADOOP-8312) | testpatch.sh should provide a simpler way to see which warnings changed | Major | scripts | Robert Joseph Evans | Robert Joseph Evans |
+| [HDFS-3293](https://issues.apache.org/jira/browse/HDFS-3293) | Implement equals for journainfo class. | Minor | namenode | Hari Mankude | Hari Mankude |
+| [HADOOP-8375](https://issues.apache.org/jira/browse/HADOOP-8375) | test-patch should stop immediately once it has found compilation errors | Major | build | Tsz Wo Nicholas Sze | Robert Joseph Evans |
+| [HADOOP-8297](https://issues.apache.org/jira/browse/HADOOP-8297) | Writable javadocs don't carry default constructor | Trivial | documentation | Harsh J | Harsh J |
+| [HADOOP-7659](https://issues.apache.org/jira/browse/HADOOP-7659) | fs -getmerge isn't guaranteed to work well over non-HDFS filesystems | Minor | fs | Harsh J | Harsh J |
+| [MAPREDUCE-3302](https://issues.apache.org/jira/browse/MAPREDUCE-3302) | Remove the last dependency call from org.apache.hadoop.record package in MR. | Minor | client | Harsh J | Harsh J |
+| [MAPREDUCE-2384](https://issues.apache.org/jira/browse/MAPREDUCE-2384) | The job submitter should make sure to validate jobs before creation of necessary files | Major | job submission, test | Denny Ye | Harsh J |
+| [HDFS-3476](https://issues.apache.org/jira/browse/HDFS-3476) | Correct the default used in TestDFSClientRetries.busyTest() after HDFS-3462 | Minor | test | Harsh J | Harsh J |
+| [HDFS-3478](https://issues.apache.org/jira/browse/HDFS-3478) | Test quotas with Long.Max\_Value | Minor | namenode | Sujay Rau | Sujay Rau |
+| [MAPREDUCE-4371](https://issues.apache.org/jira/browse/MAPREDUCE-4371) | Check for cyclic dependencies in Jobcontrol job DAG | Major | mrv1 | madhukara phatak | madhukara phatak |
+| [HADOOP-8523](https://issues.apache.org/jira/browse/HADOOP-8523) | test-patch.sh doesn't validate patches before building | Minor | build | Jack Dintruff | Jack Dintruff |
+| [HADOOP-8521](https://issues.apache.org/jira/browse/HADOOP-8521) | Port StreamInputFormat to new Map Reduce API | Major | streaming | madhukara phatak | madhukara phatak |
+| [HADOOP-8593](https://issues.apache.org/jira/browse/HADOOP-8593) | add the missed @Override to methods in Metric/Metric2 package | Minor | metrics | Brandon Li | Brandon Li |
+| [HDFS-3768](https://issues.apache.org/jira/browse/HDFS-3768) | Exception in TestJettyHelper is incorrect | Minor | test | Jakob Homan | Eli Reisman |
+| [HDFS-3834](https://issues.apache.org/jira/browse/HDFS-3834) | Unused Command.java fields NAME, DESCRIPTION and USAGE | Minor | shell | Jing Zhao | Jing Zhao |
+| [HDFS-2580](https://issues.apache.org/jira/browse/HDFS-2580) | NameNode#main(...) can make use of GenericOptionsParser. | Minor | namenode | Harsh J | Harsh J |
+| [HADOOP-8588](https://issues.apache.org/jira/browse/HADOOP-8588) | SerializationFactory shouldn't throw a NullPointerException if the serializations list is empty | Minor | io | Harsh J | Sho Shimauchi |
+| [HADOOP-7930](https://issues.apache.org/jira/browse/HADOOP-7930) | Kerberos relogin interval in UserGroupInformation should be configurable | Major | security | Alejandro Abdelnur | Robert Kanter |
+| [HADOOP-8838](https://issues.apache.org/jira/browse/HADOOP-8838) | Colorize the test-patch output sent to JIRA | Trivial | build | Harsh J | Harsh J |
+| [MAPREDUCE-4686](https://issues.apache.org/jira/browse/MAPREDUCE-4686) | hadoop-mapreduce-client-core fails compilation in Eclipse due to missing Avro-generated classes | Major | build | Chris Nauroth | Chris Nauroth |
+| [HADOOP-8864](https://issues.apache.org/jira/browse/HADOOP-8864) | Addendum to HADOOP-8840: Add a coloring case for +0 results too. | Trivial | precommit | Harsh J | Harsh J |
+| [HDFS-3995](https://issues.apache.org/jira/browse/HDFS-3995) | Use DFSTestUtil.createFile() for file creation and writing in test cases | Minor | test | Jing Zhao | Jing Zhao |
+| [HADOOP-8920](https://issues.apache.org/jira/browse/HADOOP-8920) | Add more javadoc to metrics2 related classes | Minor | metrics | Suresh Srinivas | Suresh Srinivas |
+| [HADOOP-8776](https://issues.apache.org/jira/browse/HADOOP-8776) | Provide an option in test-patch that can enable / disable compiling native code | Minor | build | Hemanth Yamijala | Chris Nauroth |
+| [MAPREDUCE-4735](https://issues.apache.org/jira/browse/MAPREDUCE-4735) | Make arguments in TestDFSIO case insensitive | Trivial | test | Robert Kanter | Brandon Li |
+| [HADOOP-6616](https://issues.apache.org/jira/browse/HADOOP-6616) | Improve documentation for rack awareness | Major | documentation | Jeff Hammerbacher | Adam Faris |
+| [HADOOP-9075](https://issues.apache.org/jira/browse/HADOOP-9075) | FileContext#FSLinkResolver should be made static | Minor | . | Suresh Srinivas | Arpit Agarwal |
+| [HDFS-3358](https://issues.apache.org/jira/browse/HDFS-3358) | Specify explicitly that the NN UI status total is talking of persistent objects on heap. | Trivial | namenode | Harsh J | Harsh J |
+| [HADOOP-9277](https://issues.apache.org/jira/browse/HADOOP-9277) | Improve javadoc for FileContext | Minor | . | Andrew Wang | Andrew Wang |
+| [HADOOP-9394](https://issues.apache.org/jira/browse/HADOOP-9394) | Port findHangingTest.sh from HBase to Hadoop | Minor | . | Andrew Wang | Andrew Wang |
+| [HDFS-4665](https://issues.apache.org/jira/browse/HDFS-4665) | Move TestNetworkTopologyWithNodeGroup to Common | Minor | test | Luke Lu | Junping Du |
+| [MAPREDUCE-5232](https://issues.apache.org/jira/browse/MAPREDUCE-5232) | log classpath and other key properties on child JVM start | Major | mrv1, mrv2 | Sangjin Lee | Sangjin Lee |
+| [HDFS-5041](https://issues.apache.org/jira/browse/HDFS-5041) | Add the time of last heartbeat to dead server Web UI | Minor | . | Ted Yu | Ted Yu |
+| [HADOOP-10044](https://issues.apache.org/jira/browse/HADOOP-10044) | Improve the javadoc of rpc code | Minor | . | Sanjay Radia | Sanjay Radia |
+| [MAPREDUCE-5197](https://issues.apache.org/jira/browse/MAPREDUCE-5197) | Checkpoint Service: a library component to facilitate checkpoint of task state | Major | mrv2 | Carlo Curino | Carlo Curino |
+| [MAPREDUCE-5196](https://issues.apache.org/jira/browse/MAPREDUCE-5196) | CheckpointAMPreemptionPolicy implements preemption in MR AM via checkpointing | Major | mr-am, mrv2 | Carlo Curino | Carlo Curino |
+| [HDFS-6228](https://issues.apache.org/jira/browse/HDFS-6228) | comments typo fix for FsDatasetImpl.java | Trivial | . | yunjiong zhao | yunjiong zhao |
+| [HADOOP-8719](https://issues.apache.org/jira/browse/HADOOP-8719) | Workaround for kerberos-related log errors upon running any hadoop command on OSX | Trivial | scripts | Jianbin Wei | Jianbin Wei |
+| [HADOOP-10834](https://issues.apache.org/jira/browse/HADOOP-10834) | Typo in CredentialShell usage | Minor | security | Benoy Antony | Benoy Antony |
+| [MAPREDUCE-6013](https://issues.apache.org/jira/browse/MAPREDUCE-6013) | mapred version is missing | Major | scripts | Allen Wittenauer | Akira Ajisaka |
+| [MAPREDUCE-2841](https://issues.apache.org/jira/browse/MAPREDUCE-2841) | Task level native optimization | Major | task | Binglin Chang | Sean Zhong |
+| [YARN-2438](https://issues.apache.org/jira/browse/YARN-2438) | yarn-env.sh cleanup | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11092](https://issues.apache.org/jira/browse/HADOOP-11092) | hadoop shell commands should print usage if not given a class | Major | scripts | Bruno Mah� | Allen Wittenauer |
+| [HDFS-7278](https://issues.apache.org/jira/browse/HDFS-7278) | Add a command that allows sysadmins to manually trigger full block reports from a DN | Major | datanode | Colin P. McCabe | Colin P. McCabe |
+| [HADOOP-11025](https://issues.apache.org/jira/browse/HADOOP-11025) | hadoop-daemons.sh should just call hdfs directly | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [HADOOP-11150](https://issues.apache.org/jira/browse/HADOOP-11150) | hadoop command should show the reason on failure by invalid COMMAND or CLASSNAME | Minor | scripts | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-11208](https://issues.apache.org/jira/browse/HADOOP-11208) | Replace "daemon" with better name in script subcommands | Major | scripts | Yongjun Zhang | Allen Wittenauer |
+| [HADOOP-10926](https://issues.apache.org/jira/browse/HADOOP-10926) | Improve smart-apply-patch.sh to apply binary diffs | Major | . | Andrew Wang | Colin P. McCabe |
+| [HADOOP-11081](https://issues.apache.org/jira/browse/HADOOP-11081) | Document hadoop properties expected to be set by the shell code in \*-env.sh | Major | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [YARN-2472](https://issues.apache.org/jira/browse/YARN-2472) | yarn-daemons.sh should just call yarn directly | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [YARN-2301](https://issues.apache.org/jira/browse/YARN-2301) | Improve yarn container command | Major | . | Jian He | Naganarasimha G R |
+| [HADOOP-11352](https://issues.apache.org/jira/browse/HADOOP-11352) | Clean up test-patch.sh to disable "+1 contrib tests" | Major | build | Akira Ajisaka | Akira Ajisaka |
+| [YARN-2437](https://issues.apache.org/jira/browse/YARN-2437) | start-yarn.sh/stop-yarn should give info | Major | scripts | Allen Wittenauer | Varun Saxena |
+| [HDFS-7531](https://issues.apache.org/jira/browse/HDFS-7531) | Improve the concurrent access on FsVolumeList | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-11058](https://issues.apache.org/jira/browse/HADOOP-11058) | Missing HADOOP\_CONF\_DIR generates strange results | Major | scripts | Allen Wittenauer | Masatake Iwasaki |
+| [HDFS-7579](https://issues.apache.org/jira/browse/HDFS-7579) | Improve log reporting during block report rpc failure | Minor | datanode | Charles Lamb | Charles Lamb |
+| [HDFS-7446](https://issues.apache.org/jira/browse/HDFS-7446) | HDFS inotify should have the ability to determine what txid it has read up to | Major | hdfs-client | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-7182](https://issues.apache.org/jira/browse/HDFS-7182) | JMX metrics aren't accessible when NN is busy | Major | . | Ming Ma | Ming Ma |
+| [HADOOP-11466](https://issues.apache.org/jira/browse/HADOOP-11466) | FastByteComparisons: do not use UNSAFE\_COMPARER on the SPARC architecture because it is slower there | Minor | io, performance, util | Suman Somasundar | Suman Somasundar |
+| [HADOOP-11460](https://issues.apache.org/jira/browse/HADOOP-11460) | Deprecate shell vars | Major | scripts | Allen Wittenauer | John Smith |
+| [HADOOP-11346](https://issues.apache.org/jira/browse/HADOOP-11346) | Rewrite sls/rumen to use new shell framework | Major | scripts, tools | Allen Wittenauer | John Smith |
+| [HADOOP-7713](https://issues.apache.org/jira/browse/HADOOP-7713) | dfs -count -q should label output column | Trivial | . | Nigel Daley | Jonathan Allen |
+| [HADOOP-6964](https://issues.apache.org/jira/browse/HADOOP-6964) | Allow compact property description in xml | Major | conf | Owen O'Malley | Kengo Seki |
+| [HADOOP-11506](https://issues.apache.org/jira/browse/HADOOP-11506) | Configuration variable expansion regex expensive for long values | Major | conf | Dmitriy V. Ryaboy | Gera Shegalov |
+| [HDFS-7546](https://issues.apache.org/jira/browse/HDFS-7546) | Document, and set an accepting default for dfs.namenode.kerberos.principal.pattern | Minor | security | Harsh J | Harsh J |
+| [HDFS-7694](https://issues.apache.org/jira/browse/HDFS-7694) | FSDataInputStream should support "unbuffer" | Major | . | Colin P. McCabe | Colin P. McCabe |
+| [HDFS-7322](https://issues.apache.org/jira/browse/HDFS-7322) | deprecate sbin/hadoop-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [YARN-2796](https://issues.apache.org/jira/browse/YARN-2796) | deprecate sbin/yarn-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6250](https://issues.apache.org/jira/browse/MAPREDUCE-6250) | deprecate sbin/mr-jobhistory-daemon.sh | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6260](https://issues.apache.org/jira/browse/MAPREDUCE-6260) | Convert site documentation to markdown | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [HADOOP-11575](https://issues.apache.org/jira/browse/HADOOP-11575) | Daemon log documentation is misleading | Major | documentation | Jagadesh Kiran N | Naganarasimha G R |
+| [HADOOP-11596](https://issues.apache.org/jira/browse/HADOOP-11596) | Allow smart-apply-patch.sh to add new files in binary git patches | Major | . | Ravi Prakash | Ravi Prakash |
+| [HADOOP-11593](https://issues.apache.org/jira/browse/HADOOP-11593) | Convert site documentation from apt to markdown (stragglers) | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [YARN-3230](https://issues.apache.org/jira/browse/YARN-3230) | Clarify application states on the web UI | Major | . | Jian He | Jian He |
+| [HDFS-7460](https://issues.apache.org/jira/browse/HDFS-7460) | Rewrite httpfs to use new shell framework | Major | scripts | Allen Wittenauer | John Smith |
+| [YARN-3168](https://issues.apache.org/jira/browse/YARN-3168) | Convert site documentation from apt to markdown | Major | documentation | Allen Wittenauer | Gururaj Shetty |
+| [YARN-3199](https://issues.apache.org/jira/browse/YARN-3199) | Fair Scheduler documentation improvements | Minor | fairscheduler | Rohit Agarwal | Rohit Agarwal |
+| [MAPREDUCE-6267](https://issues.apache.org/jira/browse/MAPREDUCE-6267) | Refactor JobSubmitter#copyAndConfigureFiles into it's own class | Minor | . | Chris Trezzo | Chris Trezzo |
+| [YARN-3249](https://issues.apache.org/jira/browse/YARN-3249) | Add a "kill application" button to Resource Manager's Web UI | Minor | resourcemanager | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-11673](https://issues.apache.org/jira/browse/HADOOP-11673) | Use org.junit.Assume to skip tests instead of return | Minor | test | Akira Ajisaka | Brahma Reddy Battula |
+| [HADOOP-11692](https://issues.apache.org/jira/browse/HADOOP-11692) | Improve authentication failure WARN message to avoid user confusion | Major | ipc | Yongjun Zhang | Yongjun Zhang |
+| [HADOOP-11226](https://issues.apache.org/jira/browse/HADOOP-11226) | Add a configuration to set ipc.Client's traffic class with IPTOS\_LOWDELAY\|IPTOS\_RELIABILITY | Major | ipc | Gopal V | Gopal V |
+| [YARN-2280](https://issues.apache.org/jira/browse/YARN-2280) | Resource manager web service fields are not accessible | Trivial | resourcemanager | Krisztian Horvath | Krisztian Horvath |
+| [HADOOP-11711](https://issues.apache.org/jira/browse/HADOOP-11711) | Provide a default value for AES/CTR/NoPadding CryptoCodec classes | Minor | . | Andrew Wang | Andrew Wang |
+| [MAPREDUCE-4414](https://issues.apache.org/jira/browse/MAPREDUCE-4414) | Add main methods to JobConf and YarnConfiguration, for debug purposes | Major | client | Harsh J | Plamen Jeliazkov |
+| [MAPREDUCE-6105](https://issues.apache.org/jira/browse/MAPREDUCE-6105) | Inconsistent configuration in property mapreduce.reduce.shuffle.merge.percent | Trivial | . | Dongwook Kwon | Ray Chiang |
+| [HDFS-2360](https://issues.apache.org/jira/browse/HDFS-2360) | Ugly stacktrace when quota exceeds | Minor | hdfs-client | Rajit Saha | Harsh J |
+| [MAPREDUCE-6100](https://issues.apache.org/jira/browse/MAPREDUCE-6100) | replace "mapreduce.job.credentials.binary" with MRJobConfig.MAPREDUCE\_JOB\_CREDENTIALS\_BINARY for better readability. | Trivial | mrv2 | zhihai xu | zhihai xu |
+| [MAPREDUCE-5755](https://issues.apache.org/jira/browse/MAPREDUCE-5755) | MapTask.MapOutputBuffer#compare/swap should have @Override annotation | Trivial | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [MAPREDUCE-4653](https://issues.apache.org/jira/browse/MAPREDUCE-4653) | TestRandomAlgorithm has an unused "import" statement | Trivial | contrib/gridmix | Amir Sanjar | Amir Sanjar |
+| [HADOOP-11659](https://issues.apache.org/jira/browse/HADOOP-11659) | o.a.h.fs.FileSystem.Cache#remove should use a single hash map lookup | Minor | fs | Gera Shegalov | Brahma Reddy Battula |
+| [HADOOP-11709](https://issues.apache.org/jira/browse/HADOOP-11709) | Time.NANOSECONDS\_PER\_MILLISECOND - use class-level final constant instead of method variable | Trivial | . | Ajith S | Ajith S |
+| [HDFS-7835](https://issues.apache.org/jira/browse/HDFS-7835) | make initial sleeptime in locateFollowingBlock configurable for DFSClient. | Major | hdfs-client | zhihai xu | zhihai xu |
+| [HDFS-7829](https://issues.apache.org/jira/browse/HDFS-7829) | Code clean up for LocatedBlock | Minor | . | Jing Zhao | Takanobu Asanuma |
+| [MAPREDUCE-6282](https://issues.apache.org/jira/browse/MAPREDUCE-6282) | Reuse historyFileAbsolute.getFileSystem in CompletedJob#loadFullHistoryData for code optimization. | Trivial | jobhistoryserver | zhihai xu | zhihai xu |
+| [HADOOP-11447](https://issues.apache.org/jira/browse/HADOOP-11447) | Add a more meaningful toString method to SampleStat and MutableStat | Minor | metrics | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3350](https://issues.apache.org/jira/browse/YARN-3350) | YARN RackResolver spams logs with messages at info level | Major | . | Wilfred Spiegelenburg | Wilfred Spiegelenburg |
+| [MAPREDUCE-6239](https://issues.apache.org/jira/browse/MAPREDUCE-6239) | Consolidate TestJobConf classes in hadoop-mapreduce-client-jobclient and hadoop-mapreduce-client-core | Minor | client | Varun Saxena | Varun Saxena |
+| [MAPREDUCE-5190](https://issues.apache.org/jira/browse/MAPREDUCE-5190) | Unnecessary condition test in RandomSampler | Minor | mrv2 | Jingguo Yao | Jingguo Yao |
+| [MAPREDUCE-6287](https://issues.apache.org/jira/browse/MAPREDUCE-6287) | Deprecated methods in org.apache.hadoop.examples.Sort | Minor | examples | Chao Zhang | Chao Zhang |
+| [HADOOP-11737](https://issues.apache.org/jira/browse/HADOOP-11737) | mockito's version in hadoop-nfs\u2019 pom.xml shouldn't be specified | Minor | nfs | Kengo Seki | Kengo Seki |
+| [YARN-2868](https://issues.apache.org/jira/browse/YARN-2868) | FairScheduler: Metric for latency to allocate first container for an application | Major | . | Ray Chiang | Ray Chiang |
+| [HDFS-7875](https://issues.apache.org/jira/browse/HDFS-7875) | Improve log message when wrong value configured for dfs.datanode.failed.volumes.tolerated | Trivial | datanode | nijel | nijel |
+| [HDFS-7793](https://issues.apache.org/jira/browse/HDFS-7793) | Refactor DFSOutputStream separating DataStreamer out | Major | hdfs-client | Kai Zheng | Li Bo |
+| [HADOOP-11741](https://issues.apache.org/jira/browse/HADOOP-11741) | Add LOG.isDebugEnabled() guard for some LOG.debug() | Major | . | Walter Su | Walter Su |
+| [MAPREDUCE-579](https://issues.apache.org/jira/browse/MAPREDUCE-579) | Streaming "slowmatch" documentation | Trivial | contrib/streaming | Bo Adler | Harsh J |
+| [HDFS-7928](https://issues.apache.org/jira/browse/HDFS-7928) | Scanning blocks from disk during rolling upgrade startup takes a lot of time if disks are busy | Major | datanode | Rushabh S Shah | Rushabh S Shah |
+| [HADOOP-11524](https://issues.apache.org/jira/browse/HADOOP-11524) | hadoop\_do\_classpath\_subcommand throws a shellcheck warning | Minor | scripts | Allen Wittenauer | Chris Nauroth |
+| [HADOOP-11719](https://issues.apache.org/jira/browse/HADOOP-11719) | [Fsshell] Remove bin/hadoop reference from GenericOptionsParser default help text | Minor | scripts | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8004](https://issues.apache.org/jira/browse/HDFS-8004) | Use KeyProviderCryptoExtension#warmUpEncryptedKeys when creating an encryption zone | Trivial | encryption | Andrew Wang | Andrew Wang |
+| [HDFS-7890](https://issues.apache.org/jira/browse/HDFS-7890) | Improve information on Top users for metrics in RollingWindowsManager and lower log level | Major | . | J.Andreina | J.Andreina |
+| [HDFS-4396](https://issues.apache.org/jira/browse/HDFS-4396) | Add START\_MSG/SHUTDOWN\_MSG for ZKFC | Major | auto-failover, ha, tools | Liang Xie | Liang Xie |
+| [HADOOP-11660](https://issues.apache.org/jira/browse/HADOOP-11660) | Add support for hardware crc of HDFS checksums on ARM aarch64 architecture | Minor | native | Edward Nevill | Edward Nevill |
+| [HDFS-7645](https://issues.apache.org/jira/browse/HDFS-7645) | Rolling upgrade is restoring blocks from trash multiple times | Major | datanode | Nathan Roberts | Keisuke Ogiwara |
+| [HDFS-3918](https://issues.apache.org/jira/browse/HDFS-3918) | EditLogTailer shouldn't log WARN when other node is in standby mode | Major | ha | Todd Lipcon | Todd Lipcon |
+| [HDFS-7944](https://issues.apache.org/jira/browse/HDFS-7944) | Minor cleanup of BlockPoolManager#getAllNamenodeThreads | Minor | . | Arpit Agarwal | Arpit Agarwal |
+| [YARN-3258](https://issues.apache.org/jira/browse/YARN-3258) | FairScheduler: Need to add more logging to investigate allocations | Minor | fairscheduler | Anubhav Dhoot | Anubhav Dhoot |
+| [HDFS-7671](https://issues.apache.org/jira/browse/HDFS-7671) | hdfs user guide should point to the common rack awareness doc | Major | documentation | Allen Wittenauer | Kai Sasaki |
+| [YARN-3412](https://issues.apache.org/jira/browse/YARN-3412) | RM tests should use MockRM where possible | Major | resourcemanager, test | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3428](https://issues.apache.org/jira/browse/YARN-3428) | Debug log resources to be localized for a container | Trivial | nodemanager | Karthik Kambatla | Karthik Kambatla |
+| [YARN-3424](https://issues.apache.org/jira/browse/YARN-3424) | Change logs for ContainerMonitorImpl's resourse monitoring from info to debug | Major | nodemanager | Anubhav Dhoot | Anubhav Dhoot |
+| [YARN-3248](https://issues.apache.org/jira/browse/YARN-3248) | Display count of nodes blacklisted by apps in the web UI | Major | capacityscheduler, resourcemanager | Varun Vasudev | Varun Vasudev |
+| [HDFS-7978](https://issues.apache.org/jira/browse/HDFS-7978) | Add LOG.isDebugEnabled() guard for some LOG.debug(..) | Major | . | Walter Su | Walter Su |
+| [HDFS-8008](https://issues.apache.org/jira/browse/HDFS-8008) | Support client-side back off when the datanodes are congested | Major | hdfs-client | Haohui Mai | Haohui Mai |
+| [HDFS-7888](https://issues.apache.org/jira/browse/HDFS-7888) | Change DataStreamer/DFSOutputStream/DFSPacket for convenience of subclassing | Minor | hdfs-client | Li Bo | Li Bo |
+| [HDFS-8035](https://issues.apache.org/jira/browse/HDFS-8035) | Move checkBlocksProperlyReplicated() in FSNamesystem to BlockManager | Minor | namenode | Haohui Mai | Haohui Mai |
+| [HADOOP-9805](https://issues.apache.org/jira/browse/HADOOP-9805) | Refactor RawLocalFileSystem#rename for improved testability. | Minor | fs, test | Chris Nauroth | Jean-Pierre Matsumoto |
+| [HADOOP-11785](https://issues.apache.org/jira/browse/HADOOP-11785) | Reduce number of listStatus operation in distcp buildListing() | Minor | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [HADOOP-11717](https://issues.apache.org/jira/browse/HADOOP-11717) | Add Redirecting WebSSO behavior with JWT Token in Hadoop Auth | Major | security | Larry McCay | Larry McCay |
+| [YARN-3294](https://issues.apache.org/jira/browse/YARN-3294) | Allow dumping of Capacity Scheduler debug logs via web UI for a fixed time period | Major | capacityscheduler | Varun Vasudev | Varun Vasudev |
+| [HDFS-8073](https://issues.apache.org/jira/browse/HDFS-8073) | Split BlockPlacementPolicyDefault.chooseTarget(..) so it can be easily overrided. | Trivial | namenode | Walter Su | Walter Su |
+| [HDFS-8076](https://issues.apache.org/jira/browse/HDFS-8076) | Code cleanup for DFSInputStream: use offset instead of LocatedBlock when possible | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-7979](https://issues.apache.org/jira/browse/HDFS-7979) | Initialize block report IDs with a random number | Minor | datanode | Andrew Wang | Andrew Wang |
+| [HDFS-8101](https://issues.apache.org/jira/browse/HDFS-8101) | DFSClient use of non-constant DFSConfigKeys pulls in WebHDFS classes at runtime | Minor | hdfs-client | Sean Busbey | Sean Busbey |
+| [YARN-3293](https://issues.apache.org/jira/browse/YARN-3293) | Track and display capacity scheduler health metrics in web UI | Major | capacityscheduler | Varun Vasudev | Varun Vasudev |
+| [MAPREDUCE-6291](https://issues.apache.org/jira/browse/MAPREDUCE-6291) | Correct mapred queue usage command | Minor | client | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6307](https://issues.apache.org/jira/browse/MAPREDUCE-6307) | Remove property mapreduce.tasktracker.taskmemorymanager.monitoringinterval | Minor | . | Akira Ajisaka | J.Andreina |
+| [YARN-3348](https://issues.apache.org/jira/browse/YARN-3348) | Add a 'yarn top' tool to help understand cluster usage | Major | resourcemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-3469](https://issues.apache.org/jira/browse/YARN-3469) | ZKRMStateStore: Avoid setting watches that are not required | Minor | . | Jun Gong | Jun Gong |
+| [HDFS-8117](https://issues.apache.org/jira/browse/HDFS-8117) | More accurate verification in SimulatedFSDataset: replace DEFAULT\_DATABYTE with patterned data | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-8144](https://issues.apache.org/jira/browse/HDFS-8144) | Split TestLazyPersistFiles into multiple tests | Major | test | Arpit Agarwal | Arpit Agarwal |
+| [YARN-3404](https://issues.apache.org/jira/browse/YARN-3404) | View the queue name to YARN Application page | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [YARN-3451](https://issues.apache.org/jira/browse/YARN-3451) | Add start time and Elapsed in ApplicationAttemptReport and display the same in RMAttemptBlock WebUI | Major | api, webapp | Rohith Sharma K S | Rohith Sharma K S |
+| [HDFS-7863](https://issues.apache.org/jira/browse/HDFS-7863) | Missing description of some methods and parameters in javadoc of FSDirDeleteOp | Minor | . | Yongjun Zhang | Brahma Reddy Battula |
+| [HDFS-8152](https://issues.apache.org/jira/browse/HDFS-8152) | Refactoring of lazy persist storage cases | Major | test | Arpit Agarwal | Arpit Agarwal |
+| [HDFS-8133](https://issues.apache.org/jira/browse/HDFS-8133) | Improve readability of deleted block check | Major | namenode | Daryn Sharp | Daryn Sharp |
+| [HADOOP-11812](https://issues.apache.org/jira/browse/HADOOP-11812) | Implement listLocatedStatus for ViewFileSystem to speed up split calculation | Blocker | fs | Gera Shegalov | Gera Shegalov |
+| [MAPREDUCE-6297](https://issues.apache.org/jira/browse/MAPREDUCE-6297) | Task Id of the failed task in diagnostics should link to the task page | Minor | jobhistoryserver | Siqi Li | Siqi Li |
+| [HADOOP-11827](https://issues.apache.org/jira/browse/HADOOP-11827) | Speed-up distcp buildListing() using threadpool | Major | tools/distcp | Zoran Dimitrijevic | Zoran Dimitrijevic |
+| [YARN-3494](https://issues.apache.org/jira/browse/YARN-3494) | Expose AM resource limit and usage in QueueMetrics | Major | . | Jian He | Rohith Sharma K S |
+| [YARN-3503](https://issues.apache.org/jira/browse/YARN-3503) | Expose disk utilization percentage and bad local and log dir counts on NM via JMX | Major | nodemanager | Varun Vasudev | Varun Vasudev |
+| [YARN-3410](https://issues.apache.org/jira/browse/YARN-3410) | YARN admin should be able to remove individual application records from RMStateStore | Critical | resourcemanager, yarn | Wangda Tan | Rohith Sharma K S |
+| [HDFS-8215](https://issues.apache.org/jira/browse/HDFS-8215) | Refactor NamenodeFsck#check method | Minor | namenode | Takanobu Asanuma | Takanobu Asanuma |
+| [YARN-3511](https://issues.apache.org/jira/browse/YARN-3511) | Add errors and warnings page to ATS | Major | timelineserver | Varun Vasudev | Varun Vasudev |
+| [HDFS-8176](https://issues.apache.org/jira/browse/HDFS-8176) | Record from/to snapshots in audit log for snapshot diff report | Minor | snapshots | J.Andreina | J.Andreina |
+| [YARN-3406](https://issues.apache.org/jira/browse/YARN-3406) | Display count of running containers in the RM's Web UI | Minor | . | Ryu Kobayashi | Ryu Kobayashi |
+| [HADOOP-11357](https://issues.apache.org/jira/browse/HADOOP-11357) | Print information of the build enviornment in test-patch.sh | Minor | scripts | Haohui Mai | Allen Wittenauer |
+| [HADOOP-11857](https://issues.apache.org/jira/browse/HADOOP-11857) | Fix CommandFormat#commandFormat java doc annotation | Trivial | . | Xiaoyu Yao | J.Andreina |
+| [MAPREDUCE-6057](https://issues.apache.org/jira/browse/MAPREDUCE-6057) | Remove obsolete entries from mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang |
+| [HDFS-8204](https://issues.apache.org/jira/browse/HDFS-8204) | Mover/Balancer should not schedule two replicas to the same DN | Minor | balancer & mover | Walter Su | Walter Su |
+| [HDFS-8280](https://issues.apache.org/jira/browse/HDFS-8280) | Code Cleanup in DFSInputStream | Minor | . | Jing Zhao | Jing Zhao |
+| [HDFS-8283](https://issues.apache.org/jira/browse/HDFS-8283) | DataStreamer cleanup and some minor improvement | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-5574](https://issues.apache.org/jira/browse/HDFS-5574) | Remove buffer copy in BlockReader.skip | Trivial | . | Binglin Chang | Binglin Chang |
+| [HDFS-7770](https://issues.apache.org/jira/browse/HDFS-7770) | Need document for storage type label of data node storage locations under dfs.datanode.data.dir | Major | documentation | Xiaoyu Yao | Xiaoyu Yao |
+| [HDFS-8200](https://issues.apache.org/jira/browse/HDFS-8200) | Refactor FSDirStatAndListingOp | Major | . | Haohui Mai | Haohui Mai |
+| [YARN-3363](https://issues.apache.org/jira/browse/YARN-3363) | add localization and container launch time to ContainerMetrics at NM to show these timing information for each active container. | Major | nodemanager | zhihai xu | zhihai xu |
+| [HDFS-7397](https://issues.apache.org/jira/browse/HDFS-7397) | Add more detail to the documentation for the conf key "dfs.client.read.shortcircuit.streams.cache.size" | Minor | hdfs-client | Tsz Wo Nicholas Sze | Brahma Reddy Battula |
+| [YARN-2980](https://issues.apache.org/jira/browse/YARN-2980) | Move health check script related functionality to hadoop-common | Blocker | . | Ming Ma | Varun Saxena |
+| [HADOOP-11911](https://issues.apache.org/jira/browse/HADOOP-11911) | test-patch should allow configuration of default branch | Minor | . | Sean Busbey | Sean Busbey |
+| [HDFS-7758](https://issues.apache.org/jira/browse/HDFS-7758) | Retire FsDatasetSpi#getVolumes() and use FsDatasetSpi#getVolumeRefs() instead | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6192](https://issues.apache.org/jira/browse/MAPREDUCE-6192) | Create unit test to automatically compare MR related classes and mapred-default.xml | Minor | . | Ray Chiang | Ray Chiang |
+| [HADOOP-11813](https://issues.apache.org/jira/browse/HADOOP-11813) | releasedocmaker.py should use today's date instead of unreleased | Minor | build | Allen Wittenauer | Darrell Taylor |
+| [YARN-3491](https://issues.apache.org/jira/browse/YARN-3491) | PublicLocalizer#addResource is too slow. | Critical | nodemanager | zhihai xu | zhihai xu |
+| [MAPREDUCE-6279](https://issues.apache.org/jira/browse/MAPREDUCE-6279) | AM should explicity exit JVM after all services have stopped | Major | . | Jason Lowe | Eric Payne |
+| [HDFS-8108](https://issues.apache.org/jira/browse/HDFS-8108) | Fsck should provide the info on mandatory option to be used along with "-blocks , -locations and -racks" | Trivial | documentation | J.Andreina | J.Andreina |
+| [HDFS-8207](https://issues.apache.org/jira/browse/HDFS-8207) | Improper log message when blockreport interval compared with initial delay | Minor | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6079](https://issues.apache.org/jira/browse/MAPREDUCE-6079) | Rename JobImpl#username to reporterUserName | Major | . | Tsuyoshi Ozawa | Tsuyoshi Ozawa |
+| [HDFS-8209](https://issues.apache.org/jira/browse/HDFS-8209) | Support different number of datanode directories in MiniDFSCluster. | Minor | test | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [YARN-3169](https://issues.apache.org/jira/browse/YARN-3169) | Drop YARN's overview document | Major | documentation | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-2784](https://issues.apache.org/jira/browse/YARN-2784) | Make POM project names consistent | Minor | build | Rohith Sharma K S | Rohith Sharma K S |
+| [MAPREDUCE-5981](https://issues.apache.org/jira/browse/MAPREDUCE-5981) | Log levels of certain MR logs can be changed to DEBUG | Major | . | Varun Saxena | Varun Saxena |
+| [HADOOP-6842](https://issues.apache.org/jira/browse/HADOOP-6842) | "hadoop fs -text" does not give a useful text representation of MapWritable objects | Major | io | Steven K. Wong | Akira Ajisaka |
+| [YARN-20](https://issues.apache.org/jira/browse/YARN-20) | More information for "yarn.resourcemanager.webapp.address" in yarn-default.xml | Trivial | documentation, resourcemanager | Nemon Lou | Bartosz \u0141ugowski |
+| [HDFS-5640](https://issues.apache.org/jira/browse/HDFS-5640) | Add snapshot methods to FileContext. | Major | hdfs-client, snapshots | Chris Nauroth | Rakesh R |
+| [HDFS-8284](https://issues.apache.org/jira/browse/HDFS-8284) | Update documentation about how to use HTrace with HDFS | Major | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-7433](https://issues.apache.org/jira/browse/HDFS-7433) | Optimize performance of DatanodeManager's node map | Critical | namenode | Daryn Sharp | Daryn Sharp |
+| [HADOOP-11590](https://issues.apache.org/jira/browse/HADOOP-11590) | Update sbin commands and documentation to use new --slaves option | Blocker | documentation, scripts | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-5248](https://issues.apache.org/jira/browse/MAPREDUCE-5248) | Let NNBenchWithoutMR specify the replication factor for its test | Minor | client, test | Erik Paulson | Erik Paulson |
+| [HADOOP-9737](https://issues.apache.org/jira/browse/HADOOP-9737) | JarFinder#getJar should delete the jar file upon destruction of the JVM | Major | util | Esteban Gutierrez | Jean-Baptiste Onofr� |
+| [YARN-1050](https://issues.apache.org/jira/browse/YARN-1050) | Document the Fair Scheduler REST API | Major | documentation, fairscheduler | Sandy Ryza | Kenji Kikushima |
+| [YARN-3271](https://issues.apache.org/jira/browse/YARN-3271) | FairScheduler: Move tests related to max-runnable-apps from TestFairScheduler to TestAppRunnability | Major | . | Karthik Kambatla | nijel |
+| [YARN-2206](https://issues.apache.org/jira/browse/YARN-2206) | Update document for applications REST API response examples | Minor | documentation | Kenji Kikushima | Brahma Reddy Battula |
+| [HDFS-6757](https://issues.apache.org/jira/browse/HDFS-6757) | Simplify lease manager with INodeID | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-8327](https://issues.apache.org/jira/browse/HDFS-8327) | Simplify quota calculations for snapshots and truncate | Major | . | Haohui Mai | Haohui Mai |
+| [YARN-1287](https://issues.apache.org/jira/browse/YARN-1287) | Consolidate MockClocks | Major | . | Sandy Ryza | Sebastian Wong |
+| [HDFS-8357](https://issues.apache.org/jira/browse/HDFS-8357) | Consolidate parameters of INode.CleanSubtree() into a parameter objects. | Major | . | Haohui Mai | Li Lu |
+| [HADOOP-11950](https://issues.apache.org/jira/browse/HADOOP-11950) | Add cli option to test-patch to set the project-under-test | Minor | . | Sean Busbey | Sean Busbey |
+| [HADOOP-11948](https://issues.apache.org/jira/browse/HADOOP-11948) | test-patch's issue matching regex should be configurable. | Major | . | Sean Busbey | Sean Busbey |
+| [MAPREDUCE-5465](https://issues.apache.org/jira/browse/MAPREDUCE-5465) | Tasks are often killed before they exit on their own | Major | mr-am, mrv2 | Radim Kolar | Ming Ma |
+| [YARN-3513](https://issues.apache.org/jira/browse/YARN-3513) | Remove unused variables in ContainersMonitorImpl and add debug log for overall resource usage by all containers | Trivial | nodemanager | Naganarasimha G R | Naganarasimha G R |
+| [HDFS-8255](https://issues.apache.org/jira/browse/HDFS-8255) | Rename getBlockReplication to getPreferredBlockReplication | Major | . | Zhe Zhang | Zhe Zhang |
+| [YARN-3613](https://issues.apache.org/jira/browse/YARN-3613) | TestContainerManagerSecurity should init and start Yarn cluster in setup instead of individual methods | Minor | test | Karthik Kambatla | nijel |
+| [HDFS-6184](https://issues.apache.org/jira/browse/HDFS-6184) | Capture NN's thread dump when it fails over | Major | namenode | Ming Ma | Ming Ma |
+| [YARN-3539](https://issues.apache.org/jira/browse/YARN-3539) | Compatibility doc to state that ATS v1 is a stable REST API | Major | documentation | Steve Loughran | Steve Loughran |
+| [HADOOP-9723](https://issues.apache.org/jira/browse/HADOOP-9723) | Improve error message when hadoop archive output path already exists | Trivial | . | Stephen Chu | Yongjun Zhang |
+| [HADOOP-10993](https://issues.apache.org/jira/browse/HADOOP-10993) | Dump java command line to \*.out file | Minor | . | Allen Wittenauer | Kengo Seki |
+| [HADOOP-11713](https://issues.apache.org/jira/browse/HADOOP-11713) | ViewFileSystem should support snapshot methods. | Major | fs | Chris Nauroth | Rakesh R |
+| [HDFS-8350](https://issues.apache.org/jira/browse/HDFS-8350) | Remove old webhdfs.xml and other outdated documentation stuff | Major | documentation | Akira Ajisaka | Brahma Reddy Battula |
+| [HADOOP-11960](https://issues.apache.org/jira/browse/HADOOP-11960) | Enable Azure-Storage Client Side logging. | Major | tools | Dushyanth | Dushyanth |
+| [HDFS-6888](https://issues.apache.org/jira/browse/HDFS-6888) | Allow selectively audit logging ops | Major | . | Kihwal Lee | Chen He |
+| [HDFS-8397](https://issues.apache.org/jira/browse/HDFS-8397) | Refactor the error handling code in DataStreamer | Minor | hdfs-client | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [HDFS-8394](https://issues.apache.org/jira/browse/HDFS-8394) | Move getAdditionalBlock() and related functionalities into a separate class | Major | . | Haohui Mai | Haohui Mai |
+| [HADOOP-11939](https://issues.apache.org/jira/browse/HADOOP-11939) | Deprecate DistCpV1 and Logalyzer | Major | . | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-4185](https://issues.apache.org/jira/browse/HDFS-4185) | Add a metric for number of active leases | Major | namenode | Kihwal Lee | Rakesh R |
+| [HADOOP-1540](https://issues.apache.org/jira/browse/HADOOP-1540) | Support file exclusion list in distcp | Minor | util | Senthil Subramanian | Rich Haase |
+| [HADOOP-11103](https://issues.apache.org/jira/browse/HADOOP-11103) | Clean up RemoteException | Trivial | ipc | Sean Busbey | Sean Busbey |
+| [HDFS-8131](https://issues.apache.org/jira/browse/HDFS-8131) | Implement a space balanced block placement policy | Minor | namenode | Liu Shaohui | Liu Shaohui |
+| [HADOOP-11970](https://issues.apache.org/jira/browse/HADOOP-11970) | Replace uses of ThreadLocal\<Random\> with JDK7 ThreadLocalRandom | Major | . | Sean Busbey | Sean Busbey |
+| [HADOOP-11995](https://issues.apache.org/jira/browse/HADOOP-11995) | Make jetty version configurable from the maven command line | Trivial | build, ld | sriharsha devineni | sriharsha devineni |
+| [HDFS-4383](https://issues.apache.org/jira/browse/HDFS-4383) | Document the lease limits | Minor | . | Eli Collins | Arshad Mohammad |
+| [HADOOP-10366](https://issues.apache.org/jira/browse/HADOOP-10366) | Add whitespaces between the classes for values in core-default.xml to fit better in browser | Minor | documentation | Chengwei Yang | Kanaka Kumar Avvaru |
+| [HADOOP-11594](https://issues.apache.org/jira/browse/HADOOP-11594) | Improve the readability of site index of documentation | Minor | documentation | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-160](https://issues.apache.org/jira/browse/YARN-160) | nodemanagers should obtain cpu/memory values from underlying OS | Major | nodemanager | Alejandro Abdelnur | Varun Vasudev |
+| [HADOOP-11242](https://issues.apache.org/jira/browse/HADOOP-11242) | Record the time of calling in tracing span of IPC server | Minor | ipc | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-5033](https://issues.apache.org/jira/browse/HDFS-5033) | Bad error message for fs -put/copyFromLocal if user doesn't have permissions to read the source | Minor | . | Karthik Kambatla | Darrell Taylor |
+| [YARN-3722](https://issues.apache.org/jira/browse/YARN-3722) | Merge multiple TestWebAppUtils into o.a.h.yarn.webapp.util.TestWebAppUtils | Minor | test | Masatake Iwasaki | Masatake Iwasaki |
+| [HADOOP-11142](https://issues.apache.org/jira/browse/HADOOP-11142) | Remove hdfs dfs reference from file system shell documentation | Minor | documentation | Jonathan Allen | Kengo Seki |
+| [HADOOP-11894](https://issues.apache.org/jira/browse/HADOOP-11894) | Bump the version of Apache HTrace to 3.2.0-incubating | Major | . | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-3489](https://issues.apache.org/jira/browse/YARN-3489) | RMServerUtils.validateResourceRequests should only obtain queue info once | Major | resourcemanager | Jason Lowe | Varun Saxena |
+| [HDFS-8443](https://issues.apache.org/jira/browse/HDFS-8443) | Document dfs.namenode.service.handler.count in hdfs-site.xml | Major | documentation | Akira Ajisaka | J.Andreina |
+| [YARN-3547](https://issues.apache.org/jira/browse/YARN-3547) | FairScheduler: Apps that have no resource demand should not participate scheduling | Major | fairscheduler | Xianyin Xin | Xianyin Xin |
+| [YARN-3713](https://issues.apache.org/jira/browse/YARN-3713) | Remove duplicate function call storeContainerDiagnostics in ContainerDiagnosticsUpdateTransition | Minor | nodemanager | zhihai xu | zhihai xu |
+| [HADOOP-12043](https://issues.apache.org/jira/browse/HADOOP-12043) | Display warning if defaultFs is not set when running fs commands. | Minor | fs | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [YARN-3467](https://issues.apache.org/jira/browse/YARN-3467) | Expose allocatedMB, allocatedVCores, and runningContainers metrics on running Applications in RM Web UI | Minor | webapp, yarn | Anthony Rojas | Anubhav Dhoot |
+| [HDFS-8490](https://issues.apache.org/jira/browse/HDFS-8490) | Typo in trace enabled log in ExceptionHandler of WebHDFS | Trivial | webhdfs | Jakob Homan | Archana T |
+| [HDFS-8521](https://issues.apache.org/jira/browse/HDFS-8521) | Add @VisibleForTesting annotation to {{BlockPoolSlice#selectReplicaToDelete}} | Trivial | . | Colin P. McCabe | Colin P. McCabe |
+| [MAPREDUCE-6174](https://issues.apache.org/jira/browse/MAPREDUCE-6174) | Combine common stream code into parent class for InMemoryMapOutput and OnDiskMapOutput. | Major | mrv2 | Eric Payne | Eric Payne |
+| [HDFS-8532](https://issues.apache.org/jira/browse/HDFS-8532) | Make the visibility of DFSOutputStream#streamer member variable to private | Trivial | . | Rakesh R | Rakesh R |
+| [HDFS-8535](https://issues.apache.org/jira/browse/HDFS-8535) | Clarify that dfs usage in dfsadmin -report output includes all block replicas. | Minor | documentation | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6383](https://issues.apache.org/jira/browse/MAPREDUCE-6383) | Pi job (QuasiMonteCarlo) should not try to read the results file if its job fails | Major | examples | Harsh J | Harsh J |
+| [YARN-3259](https://issues.apache.org/jira/browse/YARN-3259) | FairScheduler: Trigger fairShare updates on node events | Major | fairscheduler | Anubhav Dhoot | Anubhav Dhoot |
+| [HADOOP-12059](https://issues.apache.org/jira/browse/HADOOP-12059) | S3Credentials should support use of CredentialProvider | Major | fs/s3 | Sean Busbey | Sean Busbey |
+| [MAPREDUCE-6354](https://issues.apache.org/jira/browse/MAPREDUCE-6354) | ShuffleHandler should be able to log shuffle connections | Major | . | Chang Li | Chang Li |
+| [HADOOP-12055](https://issues.apache.org/jira/browse/HADOOP-12055) | Deprecate usage of NativeIO#link | Major | native | Andrew Wang | Andrew Wang |
+| [HDFS-8432](https://issues.apache.org/jira/browse/HDFS-8432) | Introduce a minimum compatible layout version to allow downgrade in more rolling upgrade use cases. | Major | namenode, rolling upgrades | Chris Nauroth | Chris Nauroth |
+| [HDFS-8116](https://issues.apache.org/jira/browse/HDFS-8116) | Cleanup uncessary if LOG.isDebugEnabled() from RollingWindowManager | Trivial | namenode | Xiaoyu Yao | Brahma Reddy Battula |
+| [YARN-2716](https://issues.apache.org/jira/browse/YARN-2716) | Refactor ZKRMStateStore retry code with Apache Curator | Major | . | Jian He | Karthik Kambatla |
+| [HDFS-8553](https://issues.apache.org/jira/browse/HDFS-8553) | Document hdfs class path options | Major | documentation | Xiaoyu Yao | Brahma Reddy Battula |
+| [YARN-3786](https://issues.apache.org/jira/browse/YARN-3786) | Document yarn class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula |
+| [MAPREDUCE-6392](https://issues.apache.org/jira/browse/MAPREDUCE-6392) | Document mapred class path options | Major | documentation | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8549](https://issues.apache.org/jira/browse/HDFS-8549) | Abort the balancer if an upgrade is in progress | Major | balancer & mover | Andrew Wang | Andrew Wang |
+| [HDFS-8573](https://issues.apache.org/jira/browse/HDFS-8573) | Move creation of restartMeta file logic from BlockReceiver to ReplicaInPipeline | Major | datanode | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-11971](https://issues.apache.org/jira/browse/HADOOP-11971) | Move test utilities for tracing from hadoop-hdfs to hadoop-common | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki |
+| [HDFS-8361](https://issues.apache.org/jira/browse/HDFS-8361) | Choose SSD over DISK in block placement | Major | namenode | Tsz Wo Nicholas Sze | Tsz Wo Nicholas Sze |
+| [YARN-3789](https://issues.apache.org/jira/browse/YARN-3789) | Improve logs for LeafQueue#activateApplications() | Minor | resourcemanager | Bibin A Chundatt | Bibin A Chundatt |
+| [HDFS-8606](https://issues.apache.org/jira/browse/HDFS-8606) | Cleanup DFSOutputStream by removing unwanted changes | Minor | hdfs-client | Rakesh R | Rakesh R |
+| [YARN-3148](https://issues.apache.org/jira/browse/YARN-3148) | Allow CORS related headers to passthrough in WebAppProxyServlet | Major | . | Prakash Ramachandran | Varun Saxena |
+| [HDFS-8589](https://issues.apache.org/jira/browse/HDFS-8589) | Fix unused imports in BPServiceActor and BlockReportLeaseManager | Trivial | . | Colin P. McCabe | Colin P. McCabe |
+| [HADOOP-7139](https://issues.apache.org/jira/browse/HADOOP-7139) | Allow appending to existing SequenceFiles | Major | io | Stephen Rose | Kanaka Kumar Avvaru |
+| [HDFS-8605](https://issues.apache.org/jira/browse/HDFS-8605) | Merge Refactor of DFSOutputStream from HDFS-7285 branch | Major | . | Vinayakumar B | Vinayakumar B |
+| [MAPREDUCE-6395](https://issues.apache.org/jira/browse/MAPREDUCE-6395) | Improve the commit failure messages in MRAppMaster recovery | Major | applicationmaster | Gera Shegalov | Brahma Reddy Battula |
+| [HDFS-8582](https://issues.apache.org/jira/browse/HDFS-8582) | Support getting a list of reconfigurable config properties and do not generate spurious reconfig warnings | Minor | datanode, hdfs-client | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [MAPREDUCE-6316](https://issues.apache.org/jira/browse/MAPREDUCE-6316) | Task Attempt List entries should link to the task overview | Major | . | Siqi Li | Siqi Li |
+| [MAPREDUCE-6305](https://issues.apache.org/jira/browse/MAPREDUCE-6305) | AM/Task log page should be able to link back to the job | Major | . | Siqi Li | Siqi Li |
+| [YARN-3834](https://issues.apache.org/jira/browse/YARN-3834) | Scrub debug logging of tokens during resource localization. | Major | nodemanager | Chris Nauroth | Chris Nauroth |
+| [MAPREDUCE-6408](https://issues.apache.org/jira/browse/MAPREDUCE-6408) | Queue name and user name should be printed on the job page | Major | applicationmaster | Siqi Li | Siqi Li |
+| [HDFS-8639](https://issues.apache.org/jira/browse/HDFS-8639) | Option for HTTP port of NameNode by MiniDFSClusterManager | Minor | test | Kai Sasaki | Kai Sasaki |
+| [YARN-3360](https://issues.apache.org/jira/browse/YARN-3360) | Add JMX metrics to TimelineDataManager | Major | timelineserver | Jason Lowe | Jason Lowe |
+| [HADOOP-12049](https://issues.apache.org/jira/browse/HADOOP-12049) | Control http authentication cookie persistence via configuration | Major | security | Benoy Antony | Huizhi Lu |
+| [HDFS-8462](https://issues.apache.org/jira/browse/HDFS-8462) | Implement GETXATTRS and LISTXATTRS operations for WebImageViewer | Major | . | Akira Ajisaka | Jagadesh Kiran N |
+| [HDFS-8640](https://issues.apache.org/jira/browse/HDFS-8640) | Make reserved RBW space visible through JMX | Major | . | Kanaka Kumar Avvaru | Kanaka Kumar Avvaru |
+| [HDFS-8546](https://issues.apache.org/jira/browse/HDFS-8546) | Use try with resources in DataStorage and Storage | Minor | datanode | Andrew Wang | Andrew Wang |
+| [HADOOP-11203](https://issues.apache.org/jira/browse/HADOOP-11203) | Allow ditscp to accept bandwitdh in fraction MegaBytes | Major | tools/distcp | Raju Bairishetti | Raju Bairishetti |
+| [HADOOP-11807](https://issues.apache.org/jira/browse/HADOOP-11807) | add a lint mode to releasedocmaker | Minor | build, documentation, yetus | Allen Wittenauer | ramtin |
+| [HDFS-8653](https://issues.apache.org/jira/browse/HDFS-8653) | Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo | Major | . | Zhe Zhang | Zhe Zhang |
+| [HDFS-8659](https://issues.apache.org/jira/browse/HDFS-8659) | Block scanner INFO message is spamming logs | Major | datanode | Yongjun Zhang | Yongjun Zhang |
+| [HADOOP-12149](https://issues.apache.org/jira/browse/HADOOP-12149) | copy all of test-patch BINDIR prior to re-exec | Blocker | yetus | Allen Wittenauer | Allen Wittenauer |
+| [MAPREDUCE-6384](https://issues.apache.org/jira/browse/MAPREDUCE-6384) | Add the last reporting reducer info for too many fetch failure diagnostics | Major | . | Chang Li | Chang Li |
+| [HADOOP-12158](https://issues.apache.org/jira/browse/HADOOP-12158) | Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed | Trivial | test | Andrew Wang | Andrew Wang |
+| [HADOOP-12172](https://issues.apache.org/jira/browse/HADOOP-12172) | FsShell mkdir -p makes an unnecessary check for the existence of the parent. | Minor | fs | Chris Nauroth | Chris Nauroth |
+| [HDFS-8703](https://issues.apache.org/jira/browse/HDFS-8703) | Merge refactor of DFSInputStream from ErasureCoding branch | Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-8709](https://issues.apache.org/jira/browse/HDFS-8709) | Clarify automatic sync in FSEditLog#logEdit | Minor | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12045](https://issues.apache.org/jira/browse/HADOOP-12045) | Enable LocalFileSystem#setTimes to change atime | Minor | fs | Kazuho Fujii | Kazuho Fujii |
+| [HADOOP-12185](https://issues.apache.org/jira/browse/HADOOP-12185) | NetworkTopology is not efficient adding/getting/removing nodes | Major | . | Inigo Goiri | Inigo Goiri |
+| [HADOOP-12135](https://issues.apache.org/jira/browse/HADOOP-12135) | cleanup releasedocmaker | Major | yetus | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12195](https://issues.apache.org/jira/browse/HADOOP-12195) | Add annotation to package-info.java file to workaround MCOMPILER-205 | Trivial | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12193](https://issues.apache.org/jira/browse/HADOOP-12193) | Rename Touchz.java to Touch.java | Trivial | . | Andrew Wang | Andrew Wang |
+| [HDFS-8711](https://issues.apache.org/jira/browse/HDFS-8711) | setSpaceQuota command should print the available storage type when input storage type is wrong | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-8712](https://issues.apache.org/jira/browse/HDFS-8712) | Remove "public" and "abstract" modifiers in FsVolumeSpi and FsDatasetSpi | Trivial | . | Lei (Eddy) Xu | Lei (Eddy) Xu |
+| [HADOOP-12194](https://issues.apache.org/jira/browse/HADOOP-12194) | Support for incremental generation in the protoc plugin | Major | . | Andrew Wang | Andrew Wang |
+| [HADOOP-12180](https://issues.apache.org/jira/browse/HADOOP-12180) | Move ResourceCalculatorPlugin from YARN to Common | Major | util | Chris Douglas | Chris Douglas |
+| [HADOOP-12210](https://issues.apache.org/jira/browse/HADOOP-12210) | Collect network usage on the node | Major | . | Robert Grandl | Robert Grandl |
+| [YARN-3069](https://issues.apache.org/jira/browse/YARN-3069) | Document missing properties in yarn-default.xml | Major | documentation | Ray Chiang | Ray Chiang |
+| [YARN-3381](https://issues.apache.org/jira/browse/YARN-3381) | Fix typo InvalidStateTransitonException | Minor | api | Xiaoshuang LU | Brahma Reddy Battula |
+| [HADOOP-12211](https://issues.apache.org/jira/browse/HADOOP-12211) | Collect disks usages on the node | Major | . | Robert Grandl | Robert Grandl |
+| [HDFS-8722](https://issues.apache.org/jira/browse/HDFS-8722) | Optimize datanode writes for small writes and flushes | Critical | . | Kihwal Lee | Kihwal Lee |
+| [HADOOP-12232](https://issues.apache.org/jira/browse/HADOOP-12232) | Upgrade Tomcat dependency to 6.0.44. | Major | build | Chris Nauroth | Chris Nauroth |
+| [YARN-3170](https://issues.apache.org/jira/browse/YARN-3170) | YARN architecture document needs updating | Major | documentation | Allen Wittenauer | Brahma Reddy Battula |
+| [YARN-3174](https://issues.apache.org/jira/browse/YARN-3174) | Consolidate the NodeManager and NodeManagerRestart documentation into one | Major | documentation | Allen Wittenauer | Masatake Iwasaki |
+| [HDFS-7314](https://issues.apache.org/jira/browse/HDFS-7314) | When the DFSClient lease cannot be renewed, abort open-for-write files rather than the entire DFSClient | Major | . | Ming Ma | Ming Ma |
+| [HADOOP-10979](https://issues.apache.org/jira/browse/HADOOP-10979) | Auto-entries in hadoop\_usage | Minor | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-11893](https://issues.apache.org/jira/browse/HADOOP-11893) | Mark org.apache.hadoop.security.token.Token as @InterfaceAudience.Public | Major | security | Steve Loughran | Brahma Reddy Battula |
+| [HADOOP-12081](https://issues.apache.org/jira/browse/HADOOP-12081) | Fix UserGroupInformation.java to support 64-bit zLinux | Major | security | Adam Roberts | Akira Ajisaka |
+| [HADOOP-12214](https://issues.apache.org/jira/browse/HADOOP-12214) | Parse 'HadoopArchive' commandline using cli Options. | Minor | . | Vinayakumar B | Vinayakumar B |
+| [YARN-3261](https://issues.apache.org/jira/browse/YARN-3261) | rewrite resourcemanager restart doc to remove roadmap bits | Major | documentation | Allen Wittenauer | Gururaj Shetty |
+| [YARN-2921](https://issues.apache.org/jira/browse/YARN-2921) | Fix MockRM/MockAM#waitForState sleep too long | Major | test | Karthik Kambatla | Tsuyoshi Ozawa |
+| [HADOOP-12161](https://issues.apache.org/jira/browse/HADOOP-12161) | Add getStoragePolicy API to the FileSystem interface | Major | . | Arpit Agarwal | Brahma Reddy Battula |
+| [HADOOP-12189](https://issues.apache.org/jira/browse/HADOOP-12189) | Improve CallQueueManager#swapQueue to make queue elements drop nearly impossible. | Major | ipc, test | zhihai xu | zhihai xu |
+| [HADOOP-12009](https://issues.apache.org/jira/browse/HADOOP-12009) | Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus | Minor | documentation, fs, test | Jakob Homan | J.Andreina |
+| [HADOOP-12259](https://issues.apache.org/jira/browse/HADOOP-12259) | Utility to Dynamic port allocation | Major | test, util | Brahma Reddy Battula | Brahma Reddy Battula |
+| [HDFS-8735](https://issues.apache.org/jira/browse/HDFS-8735) | Inotify : All events classes should implement toString() API. | Major | hdfs-client | Surendra Singh Lilhore | Surendra Singh Lilhore |
+| [HDFS-7858](https://issues.apache.org/jira/browse/HDFS-7858) | Improve HA Namenode Failover detection on the client | Major | hdfs-client | Arun Suresh | Arun Suresh |
+| [HDFS-8180](https://issues.apache.org/jira/browse/HDFS-8180) | AbstractFileSystem Implementation for WebHdfs | Major | webhdfs | Santhosh G Nayak | Santhosh G Nayak |
+| [HDFS-8811](https://issues.apache.org/jira/browse/HDFS-8811) | Move BlockStoragePolicy name's constants from HdfsServerConstants.java to HdfsConstants.java | Major | . | Vinayakumar B | Vinayakumar B |
+| [HDFS-8822](https://issues.apache.org/jira/browse/HDFS-8822) | Add SSD storagepolicy tests in TestBlockStoragePolicy#testDefaultPolicies | Major | . | Vinayakumar B | Vinayakumar B |
+| [YARN-3950](https://issues.apache.org/jira/browse/YARN-3950) | Add unique YARN\_SHELL\_ID environment variable to DistributedShell | Major | applications/distributed-shell | Robert Kanter | Robert Kanter |
+| [YARN-2768](https://issues.apache.org/jira/browse/YARN-2768) | Avoid cloning Resource in FSAppAttempt#updateDemand | Minor | fairscheduler | Hong Zhiguo | Hong Zhiguo |
+| [HDFS-8816](https://issues.apache.org/jira/browse/HDFS-8816) | Improve visualization for the Datanode tab in the NN UI | Major | . | Haohui Mai | Haohui Mai |
+| [HDFS-8821](https://issues.apache.org/jira/browse/HDFS-8821) | Explain message "Operation category X is not supported in state standby" | Minor | . | Gautam Gopalakrishnan | Gautam Gopalakrishnan |
+| [HADOOP-12271](https://issues.apache.org/jira/browse/HADOOP-12271) | Hadoop Jar Error Should Be More Explanatory | Minor | . | Jesse Anderson | Josh Elser |
+| [HADOOP-12249](https://issues.apache.org/jira/browse/HADOOP-12249) | pull argument parsing into a function | Major | scripts | Allen Wittenauer | Allen Wittenauer |
+| [HADOOP-12183](https://issues.apache.org/jira/browse/HADOOP-12183) | Annotate the HTrace span created by FsShell with the command-line arguments passed by the user | Minor | tracing | Masatake Iwasaki | Masatake Iwasaki |
+| [YARN-3978](https://issues.apache.org/jira/browse/YARN-3978) | Configurably turn off the saving of container info in Generic AHS | Major | timelineserver, yarn | Eric Payne | Eric Payne |
+| [YARN-3965](https://issues.apache.org/jira/browse/YARN-3965) | Ad
<TRUNCATED>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/48] hadoop git commit: Addendum fix for HDFS-10817 to fix failure
of the added testFSReadLockLongHoldingReport in branch-2.
Posted by aw...@apache.org.
Addendum fix for HDFS-10817 to fix failure of the added testFSReadLockLongHoldingReport in branch-2.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6c600360
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6c600360
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6c600360
Branch: refs/heads/HADOOP-13341
Commit: 6c600360ca469d5fe0f017d681585db06c80c9cc
Parents: 6f4b0d3
Author: Zhe Zhang <zh...@apache.org>
Authored: Wed Aug 31 23:43:59 2016 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Wed Aug 31 23:43:59 2016 -0700
----------------------------------------------------------------------
.../hdfs/server/namenode/TestFSNamesystem.java | 50 +++-----------------
1 file changed, 6 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6c600360/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
index ccc8a6e..b85b6aba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSNamesystem.java
@@ -59,6 +59,7 @@ import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeoutException;
+import java.util.regex.Matcher;
import java.util.regex.Pattern;
public class TestFSNamesystem {
@@ -431,54 +432,15 @@ public class TestFSNamesystem {
t2.start();
t1.join();
t2.join();
+ // Look for the differentiating class names in the stack trace
+ String stackTracePatternString =
+ String.format("INFO.+%s(.+\n){4}\\Q%%s\\E\\.run", readLockLogStmt);
Pattern t1Pattern = Pattern.compile(
- String.format("\\Q%s\\E.+%s", t1.getName(), readLockLogStmt));
+ String.format(stackTracePatternString, t1.getClass().getName()));
assertTrue(t1Pattern.matcher(logs.getOutput()).find());
Pattern t2Pattern = Pattern.compile(
- String.format("\\Q%s\\E.+%s", t2.getName(), readLockLogStmt));
+ String.format(stackTracePatternString, t2.getClass().getName()));
assertFalse(t2Pattern.matcher(logs.getOutput()).find());
-
- // Spin up a bunch of threads all grabbing the lock at once; assign some
- // to go over threshold and some under. Check that they all log correctly.
- logs.clearOutput();
- final int threadCount = 50;
- List<Thread> threads = new ArrayList<>(threadCount);
- for (int i = 0; i < threadCount; i++) {
- threads.add(new Thread() {
- @Override
- public void run() {
- try {
- long sleepTime;
- if (this.getName().hashCode() % 2 == 0) {
- sleepTime = readLockReportingThreshold + 10;
- } else {
- sleepTime = readLockReportingThreshold / 2;
- }
- fsn.readLock();
- Thread.sleep(sleepTime);
- fsn.readUnlock();
- } catch (InterruptedException e) {
- fail("Interrupted during testing");
- }
- }
- });
- }
- for (Thread t : threads) {
- t.start();
- }
- for (Thread t : threads) {
- t.join();
- }
- for (Thread t : threads) {
- Pattern p = Pattern.compile(
- String.format("\\Q%s\\E.+%s", t.getName(), readLockLogStmt));
- boolean foundLog = p.matcher(logs.getOutput()).find();
- if (t.getName().hashCode() % 2 == 0) {
- assertTrue(foundLog);
- } else {
- assertFalse(foundLog);
- }
- }
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/48] hadoop git commit: YARN-5596. Fix failing unit test in
TestDockerContainerRuntime. Contributed by Sidharta Seethana.
Posted by aw...@apache.org.
YARN-5596. Fix failing unit test in TestDockerContainerRuntime. Contributed by Sidharta Seethana.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08f55ccb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08f55ccb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08f55ccb
Branch: refs/heads/HADOOP-13341
Commit: 08f55ccbb075b90a6e534f4ac0f9840f2e6890d5
Parents: 6c60036
Author: Varun Vasudev <vv...@apache.org>
Authored: Thu Sep 1 14:08:51 2016 +0530
Committer: Varun Vasudev <vv...@apache.org>
Committed: Thu Sep 1 14:08:51 2016 +0530
----------------------------------------------------------------------
.../runtime/DockerLinuxContainerRuntime.java | 6 ++++--
.../runtime/TestDockerContainerRuntime.java | 20 ++++++++++++++++----
2 files changed, 20 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f55ccb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index 33b8add..a3aff2f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -146,6 +146,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
public static final String ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS =
"YARN_CONTAINER_RUNTIME_DOCKER_LOCAL_RESOURCE_MOUNTS";
+ static final String CGROUPS_ROOT_DIRECTORY = "/sys/fs/cgroup";
+
private Configuration conf;
private DockerClient dockerClient;
private PrivilegedOperationExecutor privilegedOperationExecutor;
@@ -437,7 +439,6 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
LOCALIZED_RESOURCES);
@SuppressWarnings("unchecked")
List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
-
Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getStrings(
YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES,
YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
@@ -449,7 +450,8 @@ public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
.setContainerWorkDir(containerWorkDir.toString())
.setNetworkType(network)
.setCapabilities(capabilities)
- .addMountLocation("/sys/fs/cgroup", "/sys/fs/cgroup:ro", false);
+ .addMountLocation(CGROUPS_ROOT_DIRECTORY,
+ CGROUPS_ROOT_DIRECTORY + ":ro", false);
List<String> allDirs = new ArrayList<>(containerLocalDirs);
allDirs.addAll(filecacheDirs);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08f55ccb/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index a05ff46..3253394 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -257,6 +257,18 @@ public class TestDockerContainerRuntime {
return expectedCapabilitiesString.toString();
}
+ private String getExpectedCGroupsMountString() {
+ boolean cGroupsMountExists = new File(
+ DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY).exists();
+
+ if(cGroupsMountExists) {
+ return "-v " + DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY
+ + ":" + DockerLinuxContainerRuntime.CGROUPS_ROOT_DIRECTORY + ":ro ";
+ } else {
+ return "";
+ }
+ }
+
@Test
public void testDockerContainerLaunch()
throws ContainerExecutionException, PrivilegedOperationException,
@@ -276,7 +288,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=host ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ")
.append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ")
@@ -382,7 +394,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + allowedNetwork + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -437,7 +449,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork1 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
@@ -474,7 +486,7 @@ public class TestDockerContainerRuntime {
.append("--workdir=%3$s ")
.append("--net=" + customNetwork2 + " ")
.append(getExpectedTestCapabilitiesArgumentString())
- .append("-v /sys/fs/cgroup:/sys/fs/cgroup:ro ")
+ .append(getExpectedCGroupsMountString())
.append("-v %4$s:%4$s ").append("-v %5$s:%5$s ")
.append("-v %6$s:%6$s ").append("-v %7$s:%7$s ")
.append("-v %8$s:%8$s ").append("%9$s ")
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/48] hadoop git commit: HDFS-9781. FsDatasetImpl#getBlockReports
can occasionally throw NullPointerException. Contributed by Manoj
Govindassamy.
Posted by aw...@apache.org.
HDFS-9781. FsDatasetImpl#getBlockReports can occasionally throw NullPointerException. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/07650bc3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/07650bc3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/07650bc3
Branch: refs/heads/HADOOP-13341
Commit: 07650bc37a3c78ecc6566d813778d0954d0b06b0
Parents: f6ea9be
Author: Xiao Chen <xi...@apache.org>
Authored: Fri Sep 2 15:26:20 2016 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Fri Sep 2 15:33:11 2016 -0700
----------------------------------------------------------------------
.../datanode/fsdataset/impl/FsDatasetImpl.java | 21 +++++++--
.../fsdataset/impl/TestFsDatasetImpl.java | 49 ++++++++++++++++----
2 files changed, 56 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07650bc3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 129024b..e0d2baf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -1827,13 +1827,24 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
Map<String, BlockListAsLongs.Builder> builders =
new HashMap<String, BlockListAsLongs.Builder>();
- List<FsVolumeImpl> curVolumes = volumes.getVolumes();
- for (FsVolumeSpi v : curVolumes) {
- builders.put(v.getStorageID(), BlockListAsLongs.builder(maxDataLength));
- }
-
+ List<FsVolumeImpl> curVolumes = null;
try (AutoCloseableLock lock = datasetLock.acquire()) {
+ curVolumes = volumes.getVolumes();
+ for (FsVolumeSpi v : curVolumes) {
+ builders.put(v.getStorageID(), BlockListAsLongs.builder(maxDataLength));
+ }
+
+ Set<String> missingVolumesReported = new HashSet<>();
for (ReplicaInfo b : volumeMap.replicas(bpid)) {
+ String volStorageID = b.getVolume().getStorageID();
+ if (!builders.containsKey(volStorageID)) {
+ if (!missingVolumesReported.contains(volStorageID)) {
+ LOG.warn("Storage volume: " + volStorageID + " missing for the"
+ + " replica block: " + b + ". Probably being removed!");
+ missingVolumesReported.add(volStorageID);
+ }
+ continue;
+ }
switch(b.getState()) {
case FINALIZED:
case RBW:
http://git-wip-us.apache.org/repos/asf/hadoop/blob/07650bc3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index e73a612..b946803 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -590,8 +590,15 @@ public class TestFsDatasetImpl {
final ExtendedBlock eb = new ExtendedBlock(BLOCK_POOL_IDS[0], 0);
final CountDownLatch startFinalizeLatch = new CountDownLatch(1);
final CountDownLatch brReceivedLatch = new CountDownLatch(1);
+ final CountDownLatch volRemovedLatch = new CountDownLatch(1);
class BlockReportThread extends Thread {
public void run() {
+ // Lets wait for the volume remove process to start
+ try {
+ volRemovedLatch.await();
+ } catch (Exception e) {
+ LOG.info("Unexpected exception when waiting for vol removal:", e);
+ }
LOG.info("Getting block report");
dataset.getBlockReports(eb.getBlockPoolId());
LOG.info("Successfully received block report");
@@ -599,18 +606,27 @@ public class TestFsDatasetImpl {
}
}
- final BlockReportThread brt = new BlockReportThread();
class ResponderThread extends Thread {
public void run() {
try (ReplicaHandler replica = dataset
.createRbw(StorageType.DEFAULT, eb, false)) {
- LOG.info("createRbw finished");
+ LOG.info("CreateRbw finished");
startFinalizeLatch.countDown();
- // Slow down while we're holding the reference to the volume
- Thread.sleep(1000);
+ // Slow down while we're holding the reference to the volume.
+ // As we finalize a block, the volume is removed in parallel.
+ // Ignore any interrupts coming out of volume shutdown.
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException ie) {
+ LOG.info("Ignoring ", ie);
+ }
+
+ // Lets wait for the other thread finish getting block report
+ brReceivedLatch.await();
+
dataset.finalizeBlock(eb);
- LOG.info("finalizeBlock finished");
+ LOG.info("FinalizeBlock finished");
} catch (Exception e) {
LOG.warn("Exception caught. This should not affect the test", e);
}
@@ -621,13 +637,28 @@ public class TestFsDatasetImpl {
res.start();
startFinalizeLatch.await();
+ // Verify if block report can be received
+ // when volume is being removed
+ final BlockReportThread brt = new BlockReportThread();
+ brt.start();
+
Set<File> volumesToRemove = new HashSet<>();
volumesToRemove.add(
StorageLocation.parse(dataset.getVolume(eb).getBasePath()).getFile());
- LOG.info("Removing volume " + volumesToRemove);
- // Verify block report can be received during this
- brt.start();
- dataset.removeVolumes(volumesToRemove, true);
+ /**
+ * TODO: {@link FsDatasetImpl#removeVolumes(Set, boolean)} is throwing
+ * IllegalMonitorStateException when there is a parallel reader/writer
+ * to the volume. Remove below try/catch block after fixing HDFS-10830.
+ */
+ try {
+ LOG.info("Removing volume " + volumesToRemove);
+ dataset.removeVolumes(volumesToRemove, true);
+ } catch (Exception e) {
+ LOG.info("Unexpected issue while removing volume: ", e);
+ } finally {
+ volRemovedLatch.countDown();
+ }
+
LOG.info("Volumes removed");
brReceivedLatch.await();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/48] hadoop git commit: YARN-5566. Client-side NM graceful decom
is not triggered when jobs finish. (Robert Kanter via kasha)
Posted by aw...@apache.org.
YARN-5566. Client-side NM graceful decom is not triggered when jobs finish. (Robert Kanter via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/74f4bae4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/74f4bae4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/74f4bae4
Branch: refs/heads/HADOOP-13341
Commit: 74f4bae45597f4794e99e33309130ddff647b21f
Parents: 76cd81f
Author: Karthik Kambatla <ka...@cloudera.com>
Authored: Thu Sep 1 14:44:01 2016 -0700
Committer: Karthik Kambatla <ka...@cloudera.com>
Committed: Thu Sep 1 14:44:01 2016 -0700
----------------------------------------------------------------------
.../resourcemanager/rmnode/RMNodeImpl.java | 13 +++++++--
.../resourcemanager/TestRMNodeTransitions.java | 29 ++++++++++++++------
.../TestResourceTrackerService.java | 4 ++-
3 files changed, 34 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74f4bae4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index d1ccecb..375b4cf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -1170,12 +1170,21 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
NodeState initialState = rmNode.getState();
boolean isNodeDecommissioning =
initialState.equals(NodeState.DECOMMISSIONING);
+ if (isNodeDecommissioning) {
+ List<ApplicationId> keepAliveApps = statusEvent.getKeepAliveAppIds();
+ if (rmNode.runningApplications.isEmpty() &&
+ (keepAliveApps == null || keepAliveApps.isEmpty())) {
+ RMNodeImpl.deactivateNode(rmNode, NodeState.DECOMMISSIONED);
+ return NodeState.DECOMMISSIONED;
+ }
+ }
+
if (!remoteNodeHealthStatus.getIsNodeHealthy()) {
LOG.info("Node " + rmNode.nodeId +
" reported UNHEALTHY with details: " +
remoteNodeHealthStatus.getHealthReport());
// if a node in decommissioning receives an unhealthy report,
- // it will keep decommissioning.
+ // it will stay in decommissioning.
if (isNodeDecommissioning) {
return NodeState.DECOMMISSIONING;
} else {
@@ -1349,7 +1358,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
+ " is the first container get launched for application "
+ containerAppId);
}
- runningApplications.add(containerAppId);
+ handleRunningAppOnNode(this, context, containerAppId, nodeId);
}
// Process running containers
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74f4bae4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
index e82b93c..6038b31 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.yarn.event.InlineDispatcher;
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.records.NodeHealthStatus;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer
.AllocationExpirationInfo;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.ContainerAllocationExpirer;
@@ -73,6 +74,7 @@ import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
@@ -279,16 +281,17 @@ public class TestRMNodeTransitions {
NodeId nodeId = BuilderUtils.newNodeId("localhost:1", 1);
RMNodeImpl node2 = new RMNodeImpl(nodeId, rmContext, null, 0, 0, null, null, null);
node2.handle(new RMNodeStartedEvent(null, null, null));
-
+
+ ApplicationId app0 = BuilderUtils.newApplicationId(0, 0);
+ ApplicationId app1 = BuilderUtils.newApplicationId(1, 1);
ContainerId completedContainerIdFromNode1 = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(0, 0), 0), 0);
+ BuilderUtils.newApplicationAttemptId(app0, 0), 0);
ContainerId completedContainerIdFromNode2_1 = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(1, 1), 1), 1);
+ BuilderUtils.newApplicationAttemptId(app1, 1), 1);
ContainerId completedContainerIdFromNode2_2 = BuilderUtils.newContainerId(
- BuilderUtils.newApplicationAttemptId(
- BuilderUtils.newApplicationId(1, 1), 1), 2);
+ BuilderUtils.newApplicationAttemptId(app1, 1), 2);
+ rmContext.getRMApps().put(app0, Mockito.mock(RMApp.class));
+ rmContext.getRMApps().put(app1, Mockito.mock(RMApp.class));
RMNodeStatusEvent statusEventFromNode1 = getMockRMNodeStatusEvent(null);
RMNodeStatusEvent statusEventFromNode2_1 = getMockRMNodeStatusEvent(null);
@@ -652,6 +655,7 @@ public class TestRMNodeTransitions {
NodeId nodeId = node.getNodeID();
ApplicationId runningAppId = BuilderUtils.newApplicationId(0, 1);
+ rmContext.getRMApps().put(runningAppId, Mockito.mock(RMApp.class));
// Create a running container
ContainerId runningContainerId = BuilderUtils.newContainerId(
BuilderUtils.newApplicationAttemptId(
@@ -919,16 +923,22 @@ public class TestRMNodeTransitions {
}
// Test unhealthy report on a decommissioning node will make it
- // keep decommissioning.
+ // keep decommissioning as long as there's a running or keep alive app.
+ // Otherwise, it will go to decommissioned
@Test
public void testDecommissioningUnhealthy() {
RMNodeImpl node = getDecommissioningNode();
NodeHealthStatus status = NodeHealthStatus.newInstance(false, "sick",
System.currentTimeMillis());
+ List<ApplicationId> keepAliveApps = new ArrayList<>();
+ keepAliveApps.add(BuilderUtils.newApplicationId(1, 1));
NodeStatus nodeStatus = NodeStatus.newInstance(node.getNodeID(), 0,
- new ArrayList<ContainerStatus>(), null, status, null, null, null);
+ null, keepAliveApps, status, null, null, null);
node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
Assert.assertEquals(NodeState.DECOMMISSIONING, node.getState());
+ nodeStatus.setKeepAliveApplications(null);
+ node.handle(new RMNodeStatusEvent(node.getNodeID(), nodeStatus, null));
+ Assert.assertEquals(NodeState.DECOMMISSIONED, node.getState());
}
@Test
@@ -951,6 +961,7 @@ public class TestRMNodeTransitions {
ApplicationId.newInstance(System.currentTimeMillis(), 1);
ApplicationAttemptId appAttemptId =
ApplicationAttemptId.newInstance(appId, 1);
+ rmContext.getRMApps().put(appId, Mockito.mock(RMApp.class));
ContainerId containerId1 = ContainerId.newContainerId(appAttemptId, 1L);
ContainerId containerId2 = ContainerId.newContainerId(appAttemptId, 2L);
AllocationExpirationInfo expirationInfo1 =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/74f4bae4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
index c2a20a1..916c942 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestResourceTrackerService.java
@@ -299,6 +299,8 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
RMApp app = rm.submitApp(2000);
MockAM am = MockRM.launchAndRegisterAM(app, rm, nm1);
ApplicationAttemptId aaid = app.getCurrentAppAttempt().getAppAttemptId();
+ nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+ nm3.nodeHeartbeat(true);
// Graceful decommission host1 and host3
writeToHostsFile("host1", "host3");
@@ -308,7 +310,7 @@ public class TestResourceTrackerService extends NodeLabelTestBase {
// host1 should be DECOMMISSIONING due to running containers.
// host3 should become DECOMMISSIONED.
- nm1.nodeHeartbeat(aaid, 2, ContainerState.RUNNING);
+ nm1.nodeHeartbeat(true);
nm3.nodeHeartbeat(true);
rm.waitForState(id1, NodeState.DECOMMISSIONING);
rm.waitForState(id3, NodeState.DECOMMISSIONED);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/48] hadoop git commit: HADOOP-13465. Design Server.Call to be
extensible for unified call queue. Contributed by Daryn Sharp.
Posted by aw...@apache.org.
HADOOP-13465. Design Server.Call to be extensible for unified call queue. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/76cd81f4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/76cd81f4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/76cd81f4
Branch: refs/heads/HADOOP-13341
Commit: 76cd81f4b656f0d40a4b2122e15f04ea53d8020b
Parents: 3069df7
Author: Kihwal Lee <ki...@apache.org>
Authored: Thu Sep 1 15:44:44 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Thu Sep 1 15:44:44 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Server.java | 336 +++++++++++--------
1 file changed, 191 insertions(+), 145 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/76cd81f4/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
index 4c73f6a..80b3303 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Server.java
@@ -354,10 +354,9 @@ public abstract class Server {
*/
public static InetAddress getRemoteIp() {
Call call = CurCall.get();
- return (call != null && call.connection != null) ? call.connection
- .getHostInetAddress() : null;
+ return (call != null ) ? call.getHostInetAddress() : null;
}
-
+
/**
* Returns the clientId from the current RPC request
*/
@@ -380,10 +379,9 @@ public abstract class Server {
*/
public static UserGroupInformation getRemoteUser() {
Call call = CurCall.get();
- return (call != null && call.connection != null) ? call.connection.user
- : null;
+ return (call != null) ? call.getRemoteUser() : null;
}
-
+
/** Return true if the invocation was through an RPC.
*/
public static boolean isRpcInvocation() {
@@ -483,7 +481,7 @@ public abstract class Server {
if ((rpcMetrics.getProcessingSampleCount() > minSampleSize) &&
(processingTime > threeSigma)) {
if(LOG.isWarnEnabled()) {
- String client = CurCall.get().connection.toString();
+ String client = CurCall.get().toString();
LOG.warn(
"Slow RPC : " + methodName + " took " + processingTime +
" milliseconds to process from client " + client);
@@ -657,62 +655,65 @@ public abstract class Server {
CommonConfigurationKeys.IPC_BACKOFF_ENABLE_DEFAULT);
}
- /** A call queued for handling. */
- public static class Call implements Schedulable {
- private final int callId; // the client's call id
- private final int retryCount; // the retry count of the call
- private final Writable rpcRequest; // Serialized Rpc request from client
- private final Connection connection; // connection to client
- private long timestamp; // time received when response is null
- // time served when response is not null
- private ByteBuffer rpcResponse; // the response for this call
+ /** A generic call queued for handling. */
+ public static class Call implements Schedulable,
+ PrivilegedExceptionAction<Void> {
+ final int callId; // the client's call id
+ final int retryCount; // the retry count of the call
+ long timestamp; // time received when response is null
+ // time served when response is not null
private AtomicInteger responseWaitCount = new AtomicInteger(1);
- private final RPC.RpcKind rpcKind;
- private final byte[] clientId;
+ final RPC.RpcKind rpcKind;
+ final byte[] clientId;
private final TraceScope traceScope; // the HTrace scope on the server side
private final CallerContext callerContext; // the call context
private int priorityLevel;
// the priority level assigned by scheduler, 0 by default
- private Call(Call call) {
- this(call.callId, call.retryCount, call.rpcRequest, call.connection,
- call.rpcKind, call.clientId, call.traceScope, call.callerContext);
+ Call(Call call) {
+ this(call.callId, call.retryCount, call.rpcKind, call.clientId,
+ call.traceScope, call.callerContext);
}
- public Call(int id, int retryCount, Writable param,
- Connection connection) {
- this(id, retryCount, param, connection, RPC.RpcKind.RPC_BUILTIN,
- RpcConstants.DUMMY_CLIENT_ID);
+ Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId) {
+ this(id, retryCount, kind, clientId, null, null);
}
- public Call(int id, int retryCount, Writable param, Connection connection,
+ @VisibleForTesting // primarily TestNamenodeRetryCache
+ public Call(int id, int retryCount, Void ignore1, Void ignore2,
RPC.RpcKind kind, byte[] clientId) {
- this(id, retryCount, param, connection, kind, clientId, null, null);
+ this(id, retryCount, kind, clientId, null, null);
}
- public Call(int id, int retryCount, Writable param, Connection connection,
- RPC.RpcKind kind, byte[] clientId, TraceScope traceScope,
- CallerContext callerContext) {
+ Call(int id, int retryCount, RPC.RpcKind kind, byte[] clientId,
+ TraceScope traceScope, CallerContext callerContext) {
this.callId = id;
this.retryCount = retryCount;
- this.rpcRequest = param;
- this.connection = connection;
this.timestamp = Time.now();
- this.rpcResponse = null;
this.rpcKind = kind;
this.clientId = clientId;
this.traceScope = traceScope;
this.callerContext = callerContext;
}
-
+
@Override
public String toString() {
- return rpcRequest + " from " + connection + " Call#" + callId + " Retry#"
- + retryCount;
+ return "Call#" + callId + " Retry#" + retryCount;
}
- public void setResponse(ByteBuffer response) {
- this.rpcResponse = response;
+ public Void run() throws Exception {
+ return null;
+ }
+ // should eventually be abstract but need to avoid breaking tests
+ public UserGroupInformation getRemoteUser() {
+ return null;
+ }
+ public InetAddress getHostInetAddress() {
+ return null;
+ }
+ public String getHostAddress() {
+ InetAddress addr = getHostInetAddress();
+ return (addr != null) ? addr.getHostAddress() : null;
}
/**
@@ -724,34 +725,36 @@ public abstract class Server {
*/
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void postponeResponse() {
+ public final void postponeResponse() {
int count = responseWaitCount.incrementAndGet();
assert count > 0 : "response has already been sent";
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void sendResponse() throws IOException {
+ public final void sendResponse() throws IOException {
int count = responseWaitCount.decrementAndGet();
assert count >= 0 : "response has already been sent";
if (count == 0) {
- connection.sendResponse(this);
+ doResponse(null);
}
}
@InterfaceStability.Unstable
@InterfaceAudience.LimitedPrivate({"HDFS"})
- public void abortResponse(Throwable t) throws IOException {
+ public final void abortResponse(Throwable t) throws IOException {
// don't send response if the call was already sent or aborted.
if (responseWaitCount.getAndSet(-1) > 0) {
- connection.abortResponse(this, t);
+ doResponse(t);
}
}
+ void doResponse(Throwable t) throws IOException {}
+
// For Schedulable
@Override
public UserGroupInformation getUserGroupInformation() {
- return connection.user;
+ return getRemoteUser();
}
@Override
@@ -764,6 +767,114 @@ public abstract class Server {
}
}
+ /** A RPC extended call queued for handling. */
+ private class RpcCall extends Call {
+ final Connection connection; // connection to client
+ final Writable rpcRequest; // Serialized Rpc request from client
+ ByteBuffer rpcResponse; // the response for this call
+
+ RpcCall(RpcCall call) {
+ super(call);
+ this.connection = call.connection;
+ this.rpcRequest = call.rpcRequest;
+ }
+
+ RpcCall(Connection connection, int id) {
+ this(connection, id, RpcConstants.INVALID_RETRY_COUNT);
+ }
+
+ RpcCall(Connection connection, int id, int retryCount) {
+ this(connection, id, retryCount, null,
+ RPC.RpcKind.RPC_BUILTIN, RpcConstants.DUMMY_CLIENT_ID,
+ null, null);
+ }
+
+ RpcCall(Connection connection, int id, int retryCount,
+ Writable param, RPC.RpcKind kind, byte[] clientId,
+ TraceScope traceScope, CallerContext context) {
+ super(id, retryCount, kind, clientId, traceScope, context);
+ this.connection = connection;
+ this.rpcRequest = param;
+ }
+
+ @Override
+ public UserGroupInformation getRemoteUser() {
+ return connection.user;
+ }
+
+ @Override
+ public InetAddress getHostInetAddress() {
+ return connection.getHostInetAddress();
+ }
+
+ @Override
+ public Void run() throws Exception {
+ if (!connection.channel.isOpen()) {
+ Server.LOG.info(Thread.currentThread().getName() + ": skipped " + this);
+ return null;
+ }
+ String errorClass = null;
+ String error = null;
+ RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
+ RpcErrorCodeProto detailedErr = null;
+ Writable value = null;
+
+ try {
+ value = call(
+ rpcKind, connection.protocolName, rpcRequest, timestamp);
+ } catch (Throwable e) {
+ if (e instanceof UndeclaredThrowableException) {
+ e = e.getCause();
+ }
+ logException(Server.LOG, e, this);
+ if (e instanceof RpcServerException) {
+ RpcServerException rse = ((RpcServerException)e);
+ returnStatus = rse.getRpcStatusProto();
+ detailedErr = rse.getRpcErrorCodeProto();
+ } else {
+ returnStatus = RpcStatusProto.ERROR;
+ detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
+ }
+ errorClass = e.getClass().getName();
+ error = StringUtils.stringifyException(e);
+ // Remove redundant error class name from the beginning of the
+ // stack trace
+ String exceptionHdr = errorClass + ": ";
+ if (error.startsWith(exceptionHdr)) {
+ error = error.substring(exceptionHdr.length());
+ }
+ }
+ setupResponse(this, returnStatus, detailedErr,
+ value, errorClass, error);
+ sendResponse();
+ return null;
+ }
+
+ void setResponse(ByteBuffer response) throws IOException {
+ this.rpcResponse = response;
+ }
+
+ @Override
+ void doResponse(Throwable t) throws IOException {
+ RpcCall call = this;
+ if (t != null) {
+ // clone the call to prevent a race with another thread stomping
+ // on the response while being sent. the original call is
+ // effectively discarded since the wait count won't hit zero
+ call = new RpcCall(this);
+ setupResponse(call,
+ RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
+ null, t.getClass().getName(), StringUtils.stringifyException(t));
+ }
+ connection.sendResponse(call);
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " " + rpcRequest + " from " + connection;
+ }
+ }
+
/** Listens on the socket. Creates jobs for the handler threads*/
private class Listener extends Thread {
@@ -1094,22 +1205,22 @@ public abstract class Server {
if(LOG.isDebugEnabled()) {
LOG.debug("Checking for old call responses.");
}
- ArrayList<Call> calls;
+ ArrayList<RpcCall> calls;
// get the list of channels from list of keys.
synchronized (writeSelector.keys()) {
- calls = new ArrayList<Call>(writeSelector.keys().size());
+ calls = new ArrayList<RpcCall>(writeSelector.keys().size());
iter = writeSelector.keys().iterator();
while (iter.hasNext()) {
SelectionKey key = iter.next();
- Call call = (Call)key.attachment();
+ RpcCall call = (RpcCall)key.attachment();
if (call != null && key.channel() == call.connection.channel) {
calls.add(call);
}
}
}
-
- for(Call call : calls) {
+
+ for (RpcCall call : calls) {
doPurge(call, now);
}
} catch (OutOfMemoryError e) {
@@ -1127,7 +1238,7 @@ public abstract class Server {
}
private void doAsyncWrite(SelectionKey key) throws IOException {
- Call call = (Call)key.attachment();
+ RpcCall call = (RpcCall)key.attachment();
if (call == null) {
return;
}
@@ -1155,10 +1266,10 @@ public abstract class Server {
// Remove calls that have been pending in the responseQueue
// for a long time.
//
- private void doPurge(Call call, long now) {
- LinkedList<Call> responseQueue = call.connection.responseQueue;
+ private void doPurge(RpcCall call, long now) {
+ LinkedList<RpcCall> responseQueue = call.connection.responseQueue;
synchronized (responseQueue) {
- Iterator<Call> iter = responseQueue.listIterator(0);
+ Iterator<RpcCall> iter = responseQueue.listIterator(0);
while (iter.hasNext()) {
call = iter.next();
if (now > call.timestamp + PURGE_INTERVAL) {
@@ -1172,12 +1283,12 @@ public abstract class Server {
// Processes one response. Returns true if there are no more pending
// data for this channel.
//
- private boolean processResponse(LinkedList<Call> responseQueue,
+ private boolean processResponse(LinkedList<RpcCall> responseQueue,
boolean inHandler) throws IOException {
boolean error = true;
boolean done = false; // there is more data for this channel.
int numElements = 0;
- Call call = null;
+ RpcCall call = null;
try {
synchronized (responseQueue) {
//
@@ -1260,7 +1371,7 @@ public abstract class Server {
//
// Enqueue a response from the application.
//
- void doRespond(Call call) throws IOException {
+ void doRespond(RpcCall call) throws IOException {
synchronized (call.connection.responseQueue) {
// must only wrap before adding to the responseQueue to prevent
// postponed responses from being encrypted and sent out of order.
@@ -1358,7 +1469,7 @@ public abstract class Server {
private SocketChannel channel;
private ByteBuffer data;
private ByteBuffer dataLengthBuffer;
- private LinkedList<Call> responseQueue;
+ private LinkedList<RpcCall> responseQueue;
// number of outstanding rpcs
private AtomicInteger rpcCount = new AtomicInteger();
private long lastContact;
@@ -1385,8 +1496,8 @@ public abstract class Server {
public UserGroupInformation attemptingUser = null; // user name before auth
// Fake 'call' for failed authorization response
- private final Call authFailedCall = new Call(AUTHORIZATION_FAILED_CALL_ID,
- RpcConstants.INVALID_RETRY_COUNT, null, this);
+ private final RpcCall authFailedCall =
+ new RpcCall(this, AUTHORIZATION_FAILED_CALL_ID);
private boolean sentNegotiate = false;
private boolean useWrap = false;
@@ -1409,7 +1520,7 @@ public abstract class Server {
this.hostAddress = addr.getHostAddress();
}
this.remotePort = socket.getPort();
- this.responseQueue = new LinkedList<Call>();
+ this.responseQueue = new LinkedList<RpcCall>();
if (socketSendBufferSize != 0) {
try {
socket.setSendBufferSize(socketSendBufferSize);
@@ -1704,8 +1815,7 @@ public abstract class Server {
}
private void doSaslReply(Message message) throws IOException {
- final Call saslCall = new Call(AuthProtocol.SASL.callId,
- RpcConstants.INVALID_RETRY_COUNT, null, this);
+ final RpcCall saslCall = new RpcCall(this, AuthProtocol.SASL.callId);
setupResponse(saslCall,
RpcStatusProto.SUCCESS, null,
RpcWritable.wrap(message), null, null);
@@ -1922,23 +2032,20 @@ public abstract class Server {
if (clientVersion >= 9) {
// Versions >>9 understand the normal response
- Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, -1);
setupResponse(fakeCall,
RpcStatusProto.FATAL, RpcErrorCodeProto.FATAL_VERSION_MISMATCH,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion >= 3) {
- Call fakeCall = new Call(-1, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, -1);
// Versions 3 to 8 use older response
setupResponseOldVersionFatal(buffer, fakeCall,
null, VersionMismatch.class.getName(), errMsg);
fakeCall.sendResponse();
} else if (clientVersion == 2) { // Hadoop 0.18.3
- Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null,
- this);
+ RpcCall fakeCall = new RpcCall(this, 0);
DataOutputStream out = new DataOutputStream(buffer);
out.writeInt(0); // call ID
out.writeBoolean(true); // error
@@ -1950,7 +2057,7 @@ public abstract class Server {
}
private void setupHttpRequestOnIpcPortResponse() throws IOException {
- Call fakeCall = new Call(0, RpcConstants.INVALID_RETRY_COUNT, null, this);
+ RpcCall fakeCall = new RpcCall(this, 0);
fakeCall.setResponse(ByteBuffer.wrap(
RECEIVED_HTTP_REQ_RESPONSE.getBytes(StandardCharsets.UTF_8)));
fakeCall.sendResponse();
@@ -2098,7 +2205,7 @@ public abstract class Server {
}
} catch (WrappedRpcServerException wrse) { // inform client of error
Throwable ioe = wrse.getCause();
- final Call call = new Call(callId, retry, null, this);
+ final RpcCall call = new RpcCall(this, callId, retry);
setupResponse(call,
RpcStatusProto.FATAL, wrse.getRpcErrorCodeProto(), null,
ioe.getClass().getName(), ioe.getMessage());
@@ -2198,8 +2305,9 @@ public abstract class Server {
.build();
}
- Call call = new Call(header.getCallId(), header.getRetryCount(),
- rpcRequest, this, ProtoUtil.convert(header.getRpcKind()),
+ RpcCall call = new RpcCall(this, header.getCallId(),
+ header.getRetryCount(), rpcRequest,
+ ProtoUtil.convert(header.getRpcKind()),
header.getClientId().toByteArray(), traceScope, callerContext);
// Save the priority level assignment by the scheduler
@@ -2323,21 +2431,10 @@ public abstract class Server {
}
}
- private void sendResponse(Call call) throws IOException {
+ private void sendResponse(RpcCall call) throws IOException {
responder.doRespond(call);
}
- private void abortResponse(Call call, Throwable t) throws IOException {
- // clone the call to prevent a race with the other thread stomping
- // on the response while being sent. the original call is
- // effectively discarded since the wait count won't hit zero
- call = new Call(call);
- setupResponse(call,
- RpcStatusProto.FATAL, RpcErrorCodeProto.ERROR_RPC_SERVER,
- null, t.getClass().getName(), StringUtils.stringifyException(t));
- call.sendResponse();
- }
-
/**
* Get service class for connection
* @return the serviceClass
@@ -2388,16 +2485,6 @@ public abstract class Server {
if (LOG.isDebugEnabled()) {
LOG.debug(Thread.currentThread().getName() + ": " + call + " for RpcKind " + call.rpcKind);
}
- if (!call.connection.channel.isOpen()) {
- LOG.info(Thread.currentThread().getName() + ": skipped " + call);
- continue;
- }
- String errorClass = null;
- String error = null;
- RpcStatusProto returnStatus = RpcStatusProto.SUCCESS;
- RpcErrorCodeProto detailedErr = null;
- Writable value = null;
-
CurCall.set(call);
if (call.traceScope != null) {
call.traceScope.reattach();
@@ -2406,53 +2493,11 @@ public abstract class Server {
}
// always update the current call context
CallerContext.setCurrent(call.callerContext);
-
- try {
- // Make the call as the user via Subject.doAs, thus associating
- // the call with the Subject
- if (call.connection.user == null) {
- value = call(call.rpcKind, call.connection.protocolName, call.rpcRequest,
- call.timestamp);
- } else {
- value =
- call.connection.user.doAs
- (new PrivilegedExceptionAction<Writable>() {
- @Override
- public Writable run() throws Exception {
- // make the call
- return call(call.rpcKind, call.connection.protocolName,
- call.rpcRequest, call.timestamp);
-
- }
- }
- );
- }
- } catch (Throwable e) {
- if (e instanceof UndeclaredThrowableException) {
- e = e.getCause();
- }
- logException(LOG, e, call);
- if (e instanceof RpcServerException) {
- RpcServerException rse = ((RpcServerException)e);
- returnStatus = rse.getRpcStatusProto();
- detailedErr = rse.getRpcErrorCodeProto();
- } else {
- returnStatus = RpcStatusProto.ERROR;
- detailedErr = RpcErrorCodeProto.ERROR_APPLICATION;
- }
- errorClass = e.getClass().getName();
- error = StringUtils.stringifyException(e);
- // Remove redundant error class name from the beginning of the stack trace
- String exceptionHdr = errorClass + ": ";
- if (error.startsWith(exceptionHdr)) {
- error = error.substring(exceptionHdr.length());
- }
- }
- CurCall.set(null);
- synchronized (call.connection.responseQueue) {
- setupResponse(call, returnStatus, detailedErr,
- value, errorClass, error);
- call.sendResponse();
+ UserGroupInformation remoteUser = call.getRemoteUser();
+ if (remoteUser != null) {
+ remoteUser.doAs(call);
+ } else {
+ call.run();
}
} catch (InterruptedException e) {
if (running) { // unexpected -- log it
@@ -2469,6 +2514,7 @@ public abstract class Server {
StringUtils.stringifyException(e));
}
} finally {
+ CurCall.set(null);
IOUtils.cleanup(LOG, traceScope);
}
}
@@ -2670,7 +2716,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponse(
- Call call, RpcStatusProto status, RpcErrorCodeProto erCode,
+ RpcCall call, RpcStatusProto status, RpcErrorCodeProto erCode,
Writable rv, String errorClass, String error)
throws IOException {
RpcResponseHeaderProto.Builder headerBuilder =
@@ -2704,7 +2750,7 @@ public abstract class Server {
}
}
- private void setupResponse(Call call,
+ private void setupResponse(RpcCall call,
RpcResponseHeaderProto header, Writable rv) throws IOException {
ResponseBuffer buf = responseBuffer.get().reset();
try {
@@ -2738,7 +2784,7 @@ public abstract class Server {
* @throws IOException
*/
private void setupResponseOldVersionFatal(ByteArrayOutputStream response,
- Call call,
+ RpcCall call,
Writable rv, String errorClass, String error)
throws IOException {
final int OLD_VERSION_FATAL_STATUS = -1;
@@ -2751,7 +2797,7 @@ public abstract class Server {
call.setResponse(ByteBuffer.wrap(response.toByteArray()));
}
- private void wrapWithSasl(Call call) throws IOException {
+ private void wrapWithSasl(RpcCall call) throws IOException {
if (call.connection.saslServer != null) {
byte[] token = call.rpcResponse.array();
// synchronization may be needed since there can be multiple Handler
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/48] hadoop git commit: HADOOP-13447. Refactor S3AFileSystem to
support introduction of separate metadata repository and tests. Contributed
by Chris Nauroth.
Posted by aw...@apache.org.
HADOOP-13447. Refactor S3AFileSystem to support introduction of separate metadata repository and tests. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d152557c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d152557c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d152557c
Branch: refs/heads/HADOOP-13341
Commit: d152557cf7f4d2288524c222fcbaf152bdc038b0
Parents: 39d1b1d
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 6 09:36:21 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Sep 6 09:36:21 2016 -0700
----------------------------------------------------------------------
hadoop-tools/hadoop-aws/pom.xml | 5 +
.../org/apache/hadoop/fs/s3a/Constants.java | 29 ++-
.../hadoop/fs/s3a/S3AFastOutputStream.java | 6 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 192 ++-------------
.../apache/hadoop/fs/s3a/S3AInputStream.java | 6 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 40 ++++
.../apache/hadoop/fs/s3a/S3ClientFactory.java | 232 +++++++++++++++++++
.../hadoop/fs/s3native/S3xLoginHelper.java | 3 +
.../hadoop/fs/s3a/AbstractS3AMockTest.java | 70 ++++++
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 10 +-
.../hadoop/fs/s3a/MockS3ClientFactory.java | 40 ++++
.../hadoop/fs/s3a/TestS3AGetFileStatus.java | 126 ++++++++++
12 files changed, 567 insertions(+), 192 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/pom.xml b/hadoop-tools/hadoop-aws/pom.xml
index 82a8581..13dcdf1 100644
--- a/hadoop-tools/hadoop-aws/pom.xml
+++ b/hadoop-tools/hadoop-aws/pom.xml
@@ -296,6 +296,11 @@
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>org.mockito</groupId>
+ <artifactId>mockito-all</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-jobclient</artifactId>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
index 6b8f9f5..64fd8e5 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/Constants.java
@@ -53,7 +53,8 @@ public final class Constants {
public static final int DEFAULT_MAXIMUM_CONNECTIONS = 15;
// connect to s3 over ssl?
- public static final String SECURE_CONNECTIONS = "fs.s3a.connection.ssl.enabled";
+ public static final String SECURE_CONNECTIONS =
+ "fs.s3a.connection.ssl.enabled";
public static final boolean DEFAULT_SECURE_CONNECTIONS = true;
//use a custom endpoint?
@@ -75,7 +76,8 @@ public final class Constants {
public static final int DEFAULT_MAX_ERROR_RETRIES = 20;
// seconds until we give up trying to establish a connection to s3
- public static final String ESTABLISH_TIMEOUT = "fs.s3a.connection.establish.timeout";
+ public static final String ESTABLISH_TIMEOUT =
+ "fs.s3a.connection.establish.timeout";
public static final int DEFAULT_ESTABLISH_TIMEOUT = 50000;
// seconds until we give up on a connection to s3
@@ -111,11 +113,13 @@ public final class Constants {
public static final long DEFAULT_MULTIPART_SIZE = 104857600; // 100 MB
// minimum size in bytes before we start a multipart uploads or copy
- public static final String MIN_MULTIPART_THRESHOLD = "fs.s3a.multipart.threshold";
+ public static final String MIN_MULTIPART_THRESHOLD =
+ "fs.s3a.multipart.threshold";
public static final long DEFAULT_MIN_MULTIPART_THRESHOLD = Integer.MAX_VALUE;
//enable multiobject-delete calls?
- public static final String ENABLE_MULTI_DELETE = "fs.s3a.multiobjectdelete.enable";
+ public static final String ENABLE_MULTI_DELETE =
+ "fs.s3a.multiobjectdelete.enable";
// comma separated list of directories
public static final String BUFFER_DIR = "fs.s3a.buffer.dir";
@@ -134,11 +138,13 @@ public final class Constants {
public static final String DEFAULT_CANNED_ACL = "";
// should we try to purge old multipart uploads when starting up
- public static final String PURGE_EXISTING_MULTIPART = "fs.s3a.multipart.purge";
+ public static final String PURGE_EXISTING_MULTIPART =
+ "fs.s3a.multipart.purge";
public static final boolean DEFAULT_PURGE_EXISTING_MULTIPART = false;
// purge any multipart uploads older than this number of seconds
- public static final String PURGE_EXISTING_MULTIPART_AGE = "fs.s3a.multipart.purge.age";
+ public static final String PURGE_EXISTING_MULTIPART_AGE =
+ "fs.s3a.multipart.purge.age";
public static final long DEFAULT_PURGE_EXISTING_MULTIPART_AGE = 14400;
// s3 server-side encryption
@@ -198,4 +204,15 @@ public final class Constants {
*/
@InterfaceStability.Unstable
public static final String INPUT_FADV_RANDOM = "random";
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static final String S3_CLIENT_FACTORY_IMPL =
+ "fs.s3a.s3.client.factory.impl";
+
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public static final Class<? extends S3ClientFactory>
+ DEFAULT_S3_CLIENT_FACTORY_IMPL =
+ S3ClientFactory.DefaultS3ClientFactory.class;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
index 5509d36..c25d0fb 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFastOutputStream.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
import com.amazonaws.event.ProgressEvent;
import com.amazonaws.event.ProgressListener;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
@@ -71,7 +71,7 @@ public class S3AFastOutputStream extends OutputStream {
private static final Logger LOG = S3AFileSystem.LOG;
private final String key;
private final String bucket;
- private final AmazonS3Client client;
+ private final AmazonS3 client;
private final int partSize;
private final int multiPartThreshold;
private final S3AFileSystem fs;
@@ -102,7 +102,7 @@ public class S3AFastOutputStream extends OutputStream {
* @param threadPoolExecutor thread factory
* @throws IOException on any problem
*/
- public S3AFastOutputStream(AmazonS3Client client,
+ public S3AFastOutputStream(AmazonS3 client,
S3AFileSystem fs,
String bucket,
String key,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
index f8c65da..6a6c26b 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java
@@ -34,11 +34,7 @@ import java.util.concurrent.TimeUnit;
import com.amazonaws.AmazonClientException;
import com.amazonaws.AmazonServiceException;
-import com.amazonaws.ClientConfiguration;
-import com.amazonaws.Protocol;
-import com.amazonaws.auth.AWSCredentialsProvider;
-import com.amazonaws.services.s3.AmazonS3Client;
-import com.amazonaws.services.s3.S3ClientOptions;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.DeleteObjectsRequest;
@@ -57,7 +53,6 @@ import com.amazonaws.services.s3.transfer.Upload;
import com.amazonaws.event.ProgressListener;
import com.amazonaws.event.ProgressEvent;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -78,7 +73,7 @@ import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.s3native.S3xLoginHelper;
import org.apache.hadoop.util.Progressable;
-import org.apache.hadoop.util.VersionInfo;
+import org.apache.hadoop.util.ReflectionUtils;
import static org.apache.hadoop.fs.s3a.Constants.*;
import static org.apache.hadoop.fs.s3a.Listing.ACCEPT_ALL;
@@ -110,7 +105,7 @@ public class S3AFileSystem extends FileSystem {
public static final int DEFAULT_BLOCKSIZE = 32 * 1024 * 1024;
private URI uri;
private Path workingDir;
- private AmazonS3Client s3;
+ private AmazonS3 s3;
private String bucket;
private int maxKeys;
private Listing listing;
@@ -147,37 +142,11 @@ public class S3AFileSystem extends FileSystem {
bucket = name.getHost();
- AWSCredentialsProvider credentials =
- createAWSCredentialProviderSet(name, conf, uri);
-
- ClientConfiguration awsConf = new ClientConfiguration();
- awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
- DEFAULT_MAXIMUM_CONNECTIONS, 1));
- boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
- DEFAULT_SECURE_CONNECTIONS);
- awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
- awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
- DEFAULT_MAX_ERROR_RETRIES, 0));
- awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
- DEFAULT_ESTABLISH_TIMEOUT, 0));
- awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
- DEFAULT_SOCKET_TIMEOUT, 0));
- int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
- DEFAULT_SOCKET_SEND_BUFFER, 2048);
- int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
- DEFAULT_SOCKET_RECV_BUFFER, 2048);
- awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
- String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
- if (!signerOverride.isEmpty()) {
- LOG.debug("Signer override = {}", signerOverride);
- awsConf.setSignerOverride(signerOverride);
- }
-
- initProxySupport(conf, awsConf, secureConnections);
-
- initUserAgent(conf, awsConf);
-
- initAmazonS3Client(conf, credentials, awsConf);
+ Class<? extends S3ClientFactory> s3ClientFactoryClass = conf.getClass(
+ S3_CLIENT_FACTORY_IMPL, DEFAULT_S3_CLIENT_FACTORY_IMPL,
+ S3ClientFactory.class);
+ s3 = ReflectionUtils.newInstance(s3ClientFactoryClass, conf)
+ .createS3Client(name, uri);
maxKeys = intOption(conf, MAX_PAGING_KEYS, DEFAULT_MAX_PAGING_KEYS, 1);
listing = new Listing(this);
@@ -266,50 +235,6 @@ public class S3AFileSystem extends FileSystem {
}
}
- void initProxySupport(Configuration conf, ClientConfiguration awsConf,
- boolean secureConnections) throws IllegalArgumentException {
- String proxyHost = conf.getTrimmed(PROXY_HOST, "");
- int proxyPort = conf.getInt(PROXY_PORT, -1);
- if (!proxyHost.isEmpty()) {
- awsConf.setProxyHost(proxyHost);
- if (proxyPort >= 0) {
- awsConf.setProxyPort(proxyPort);
- } else {
- if (secureConnections) {
- LOG.warn("Proxy host set without port. Using HTTPS default 443");
- awsConf.setProxyPort(443);
- } else {
- LOG.warn("Proxy host set without port. Using HTTP default 80");
- awsConf.setProxyPort(80);
- }
- }
- String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
- String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
- if ((proxyUsername == null) != (proxyPassword == null)) {
- String msg = "Proxy error: " + PROXY_USERNAME + " or " +
- PROXY_PASSWORD + " set without the other.";
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- awsConf.setProxyUsername(proxyUsername);
- awsConf.setProxyPassword(proxyPassword);
- awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
- awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
- if (LOG.isDebugEnabled()) {
- LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
- "domain {} as workstation {}", awsConf.getProxyHost(),
- awsConf.getProxyPort(),
- String.valueOf(awsConf.getProxyUsername()),
- awsConf.getProxyPassword(), awsConf.getProxyDomain(),
- awsConf.getProxyWorkstation());
- }
- } else if (proxyPort >= 0) {
- String msg = "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
- LOG.error(msg);
- throw new IllegalArgumentException(msg);
- }
- }
-
/**
* Get S3A Instrumentation. For test purposes.
* @return this instance's instrumentation.
@@ -318,53 +243,9 @@ public class S3AFileSystem extends FileSystem {
return instrumentation;
}
- /**
- * Initializes the User-Agent header to send in HTTP requests to the S3
- * back-end. We always include the Hadoop version number. The user also may
- * set an optional custom prefix to put in front of the Hadoop version number.
- * The AWS SDK interally appends its own information, which seems to include
- * the AWS SDK version, OS and JVM version.
- *
- * @param conf Hadoop configuration
- * @param awsConf AWS SDK configuration
- */
- private void initUserAgent(Configuration conf, ClientConfiguration awsConf) {
- String userAgent = "Hadoop " + VersionInfo.getVersion();
- String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
- if (!userAgentPrefix.isEmpty()) {
- userAgent = userAgentPrefix + ", " + userAgent;
- }
- LOG.debug("Using User-Agent: {}", userAgent);
- awsConf.setUserAgent(userAgent);
- }
-
- private void initAmazonS3Client(Configuration conf,
- AWSCredentialsProvider credentials, ClientConfiguration awsConf)
- throws IllegalArgumentException {
- s3 = new AmazonS3Client(credentials, awsConf);
- String endPoint = conf.getTrimmed(ENDPOINT, "");
- if (!endPoint.isEmpty()) {
- try {
- s3.setEndpoint(endPoint);
- } catch (IllegalArgumentException e) {
- String msg = "Incorrect endpoint: " + e.getMessage();
- LOG.error(msg);
- throw new IllegalArgumentException(msg, e);
- }
- }
- enablePathStyleAccessIfRequired(conf);
- }
-
- private void enablePathStyleAccessIfRequired(Configuration conf) {
- final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
- if (pathStyleAccess) {
- LOG.debug("Enabling path style access!");
- s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
- }
- }
-
private void initTransferManager() {
- TransferManagerConfiguration transferConfiguration = new TransferManagerConfiguration();
+ TransferManagerConfiguration transferConfiguration =
+ new TransferManagerConfiguration();
transferConfiguration.setMinimumUploadPartSize(partSize);
transferConfiguration.setMultipartUploadThreshold(multiPartThreshold);
transferConfiguration.setMultipartCopyPartSize(partSize);
@@ -435,7 +316,7 @@ public class S3AFileSystem extends FileSystem {
* @return AmazonS3Client
*/
@VisibleForTesting
- AmazonS3Client getAmazonS3Client() {
+ AmazonS3 getAmazonS3Client() {
return s3;
}
@@ -459,10 +340,6 @@ public class S3AFileSystem extends FileSystem {
this.inputPolicy = inputPolicy;
}
- public S3AFileSystem() {
- super();
- }
-
/**
* Turns a path (relative or otherwise) into an S3 key.
*
@@ -782,8 +659,10 @@ public class S3AFileSystem extends FileSystem {
while (true) {
for (S3ObjectSummary summary : objects.getObjectSummaries()) {
- keysToDelete.add(new DeleteObjectsRequest.KeyVersion(summary.getKey()));
- String newDstKey = dstKey + summary.getKey().substring(srcKey.length());
+ keysToDelete.add(
+ new DeleteObjectsRequest.KeyVersion(summary.getKey()));
+ String newDstKey =
+ dstKey + summary.getKey().substring(srcKey.length());
copyFile(summary.getKey(), newDstKey, summary.getSize());
if (keysToDelete.size() == MAX_ENTRIES_TO_DELETE) {
@@ -1388,7 +1267,8 @@ public class S3AFileSystem extends FileSystem {
LOG.debug("Found file (with /): fake directory");
return new S3AFileStatus(true, true, path);
} else {
- LOG.warn("Found file (with /): real file? should not happen: {}", key);
+ LOG.warn("Found file (with /): real file? should not happen: {}",
+ key);
return new S3AFileStatus(meta.getContentLength(),
dateToLong(meta.getLastModified()),
@@ -1985,42 +1865,4 @@ public class S3AFileSystem extends FileSystem {
getFileBlockLocations(status, 0, status.getLen())
: null);
}
-
- /**
- * Get a integer option >= the minimum allowed value.
- * @param conf configuration
- * @param key key to look up
- * @param defVal default value
- * @param min minimum value
- * @return the value
- * @throws IllegalArgumentException if the value is below the minimum
- */
- static int intOption(Configuration conf, String key, int defVal, int min) {
- int v = conf.getInt(key, defVal);
- Preconditions.checkArgument(v >= min,
- String.format("Value of %s: %d is below the minimum value %d",
- key, v, min));
- return v;
- }
-
- /**
- * Get a long option >= the minimum allowed value.
- * @param conf configuration
- * @param key key to look up
- * @param defVal default value
- * @param min minimum value
- * @return the value
- * @throws IllegalArgumentException if the value is below the minimum
- */
- static long longOption(Configuration conf,
- String key,
- long defVal,
- long min) {
- long v = conf.getLong(key, defVal);
- Preconditions.checkArgument(v >= min,
- String.format("Value of %s: %d is below the minimum value %d",
- key, v, min));
- return v;
- }
-
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
index ccb9726..dd6cdd7 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AInputStream.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.AmazonClientException;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.google.common.base.Preconditions;
@@ -71,7 +71,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
private volatile boolean closed;
private S3ObjectInputStream wrappedStream;
private final FileSystem.Statistics stats;
- private final AmazonS3Client client;
+ private final AmazonS3 client;
private final String bucket;
private final String key;
private final long contentLength;
@@ -101,7 +101,7 @@ public class S3AInputStream extends FSInputStream implements CanSetReadahead {
public S3AInputStream(String bucket,
String key,
long contentLength,
- AmazonS3Client client,
+ AmazonS3 client,
FileSystem.Statistics stats,
S3AInstrumentation instrumentation,
long readahead,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
index 699676d..a5e8e7a 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AUtils.java
@@ -25,6 +25,9 @@ import com.amazonaws.auth.EnvironmentVariableCredentialsProvider;
import com.amazonaws.auth.InstanceProfileCredentialsProvider;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import com.google.common.base.Preconditions;
+
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -403,4 +406,41 @@ public final class S3AUtils {
builder.append("size=").append(summary.getSize());
return builder.toString();
}
+
+ /**
+ * Get a integer option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static int intOption(Configuration conf, String key, int defVal, int min) {
+ int v = conf.getInt(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
+
+ /**
+ * Get a long option >= the minimum allowed value.
+ * @param conf configuration
+ * @param key key to look up
+ * @param defVal default value
+ * @param min minimum value
+ * @return the value
+ * @throws IllegalArgumentException if the value is below the minimum
+ */
+ static long longOption(Configuration conf,
+ String key,
+ long defVal,
+ long min) {
+ long v = conf.getLong(key, defVal);
+ Preconditions.checkArgument(v >= min,
+ String.format("Value of %s: %d is below the minimum value %d",
+ key, v, min));
+ return v;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
new file mode 100644
index 0000000..0a4dd02
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3ClientFactory.java
@@ -0,0 +1,232 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+import static org.apache.hadoop.fs.s3a.S3AUtils.*;
+
+import java.io.IOException;
+import java.net.URI;
+
+import com.amazonaws.ClientConfiguration;
+import com.amazonaws.Protocol;
+import com.amazonaws.auth.AWSCredentialsProvider;
+import com.amazonaws.services.s3.AmazonS3;
+import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.S3ClientOptions;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.util.VersionInfo;
+
+import org.slf4j.Logger;
+
+/**
+ * Factory for creation of S3 client instances to be used by {@link S3Store}.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface S3ClientFactory {
+
+ /**
+ * Creates a new {@link AmazonS3} client. This method accepts the S3A file
+ * system URI both in raw input form and validated form as separate arguments,
+ * because both values may be useful in logging.
+ *
+ * @param name raw input S3A file system URI
+ * @param uri validated form of S3A file system URI
+ * @return S3 client
+ * @throws IOException IO problem
+ */
+ AmazonS3 createS3Client(URI name, URI uri) throws IOException;
+
+ /**
+ * The default factory implementation, which calls the AWS SDK to configure
+ * and create an {@link AmazonS3Client} that communicates with the S3 service.
+ */
+ static class DefaultS3ClientFactory extends Configured
+ implements S3ClientFactory {
+
+ private static final Logger LOG = S3AFileSystem.LOG;
+
+ @Override
+ public AmazonS3 createS3Client(URI name, URI uri) throws IOException {
+ Configuration conf = getConf();
+ AWSCredentialsProvider credentials =
+ createAWSCredentialProviderSet(name, conf, uri);
+ ClientConfiguration awsConf = new ClientConfiguration();
+ initConnectionSettings(conf, awsConf);
+ initProxySupport(conf, awsConf);
+ initUserAgent(conf, awsConf);
+ return createAmazonS3Client(conf, credentials, awsConf);
+ }
+
+ /**
+ * Initializes all AWS SDK settings related to connection management.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ */
+ private static void initConnectionSettings(Configuration conf,
+ ClientConfiguration awsConf) {
+ awsConf.setMaxConnections(intOption(conf, MAXIMUM_CONNECTIONS,
+ DEFAULT_MAXIMUM_CONNECTIONS, 1));
+ boolean secureConnections = conf.getBoolean(SECURE_CONNECTIONS,
+ DEFAULT_SECURE_CONNECTIONS);
+ awsConf.setProtocol(secureConnections ? Protocol.HTTPS : Protocol.HTTP);
+ awsConf.setMaxErrorRetry(intOption(conf, MAX_ERROR_RETRIES,
+ DEFAULT_MAX_ERROR_RETRIES, 0));
+ awsConf.setConnectionTimeout(intOption(conf, ESTABLISH_TIMEOUT,
+ DEFAULT_ESTABLISH_TIMEOUT, 0));
+ awsConf.setSocketTimeout(intOption(conf, SOCKET_TIMEOUT,
+ DEFAULT_SOCKET_TIMEOUT, 0));
+ int sockSendBuffer = intOption(conf, SOCKET_SEND_BUFFER,
+ DEFAULT_SOCKET_SEND_BUFFER, 2048);
+ int sockRecvBuffer = intOption(conf, SOCKET_RECV_BUFFER,
+ DEFAULT_SOCKET_RECV_BUFFER, 2048);
+ awsConf.setSocketBufferSizeHints(sockSendBuffer, sockRecvBuffer);
+ String signerOverride = conf.getTrimmed(SIGNING_ALGORITHM, "");
+ if (!signerOverride.isEmpty()) {
+ LOG.debug("Signer override = {}", signerOverride);
+ awsConf.setSignerOverride(signerOverride);
+ }
+ }
+
+ /**
+ * Initializes AWS SDK proxy support if configured.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ * @throws IllegalArgumentException if misconfigured
+ */
+ private static void initProxySupport(Configuration conf,
+ ClientConfiguration awsConf) throws IllegalArgumentException {
+ String proxyHost = conf.getTrimmed(PROXY_HOST, "");
+ int proxyPort = conf.getInt(PROXY_PORT, -1);
+ if (!proxyHost.isEmpty()) {
+ awsConf.setProxyHost(proxyHost);
+ if (proxyPort >= 0) {
+ awsConf.setProxyPort(proxyPort);
+ } else {
+ if (conf.getBoolean(SECURE_CONNECTIONS, DEFAULT_SECURE_CONNECTIONS)) {
+ LOG.warn("Proxy host set without port. Using HTTPS default 443");
+ awsConf.setProxyPort(443);
+ } else {
+ LOG.warn("Proxy host set without port. Using HTTP default 80");
+ awsConf.setProxyPort(80);
+ }
+ }
+ String proxyUsername = conf.getTrimmed(PROXY_USERNAME);
+ String proxyPassword = conf.getTrimmed(PROXY_PASSWORD);
+ if ((proxyUsername == null) != (proxyPassword == null)) {
+ String msg = "Proxy error: " + PROXY_USERNAME + " or " +
+ PROXY_PASSWORD + " set without the other.";
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ awsConf.setProxyUsername(proxyUsername);
+ awsConf.setProxyPassword(proxyPassword);
+ awsConf.setProxyDomain(conf.getTrimmed(PROXY_DOMAIN));
+ awsConf.setProxyWorkstation(conf.getTrimmed(PROXY_WORKSTATION));
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Using proxy server {}:{} as user {} with password {} on " +
+ "domain {} as workstation {}", awsConf.getProxyHost(),
+ awsConf.getProxyPort(),
+ String.valueOf(awsConf.getProxyUsername()),
+ awsConf.getProxyPassword(), awsConf.getProxyDomain(),
+ awsConf.getProxyWorkstation());
+ }
+ } else if (proxyPort >= 0) {
+ String msg =
+ "Proxy error: " + PROXY_PORT + " set without " + PROXY_HOST;
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg);
+ }
+ }
+
+ /**
+ * Initializes the User-Agent header to send in HTTP requests to the S3
+ * back-end. We always include the Hadoop version number. The user also
+ * may set an optional custom prefix to put in front of the Hadoop version
+ * number. The AWS SDK interally appends its own information, which seems
+ * to include the AWS SDK version, OS and JVM version.
+ *
+ * @param conf Hadoop configuration
+ * @param awsConf AWS SDK configuration
+ */
+ private static void initUserAgent(Configuration conf,
+ ClientConfiguration awsConf) {
+ String userAgent = "Hadoop " + VersionInfo.getVersion();
+ String userAgentPrefix = conf.getTrimmed(USER_AGENT_PREFIX, "");
+ if (!userAgentPrefix.isEmpty()) {
+ userAgent = userAgentPrefix + ", " + userAgent;
+ }
+ LOG.debug("Using User-Agent: {}", userAgent);
+ awsConf.setUserAgent(userAgent);
+ }
+
+ /**
+ * Creates an {@link AmazonS3Client} from the established configuration.
+ *
+ * @param conf Hadoop configuration
+ * @param credentials AWS credentials
+ * @param awsConf AWS SDK configuration
+ * @return S3 client
+ * @throws IllegalArgumentException if misconfigured
+ */
+ private static AmazonS3 createAmazonS3Client(Configuration conf,
+ AWSCredentialsProvider credentials, ClientConfiguration awsConf)
+ throws IllegalArgumentException {
+ AmazonS3 s3 = new AmazonS3Client(credentials, awsConf);
+ String endPoint = conf.getTrimmed(ENDPOINT, "");
+ if (!endPoint.isEmpty()) {
+ try {
+ s3.setEndpoint(endPoint);
+ } catch (IllegalArgumentException e) {
+ String msg = "Incorrect endpoint: " + e.getMessage();
+ LOG.error(msg);
+ throw new IllegalArgumentException(msg, e);
+ }
+ }
+ enablePathStyleAccessIfRequired(s3, conf);
+ return s3;
+ }
+
+ /**
+ * Enables path-style access to S3 buckets if configured. By default, the
+ * behavior is to use virtual hosted-style access with URIs of the form
+ * http://bucketname.s3.amazonaws.com. Enabling path-style access and a
+ * region-specific endpoint switches the behavior to use URIs of the form
+ * http://s3-eu-west-1.amazonaws.com/bucketname.
+ *
+ * @param s3 S3 client
+ * @param conf Hadoop configuration
+ */
+ private static void enablePathStyleAccessIfRequired(AmazonS3 s3,
+ Configuration conf) {
+ final boolean pathStyleAccess = conf.getBoolean(PATH_STYLE_ACCESS, false);
+ if (pathStyleAccess) {
+ LOG.debug("Enabling path style access!");
+ s3.setS3ClientOptions(new S3ClientOptions().withPathStyleAccess(true));
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
index bc8c2e6..97ece37 100644
--- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
+++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3native/S3xLoginHelper.java
@@ -132,6 +132,9 @@ public final class S3xLoginHelper {
*
* This strips out login information.
*
+ * @param uri the URI to canonicalize
+ * @param defaultPort default port to use in canonicalized URI if the input
+ * URI has no port and this value is greater than 0
* @return a new, canonicalized URI.
*/
public static URI canonicalizeUri(URI uri, int defaultPort) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
new file mode 100644
index 0000000..6734947
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/AbstractS3AMockTest.java
@@ -0,0 +1,70 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.apache.hadoop.fs.s3a.Constants.*;
+
+import com.amazonaws.AmazonServiceException;
+import com.amazonaws.services.s3.AmazonS3;
+
+import java.net.URI;
+
+import org.apache.hadoop.conf.Configuration;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.rules.ExpectedException;
+
+/**
+ * Abstract base class for S3A unit tests using a mock S3 client.
+ */
+public abstract class AbstractS3AMockTest {
+
+ protected static final String BUCKET = "mock-bucket";
+ protected static final AmazonServiceException NOT_FOUND;
+ static {
+ NOT_FOUND = new AmazonServiceException("Not Found");
+ NOT_FOUND.setStatusCode(404);
+ }
+
+ @Rule
+ public ExpectedException exception = ExpectedException.none();
+
+ protected S3AFileSystem fs;
+ protected AmazonS3 s3;
+
+ @Before
+ public void setup() throws Exception {
+ Configuration conf = new Configuration();
+ conf.setClass(S3_CLIENT_FACTORY_IMPL, MockS3ClientFactory.class,
+ S3ClientFactory.class);
+ fs = new S3AFileSystem();
+ URI uri = URI.create(FS_S3A + "://" + BUCKET);
+ fs.initialize(uri, conf);
+ s3 = fs.getAmazonS3Client();
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (fs != null) {
+ fs.close();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
index 4e99339..fca8e49 100644
--- a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/ITestS3AConfiguration.java
@@ -19,7 +19,7 @@
package org.apache.hadoop.fs.s3a;
import com.amazonaws.ClientConfiguration;
-import com.amazonaws.services.s3.AmazonS3Client;
+import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.S3ClientOptions;
import org.apache.commons.lang.StringUtils;
@@ -96,7 +96,7 @@ public class ITestS3AConfiguration {
} else {
conf.set(Constants.ENDPOINT, endpoint);
fs = S3ATestUtils.createTestFileSystem(conf);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
String endPointRegion = "";
// Differentiate handling of "s3-" and "s3." based endpoint identifiers
String[] endpointParts = StringUtils.split(endpoint, '.');
@@ -364,7 +364,7 @@ public class ITestS3AConfiguration {
try {
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
S3ClientOptions clientOptions = getField(s3, S3ClientOptions.class,
"clientOptions");
@@ -388,7 +388,7 @@ public class ITestS3AConfiguration {
conf = new Configuration();
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
@@ -401,7 +401,7 @@ public class ITestS3AConfiguration {
conf.set(Constants.USER_AGENT_PREFIX, "MyApp");
fs = S3ATestUtils.createTestFileSystem(conf);
assertNotNull(fs);
- AmazonS3Client s3 = fs.getAmazonS3Client();
+ AmazonS3 s3 = fs.getAmazonS3Client();
assertNotNull(s3);
ClientConfiguration awsConf = getField(s3, ClientConfiguration.class,
"clientConfiguration");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
new file mode 100644
index 0000000..41f04ee
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/MockS3ClientFactory.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.mockito.Mockito.*;
+
+import java.net.URI;
+
+import com.amazonaws.services.s3.AmazonS3;
+
+/**
+ * An {@link S3ClientFactory} that returns Mockito mocks of the {@link AmazonS3}
+ * interface suitable for unit testing.
+ */
+public class MockS3ClientFactory implements S3ClientFactory {
+
+ @Override
+ public AmazonS3 createS3Client(URI name, URI uri) {
+ String bucket = name.getHost();
+ AmazonS3 s3 = mock(AmazonS3.class);
+ when(s3.doesBucketExist(bucket)).thenReturn(true);
+ return s3;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d152557c/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
new file mode 100644
index 0000000..f9e9c6b
--- /dev/null
+++ b/hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/TestS3AGetFileStatus.java
@@ -0,0 +1,126 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.s3a;
+
+import static org.junit.Assert.*;
+import static org.mockito.Matchers.*;
+import static org.mockito.Mockito.*;
+
+import java.io.FileNotFoundException;
+import java.util.Collections;
+import java.util.Date;
+
+import com.amazonaws.services.s3.model.ListObjectsRequest;
+import com.amazonaws.services.s3.model.ObjectListing;
+import com.amazonaws.services.s3.model.ObjectMetadata;
+import com.amazonaws.services.s3.model.S3ObjectSummary;
+
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+
+import org.junit.Test;
+
+/**
+ * S3A tests for getFileStatus using mock S3 client.
+ */
+public class TestS3AGetFileStatus extends AbstractS3AMockTest {
+
+ @Test
+ public void testFile() throws Exception {
+ Path path = new Path("/file");
+ String key = path.toUri().getPath().substring(1);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(1L);
+ meta.setLastModified(new Date(2L));
+ when(s3.getObjectMetadata(BUCKET, key)).thenReturn(meta);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isFile());
+ assertEquals(meta.getContentLength(), stat.getLen());
+ assertEquals(meta.getLastModified().getTime(), stat.getModificationTime());
+ }
+
+ @Test
+ public void testFakeDirectory() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ ObjectMetadata meta = new ObjectMetadata();
+ meta.setContentLength(0L);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenReturn(meta);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ }
+
+ @Test
+ public void testImplicitDirectory() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.singletonList("dir/"));
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ }
+
+ @Test
+ public void testRoot() throws Exception {
+ Path path = new Path("/");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.<String>emptyList());
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ FileStatus stat = fs.getFileStatus(path);
+ assertNotNull(stat);
+ assertEquals(fs.makeQualified(path), stat.getPath());
+ assertTrue(stat.isDirectory());
+ assertTrue(stat.getPath().isRoot());
+ }
+
+ @Test
+ public void testNotFound() throws Exception {
+ Path path = new Path("/dir");
+ String key = path.toUri().getPath().substring(1);
+ when(s3.getObjectMetadata(BUCKET, key)).thenThrow(NOT_FOUND);
+ when(s3.getObjectMetadata(BUCKET, key + "/")).thenThrow(NOT_FOUND);
+ ObjectListing objects = mock(ObjectListing.class);
+ when(objects.getCommonPrefixes()).thenReturn(
+ Collections.<String>emptyList());
+ when(objects.getObjectSummaries()).thenReturn(
+ Collections.<S3ObjectSummary>emptyList());
+ when(s3.listObjects(any(ListObjectsRequest.class))).thenReturn(objects);
+ exception.expect(FileNotFoundException.class);
+ fs.getFileStatus(path);
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/48] hadoop git commit: HDFS-10784. Implement
WebHdfsFileSystem#listStatusIterator.
Posted by aw...@apache.org.
HDFS-10784. Implement WebHdfsFileSystem#listStatusIterator.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/85bab5fb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/85bab5fb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/85bab5fb
Branch: refs/heads/HADOOP-13341
Commit: 85bab5fb572194fda38854f1f21c670925058009
Parents: 01721dd
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Aug 31 14:29:37 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Aug 31 14:29:37 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 21 ++++
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 55 ++++++++++
.../hadoop/hdfs/web/resources/GetOpParam.java | 3 +-
.../hdfs/web/resources/StartAfterParam.java | 38 +++++++
.../web/resources/NamenodeWebHdfsMethods.java | 27 +++--
.../org/apache/hadoop/hdfs/web/JsonUtil.java | 45 ++++++--
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 105 +++++++++++++++++++
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 55 ++++++++--
.../apache/hadoop/hdfs/web/TestWebHdfsUrl.java | 25 +++++
.../hadoop/hdfs/web/resources/TestParam.java | 7 ++
10 files changed, 361 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
index baebff2..35f0f9a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/JsonUtilClient.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.web;
+import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import org.apache.hadoop.fs.ContentSummary;
@@ -33,6 +34,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -141,6 +143,25 @@ class JsonUtilClient {
storagePolicy, null);
}
+ static DirectoryListing toDirectoryListing(final Map<?, ?> json) {
+ if (json == null) {
+ return null;
+ }
+ final List<?> list = JsonUtilClient.getList(json,
+ "partialListing");
+
+ HdfsFileStatus[] partialListing = new HdfsFileStatus[list.size()];
+ int i = 0;
+ for (Object o : list) {
+ final Map<?, ?> m = (Map<?, ?>) o;
+ partialListing[i++] = toFileStatus(m, false);
+ }
+ int remainingEntries = getInt(json, "remainingEntries", -1);
+ Preconditions.checkState(remainingEntries != -1,
+ "remainingEntries was not set");
+ return new DirectoryListing(partialListing, remainingEntries);
+ }
+
/** Convert a Json map to an ExtendedBlock object. */
static ExtendedBlock toExtendedBlock(final Map<?, ?> m) {
if (m == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 9e42b24..968eea3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -64,6 +64,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
@@ -79,6 +80,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HAUtilClient;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
+import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -106,6 +108,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Charsets;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
@@ -1499,6 +1502,58 @@ public class WebHdfsFileSystem extends FileSystem
}.run();
}
+ private static final byte[] EMPTY_ARRAY = new byte[] {};
+ private class DirListingIterator<T extends FileStatus> implements
+ RemoteIterator<T> {
+
+ private final Path path;
+ private DirectoryListing thisListing;
+ private int i = 0;
+ private byte[] prevKey = EMPTY_ARRAY;
+
+ DirListingIterator(Path path) {
+ this.path = path;
+ }
+
+ @Override
+ public boolean hasNext() throws IOException {
+ if (thisListing == null) {
+ fetchMore();
+ }
+ return i < thisListing.getPartialListing().length ||
+ thisListing.hasMore();
+ }
+
+ private void fetchMore() throws IOException {
+ thisListing = new FsPathResponseRunner<DirectoryListing>(
+ GetOpParam.Op.LISTSTATUS_BATCH,
+ path, new StartAfterParam(new String(prevKey, Charsets.UTF_8))) {
+ @Override
+ DirectoryListing decodeResponse(Map<?, ?> json) throws IOException {
+ return JsonUtilClient.toDirectoryListing(json);
+ }
+ }.run();
+ i = 0;
+ prevKey = thisListing.getLastName();
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ public T next() throws IOException {
+ Preconditions.checkState(hasNext(), "No more items in iterator");
+ if (i == thisListing.getPartialListing().length) {
+ fetchMore();
+ }
+ return (T)makeQualified(thisListing.getPartialListing()[i++], path);
+ }
+ }
+
+ @Override
+ public RemoteIterator<FileStatus> listStatusIterator(final Path f)
+ throws FileNotFoundException, IOException {
+ return new DirListingIterator<>(f);
+ }
+
@Override
public Token<DelegationTokenIdentifier> getDelegationToken(
final String renewer) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
index 119f15d..8fdf581 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/GetOpParam.java
@@ -41,7 +41,8 @@ public class GetOpParam extends HttpOpParam<GetOpParam.Op> {
NULL(false, HttpURLConnection.HTTP_NOT_IMPLEMENTED),
- CHECKACCESS(false, HttpURLConnection.HTTP_OK);
+ CHECKACCESS(false, HttpURLConnection.HTTP_OK),
+ LISTSTATUS_BATCH(false, HttpURLConnection.HTTP_OK);
final boolean redirect;
final int expectedHttpResponseCode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/StartAfterParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/StartAfterParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/StartAfterParam.java
new file mode 100644
index 0000000..e11dbd1
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/StartAfterParam.java
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+/**
+ * Used during batched ListStatus operations.
+ */
+public class StartAfterParam extends StringParam {
+
+ public static final String NAME = "startafter";
+ public static final String DEFAULT = "";
+
+ private static final Domain DOMAIN = new Domain(NAME, null);
+
+ public StartAfterParam(final String str) {
+ super(DOMAIN, str != null && !str.equals(DEFAULT) ? str : null);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index 7104d25..e3a5bad 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -789,11 +789,13 @@ public class NamenodeWebHdfsMethods {
@QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT)
final TokenServiceParam tokenService,
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
- final NoRedirectParam noredirect
+ final NoRedirectParam noredirect,
+ @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT)
+ final StartAfterParam startAfter
) throws IOException, InterruptedException {
return get(ugi, delegation, username, doAsUser, ROOT, op, offset, length,
renewer, bufferSize, xattrNames, xattrEncoding, excludeDatanodes, fsAction,
- tokenKind, tokenService, noredirect);
+ tokenKind, tokenService, noredirect, startAfter);
}
/** Handle HTTP GET request. */
@@ -832,12 +834,14 @@ public class NamenodeWebHdfsMethods {
@QueryParam(TokenServiceParam.NAME) @DefaultValue(TokenServiceParam.DEFAULT)
final TokenServiceParam tokenService,
@QueryParam(NoRedirectParam.NAME) @DefaultValue(NoRedirectParam.DEFAULT)
- final NoRedirectParam noredirect
+ final NoRedirectParam noredirect,
+ @QueryParam(StartAfterParam.NAME) @DefaultValue(StartAfterParam.DEFAULT)
+ final StartAfterParam startAfter
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, offset, length,
renewer, bufferSize, xattrEncoding, excludeDatanodes, fsAction,
- tokenKind, tokenService);
+ tokenKind, tokenService, startAfter);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -846,7 +850,7 @@ public class NamenodeWebHdfsMethods {
return get(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, offset, length, renewer, bufferSize,
xattrNames, xattrEncoding, excludeDatanodes, fsAction, tokenKind,
- tokenService, noredirect);
+ tokenService, noredirect, startAfter);
} finally {
reset();
}
@@ -871,7 +875,8 @@ public class NamenodeWebHdfsMethods {
final FsActionParam fsAction,
final TokenKindParam tokenKind,
final TokenServiceParam tokenService,
- final NoRedirectParam noredirectParam
+ final NoRedirectParam noredirectParam,
+ final StartAfterParam startAfter
) throws IOException, URISyntaxException {
final NameNode namenode = (NameNode)context.getAttribute("name.node");
final Configuration conf = (Configuration) context
@@ -994,6 +999,16 @@ public class NamenodeWebHdfsMethods {
np.checkAccess(fullpath, FsAction.getFsAction(fsAction.getValue()));
return Response.ok().build();
}
+ case LISTSTATUS_BATCH:
+ {
+ byte[] start = HdfsFileStatus.EMPTY_NAME;
+ if (startAfter != null && startAfter.getValue() != null) {
+ start = startAfter.getValue().getBytes(Charsets.UTF_8);
+ }
+ final DirectoryListing listing = getDirectoryListing(np, fullpath, start);
+ final String js = JsonUtil.toJsonString(listing);
+ return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
+ }
default:
throw new UnsupportedOperationException(op + " is not supported");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
index 368e879..2960f14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/web/JsonUtil.java
@@ -96,6 +96,16 @@ public class JsonUtil {
if (status == null) {
return null;
}
+ final Map<String, Object> m = toJsonMap(status);
+ try {
+ return includeType ?
+ toJsonString(FileStatus.class, m) : MAPPER.writeValueAsString(m);
+ } catch (IOException ignored) {
+ }
+ return null;
+ }
+
+ private static Map<String, Object> toJsonMap(HdfsFileStatus status) {
final Map<String, Object> m = new TreeMap<String, Object>();
m.put("pathSuffix", status.getLocalName());
m.put("type", WebHdfsConstants.PathType.valueOf(status));
@@ -121,12 +131,7 @@ public class JsonUtil {
m.put("fileId", status.getFileId());
m.put("childrenNum", status.getChildrenNum());
m.put("storagePolicy", status.getStoragePolicy());
- try {
- return includeType ?
- toJsonString(FileStatus.class, m) : MAPPER.writeValueAsString(m);
- } catch (IOException ignored) {
- }
- return null;
+ return m;
}
/** Convert an ExtendedBlock to a Json map. */
@@ -227,6 +232,34 @@ public class JsonUtil {
return m;
}
+ public static String toJsonString(final DirectoryListing listing) throws
+ IOException {
+
+ if (listing == null) {
+ return null;
+ }
+
+ final Map<String, Object> m = new TreeMap<>();
+ m.put("partialListing", toJsonArray(listing.getPartialListing()));
+ m.put("remainingEntries", listing.getRemainingEntries());
+ return MAPPER.writeValueAsString(m);
+ }
+
+ private static Object[] toJsonArray(HdfsFileStatus[] statuses) throws
+ IOException {
+ if (statuses == null) {
+ return null;
+ }
+ if (statuses.length == 0) {
+ return EMPTY_OBJECT_ARRAY;
+ }
+ final Object[] a = new Object[statuses.length];
+ for (int i = 0; i < statuses.length; i++) {
+ a[i] = toJsonMap(statuses[i]);
+ }
+ return a;
+ }
+
/** Convert a LocatedBlock[] to a Json array. */
private static Object[] toJsonArray(final List<LocatedBlock> array
) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 9ede4da..30f583e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -37,6 +37,7 @@ WebHDFS REST API
* [Truncate a File](#Truncate_a_File)
* [Status of a File/Directory](#Status_of_a_FileDirectory)
* [List a Directory](#List_a_Directory)
+ * [Iteratively List a Directory](#Iteratively_List_a_Directory)
* [Other File System Operations](#Other_File_System_Operations)
* [Get Content Summary of a Directory](#Get_Content_Summary_of_a_Directory)
* [Get File Checksum](#Get_File_Checksum)
@@ -143,6 +144,7 @@ The HTTP REST API supports the complete [FileSystem](../../api/org/apache/hadoop
* [`OPEN`](#Open_and_Read_a_File) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).open)
* [`GETFILESTATUS`](#Status_of_a_FileDirectory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileStatus)
* [`LISTSTATUS`](#List_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listStatus)
+ * [`LISTSTATUS_BATCH`](#Iteratively_List_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listStatusIterator)
* [`GETCONTENTSUMMARY`](#Get_Content_Summary_of_a_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getContentSummary)
* [`GETFILECHECKSUM`](#Get_File_Checksum) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileChecksum)
* [`GETHOMEDIRECTORY`](#Get_Home_Directory) (see [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getHomeDirectory)
@@ -589,6 +591,109 @@ See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).getFileSt
See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listStatus
+### Iteratively List a Directory
+
+* Submit a HTTP GET request.
+
+ curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS_BATCH&startAfter=<CHILD>"
+
+ The client receives a response with a batch of [`FileStatuses` JSON object](#FileStatuses_JSON_Schema), as well as iteration information:
+
+ HTTP/1.1 200 OK
+ Cache-Control: no-cache
+ Expires: Tue, 30 Aug 2016 16:42:16 GMT
+ Date: Tue, 30 Aug 2016 16:42:16 GMT
+ Pragma: no-cache
+ Expires: Tue, 30 Aug 2016 16:42:16 GMT
+ Date: Tue, 30 Aug 2016 16:42:16 GMT
+ Pragma: no-cache
+ Content-Type: application/json
+ X-FRAME-OPTIONS: SAMEORIGIN
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+ {
+ "partialListing": [
+ {
+ "accessTime": 0,
+ "blockSize": 0,
+ "childrenNum": 0,
+ "fileId": 16389,
+ "group": "supergroup",
+ "length": 0,
+ "modificationTime": 1472575493064,
+ "owner": "andrew",
+ "pathSuffix": "anotherdir",
+ "permission": "755",
+ "replication": 0,
+ "storagePolicy": 0,
+ "type": "DIRECTORY"
+ },
+ {
+ "accessTime": 0,
+ "blockSize": 0,
+ "childrenNum": 0,
+ "fileId": 16386,
+ "group": "supergroup",
+ "length": 0,
+ "modificationTime": 1472575274776,
+ "owner": "andrew",
+ "pathSuffix": "somedir",
+ "permission": "755",
+ "replication": 0,
+ "storagePolicy": 0,
+ "type": "DIRECTORY"
+ }
+ ],
+ "remainingEntries": 1
+ }
+
+
+If `remainingEntries` is non-zero, there are additional entries in the directory.
+To query the next batch, set the `startAfter` parameter to the `pathSuffix` of the last item returned in the current batch. For example:
+
+ curl -i "http://<HOST>:<PORT>/webhdfs/v1/<PATH>?op=LISTSTATUS_BATCH&startAfter=somedir"
+
+Which will return the next batch of directory entries:
+
+ HTTP/1.1 200 OK
+ Cache-Control: no-cache
+ Expires: Tue, 30 Aug 2016 16:46:23 GMT
+ Date: Tue, 30 Aug 2016 16:46:23 GMT
+ Pragma: no-cache
+ Expires: Tue, 30 Aug 2016 16:46:23 GMT
+ Date: Tue, 30 Aug 2016 16:46:23 GMT
+ Pragma: no-cache
+ Content-Type: application/json
+ X-FRAME-OPTIONS: SAMEORIGIN
+ Transfer-Encoding: chunked
+ Server: Jetty(6.1.26)
+
+ {
+ "partialListing": [
+ {
+ "accessTime": 1472575333568,
+ "blockSize": 1024,
+ "childrenNum": 0,
+ "fileId": 16388,
+ "group": "supergroup",
+ "length": 224,
+ "modificationTime": 1472575334222,
+ "owner": "andrew",
+ "pathSuffix": "somefile",
+ "permission": "644",
+ "replication": 3,
+ "storagePolicy": 0,
+ "type": "FILE"
+ }
+ ],
+ "remainingEntries": 0
+ }
+
+Batch size is controlled by the `dfs.ls.limit` option on the NameNode.
+
+See also: [FileSystem](../../api/org/apache/hadoop/fs/FileSystem.html).listStatusIterator
+
Other File System Operations
----------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
index 7aa4d78..604a063 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHDFS.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.hdfs.web;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
@@ -50,6 +51,7 @@ import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -299,11 +301,50 @@ public class TestWebHDFS {
WebHdfsConstants.WEBHDFS_SCHEME);
Path d = new Path("/my-dir");
Assert.assertTrue(fs.mkdirs(d));
- for (int i=0; i < listLimit*3; i++) {
- Path p = new Path(d, "file-"+i);
+ // Iterator should have no items when dir is empty
+ RemoteIterator<FileStatus> it = fs.listStatusIterator(d);
+ assertFalse(it.hasNext());
+ Path p = new Path(d, "file-"+0);
+ Assert.assertTrue(fs.createNewFile(p));
+ // Iterator should have an item when dir is not empty
+ it = fs.listStatusIterator(d);
+ assertTrue(it.hasNext());
+ it.next();
+ assertFalse(it.hasNext());
+ for (int i=1; i < listLimit*3; i++) {
+ p = new Path(d, "file-"+i);
Assert.assertTrue(fs.createNewFile(p));
}
- Assert.assertEquals(listLimit*3, fs.listStatus(d).length);
+ // Check the FileStatus[] listing
+ FileStatus[] statuses = fs.listStatus(d);
+ Assert.assertEquals(listLimit*3, statuses.length);
+ // Check the iterator-based listing
+ GenericTestUtils.setLogLevel(WebHdfsFileSystem.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(NamenodeWebHdfsMethods.LOG, Level
+ .TRACE);
+ it = fs.listStatusIterator(d);
+ int count = 0;
+ while (it.hasNext()) {
+ FileStatus stat = it.next();
+ assertEquals("FileStatuses not equal", statuses[count], stat);
+ count++;
+ }
+ assertEquals("Different # of statuses!", statuses.length, count);
+ // Do some more basic iterator tests
+ it = fs.listStatusIterator(d);
+ // Try advancing the iterator without calling hasNext()
+ for (int i = 0; i < statuses.length; i++) {
+ FileStatus stat = it.next();
+ assertEquals("FileStatuses not equal", statuses[i], stat);
+ }
+ assertFalse("No more items expected", it.hasNext());
+ // Try doing next when out of items
+ try {
+ it.next();
+ fail("Iterator should error if out of elements.");
+ } catch (IllegalStateException e) {
+ // pass
+ }
return null;
}
});
@@ -488,9 +529,9 @@ public class TestWebHDFS {
// delete the two snapshots
webHdfs.deleteSnapshot(foo, "s1");
- Assert.assertFalse(webHdfs.exists(s1path));
+ assertFalse(webHdfs.exists(s1path));
webHdfs.deleteSnapshot(foo, spath.getName());
- Assert.assertFalse(webHdfs.exists(spath));
+ assertFalse(webHdfs.exists(spath));
} finally {
if (cluster != null) {
cluster.shutdown();
@@ -546,12 +587,12 @@ public class TestWebHDFS {
// rename s1 to s2
webHdfs.renameSnapshot(foo, "s1", "s2");
- Assert.assertFalse(webHdfs.exists(s1path));
+ assertFalse(webHdfs.exists(s1path));
final Path s2path = SnapshotTestHelper.getSnapshotRoot(foo, "s2");
Assert.assertTrue(webHdfs.exists(s2path));
webHdfs.deleteSnapshot(foo, "s2");
- Assert.assertFalse(webHdfs.exists(s2path));
+ assertFalse(webHdfs.exists(s2path));
} finally {
if (cluster != null) {
cluster.shutdown();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
index 24c13af..9b297d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/TestWebHdfsUrl.java
@@ -39,6 +39,7 @@ import org.apache.hadoop.hdfs.web.resources.DelegationParam;
import org.apache.hadoop.hdfs.web.resources.DoAsParam;
import org.apache.hadoop.hdfs.web.resources.GetOpParam;
import org.apache.hadoop.hdfs.web.resources.PutOpParam;
+import org.apache.hadoop.hdfs.web.resources.StartAfterParam;
import org.apache.hadoop.hdfs.web.resources.TokenArgumentParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.hdfs.web.resources.FsActionParam;
@@ -306,6 +307,30 @@ public class TestWebHdfsUrl {
},
checkAccessUrl);
}
+
+ @Test(timeout=60000)
+ public void testBatchedListingUrl() throws Exception {
+ Configuration conf = new Configuration();
+
+ UserGroupInformation ugi =
+ UserGroupInformation.createRemoteUser("test-user");
+ UserGroupInformation.setLoginUser(ugi);
+
+ WebHdfsFileSystem webhdfs = getWebHdfsFileSystem(ugi, conf);
+ Path fsPath = new Path("/p1");
+
+ final StartAfterParam startAfter =
+ new StartAfterParam("last");
+ URL url = webhdfs.toUrl(GetOpParam.Op.LISTSTATUS_BATCH,
+ fsPath, startAfter);
+ checkQueryParams(
+ new String[]{
+ GetOpParam.Op.LISTSTATUS_BATCH.toQueryString(),
+ new UserParam(ugi.getShortUserName()).toString(),
+ StartAfterParam.NAME + "=" + "last"
+ },
+ url);
+ }
private void checkQueryParams(String[] expected, URL url) {
Arrays.sort(expected);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/85bab5fb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
index aebec89..fce8917 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/web/resources/TestParam.java
@@ -454,4 +454,11 @@ public class TestParam {
LOG.info("EXPECTED: " + e);
}
}
+
+ @Test
+ public void testStartAfterParam() throws Exception {
+ String s = "/helloWorld";
+ StartAfterParam param = new StartAfterParam(s);
+ Assert.assertEquals(s, param.getValue());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/48] hadoop git commit: HADOOP-13359. Modify YARN to use
hadoop_subcommand_opts
Posted by aw...@apache.org.
HADOOP-13359. Modify YARN to use hadoop_subcommand_opts
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7f75b084
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7f75b084
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7f75b084
Branch: refs/heads/HADOOP-13341
Commit: 7f75b084ebb1630be935642f69033c91a242b171
Parents: 1a3e102
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 13:03:45 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
hadoop-yarn-project/hadoop-yarn/bin/yarn | 17 +++++------------
1 file changed, 5 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7f75b084/hadoop-yarn-project/hadoop-yarn/bin/yarn
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn
index 66a87b6..d2a8a50 100755
--- a/hadoop-yarn-project/hadoop-yarn/bin/yarn
+++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn
@@ -110,8 +110,6 @@ function yarncmd_case
nodemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.nodemanager.NodeManager'
- hadoop_debug "Append YARN_NODEMANAGER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_NODEMANAGER_OPTS}"
# Backwards compatibility
if [[ -n "${YARN_NODEMANAGER_HEAPSIZE}" ]]; then
HADOOP_HEAPSIZE_MAX="${YARN_NODEMANAGER_HEAPSIZE}"
@@ -120,8 +118,6 @@ function yarncmd_case
proxyserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.webproxy.WebAppProxyServer'
- hadoop_debug "Append YARN_PROXYSERVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_PROXYSERVER_OPTS}"
# Backwards compatibility
if [[ -n "${YARN_PROXYSERVER_HEAPSIZE}" ]]; then
# shellcheck disable=SC2034
@@ -134,8 +130,6 @@ function yarncmd_case
resourcemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.resourcemanager.ResourceManager'
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_RESOURCEMANAGER_OPTS}"
- hadoop_debug "Append YARN_RESOURCEMANAGER_OPTS onto HADOOP_OPTS"
# Backwards compatibility
if [[ -n "${YARN_RESOURCEMANAGER_HEAPSIZE}" ]]; then
# shellcheck disable=SC2034
@@ -151,20 +145,14 @@ function yarncmd_case
sharedcachemanager)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.sharedcachemanager.SharedCacheManager'
- hadoop_debug "Append YARN_SHAREDCACHEMANAGER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_SHAREDCACHEMANAGER_OPTS}"
;;
timelinereader)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.timelineservice.reader.TimelineReaderServer'
- hadoop_debug "Append YARN_TIMELINEREADER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINEREADER_OPTS}"
;;
timelineserver)
HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true"
HADOOP_CLASSNAME='org.apache.hadoop.yarn.server.applicationhistoryservice.ApplicationHistoryServer'
- hadoop_debug "Append YARN_TIMELINESERVER_OPTS onto HADOOP_OPTS"
- HADOOP_OPTS="${HADOOP_OPTS} ${YARN_TIMELINESERVER_OPTS}"
# Backwards compatibility
if [[ -n "${YARN_TIMELINESERVER_HEAPSIZE}" ]]; then
# shellcheck disable=SC2034
@@ -268,8 +256,13 @@ if [[ ${HADOOP_WORKER_MODE} = true ]]; then
exit $?
fi
+hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then
HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}"
+
+ hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}"
+
hadoop_verify_secure_prereq
hadoop_setup_secure_service
priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out"
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/48] hadoop git commit: HADOOP-13563. hadoop_subcommand_opts
should print name not actual content during debug
Posted by aw...@apache.org.
HADOOP-13563. hadoop_subcommand_opts should print name not actual content during debug
Signed-off-by: Allen Wittenauer <aw...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dc6d4900
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dc6d4900
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dc6d4900
Branch: refs/heads/HADOOP-13341
Commit: dc6d4900d270a714d05055b822983cd3198aba61
Parents: 930e03b
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Aug 30 15:23:29 2016 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Thu Sep 8 07:57:19 2016 -0700
----------------------------------------------------------------------
.../hadoop-common/src/main/bin/hadoop-functions.sh | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dc6d4900/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
index 91546d4..b6e2b59 100755
--- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
+++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh
@@ -2057,7 +2057,7 @@ function hadoop_subcommand_opts
fi
if [[ -n ${!uvar} ]]; then
- hadoop_debug "Appending ${!uvar} onto HADOOP_OPTS"
+ hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
return 0
fi
@@ -2102,7 +2102,7 @@ function hadoop_subcommand_secure_opts
uvar="${uprogram}_${ucommand}_SECURE_EXTRA_OPTS"
if [[ -n ${!uvar} ]]; then
- hadoop_debug "Appending ${!uvar} onto HADOOP_OPTS"
+ hadoop_debug "Appending ${uvar} onto HADOOP_OPTS"
HADOOP_OPTS="${HADOOP_OPTS} ${!uvar}"
return 0
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/48] hadoop git commit: Add CHANGES and release notes for
3.0.0-alpha1 to site
Posted by aw...@apache.org.
Add CHANGES and release notes for 3.0.0-alpha1 to site
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f388a209
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f388a209
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f388a209
Branch: refs/heads/HADOOP-13341
Commit: f388a209431c51e4062cc4a947274220870558a7
Parents: d152557
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Sep 6 09:51:48 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Sep 6 09:51:55 2016 -0700
----------------------------------------------------------------------
.../3.0.0-alpha1/CHANGES.3.0.0-alpha1.md | 3712 ++++++++++++++++++
.../3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md | 1854 +++++++++
2 files changed, 5566 insertions(+)
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/48] hadoop git commit: HADOOP-13547. Optimize IPC client protobuf
decoding. Contributed by Daryn Sharp.
Posted by aw...@apache.org.
HADOOP-13547. Optimize IPC client protobuf decoding. Contributed by Daryn Sharp.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/23abb09c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/23abb09c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/23abb09c
Branch: refs/heads/HADOOP-13341
Commit: 23abb09c1f979d8c18ece81e32630a35ed569399
Parents: 05f5c0f
Author: Kihwal Lee <ki...@apache.org>
Authored: Fri Sep 2 11:03:18 2016 -0500
Committer: Kihwal Lee <ki...@apache.org>
Committed: Fri Sep 2 11:03:18 2016 -0500
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/ipc/Client.java | 77 +++---
.../apache/hadoop/ipc/ProtobufRpcEngine.java | 241 +++----------------
.../org/apache/hadoop/ipc/ResponseBuffer.java | 12 +-
.../java/org/apache/hadoop/ipc/RpcWritable.java | 20 +-
.../apache/hadoop/security/SaslRpcClient.java | 45 ++--
5 files changed, 102 insertions(+), 293 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
index 183bad4..567b932 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/Client.java
@@ -21,7 +21,6 @@ package org.apache.hadoop.ipc;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
-import com.google.protobuf.CodedOutputStream;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -31,13 +30,11 @@ import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicies;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.io.retry.RetryPolicy.RetryAction;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.Server.AuthProtocol;
import org.apache.hadoop.ipc.protobuf.IpcConnectionContextProtos.IpcConnectionContextProto;
@@ -54,7 +51,6 @@ import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.ProtoUtil;
-import org.apache.hadoop.util.ReflectionUtils;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.AsyncGet;
@@ -65,6 +61,7 @@ import javax.net.SocketFactory;
import javax.security.sasl.Sasl;
import java.io.*;
import java.net.*;
+import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.*;
import java.util.Map.Entry;
@@ -429,7 +426,7 @@ public class Client implements AutoCloseable {
private final boolean doPing; //do we need to send ping message
private final int pingInterval; // how often sends ping to the server
private final int soTimeout; // used by ipc ping and rpc timeout
- private ByteArrayOutputStream pingRequest; // ping message
+ private ResponseBuffer pingRequest; // ping message
// currently active calls
private Hashtable<Integer, Call> calls = new Hashtable<Integer, Call>();
@@ -459,7 +456,7 @@ public class Client implements AutoCloseable {
this.doPing = remoteId.getDoPing();
if (doPing) {
// construct a RPC header with the callId as the ping callId
- pingRequest = new ByteArrayOutputStream();
+ pingRequest = new ResponseBuffer();
RpcRequestHeaderProto pingHeader = ProtoUtil
.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
OperationProto.RPC_FINAL_PACKET, PING_CALL_ID,
@@ -979,12 +976,10 @@ public class Client implements AutoCloseable {
.makeRpcRequestHeader(RpcKind.RPC_PROTOCOL_BUFFER,
OperationProto.RPC_FINAL_PACKET, CONNECTION_CONTEXT_CALL_ID,
RpcConstants.INVALID_RETRY_COUNT, clientId);
- RpcRequestMessageWrapper request =
- new RpcRequestMessageWrapper(connectionContextHeader, message);
-
- // Write out the packet length
- out.writeInt(request.getLength());
- request.write(out);
+ final ResponseBuffer buf = new ResponseBuffer();
+ connectionContextHeader.writeDelimitedTo(buf);
+ message.writeDelimitedTo(buf);
+ buf.writeTo(out);
}
/* wait till someone signals us to start reading RPC response or
@@ -1030,7 +1025,6 @@ public class Client implements AutoCloseable {
if ( curTime - lastActivity.get() >= pingInterval) {
lastActivity.set(curTime);
synchronized (out) {
- out.writeInt(pingRequest.size());
pingRequest.writeTo(out);
out.flush();
}
@@ -1085,12 +1079,13 @@ public class Client implements AutoCloseable {
// 2) RpcRequest
//
// Items '1' and '2' are prepared here.
- final DataOutputBuffer d = new DataOutputBuffer();
RpcRequestHeaderProto header = ProtoUtil.makeRpcRequestHeader(
call.rpcKind, OperationProto.RPC_FINAL_PACKET, call.id, call.retry,
clientId);
- header.writeDelimitedTo(d);
- call.rpcRequest.write(d);
+
+ final ResponseBuffer buf = new ResponseBuffer();
+ header.writeDelimitedTo(buf);
+ RpcWritable.wrap(call.rpcRequest).writeTo(buf);
synchronized (sendRpcRequestLock) {
Future<?> senderFuture = sendParamsExecutor.submit(new Runnable() {
@@ -1101,14 +1096,10 @@ public class Client implements AutoCloseable {
if (shouldCloseConnection.get()) {
return;
}
-
- if (LOG.isDebugEnabled())
+ if (LOG.isDebugEnabled()) {
LOG.debug(getName() + " sending #" + call.id);
-
- byte[] data = d.getData();
- int totalLength = d.getLength();
- out.writeInt(totalLength); // Total Length
- out.write(data, 0, totalLength);// RpcRequestHeader + RpcRequest
+ }
+ buf.writeTo(out); // RpcRequestHeader + RpcRequest
out.flush();
}
} catch (IOException e) {
@@ -1119,7 +1110,7 @@ public class Client implements AutoCloseable {
} finally {
//the buffer is just an in-memory buffer, but it is still polite to
// close early
- IOUtils.closeStream(d);
+ IOUtils.closeStream(buf);
}
}
});
@@ -1151,12 +1142,13 @@ public class Client implements AutoCloseable {
try {
int totalLen = in.readInt();
- RpcResponseHeaderProto header =
- RpcResponseHeaderProto.parseDelimitedFrom(in);
- checkResponse(header);
+ ByteBuffer bb = ByteBuffer.allocate(totalLen);
+ in.readFully(bb.array());
- int headerLen = header.getSerializedSize();
- headerLen += CodedOutputStream.computeRawVarint32Size(headerLen);
+ RpcWritable.Buffer packet = RpcWritable.Buffer.wrap(bb);
+ RpcResponseHeaderProto header =
+ packet.getValue(RpcResponseHeaderProto.getDefaultInstance());
+ checkResponse(header);
int callId = header.getCallId();
if (LOG.isDebugEnabled())
@@ -1164,28 +1156,15 @@ public class Client implements AutoCloseable {
RpcStatusProto status = header.getStatus();
if (status == RpcStatusProto.SUCCESS) {
- Writable value = ReflectionUtils.newInstance(valueClass, conf);
- value.readFields(in); // read value
+ Writable value = packet.newInstance(valueClass, conf);
final Call call = calls.remove(callId);
call.setRpcResponse(value);
-
- // verify that length was correct
- // only for ProtobufEngine where len can be verified easily
- if (call.getRpcResponse() instanceof ProtobufRpcEngine.RpcWrapper) {
- ProtobufRpcEngine.RpcWrapper resWrapper =
- (ProtobufRpcEngine.RpcWrapper) call.getRpcResponse();
- if (totalLen != headerLen + resWrapper.getLength()) {
- throw new RpcClientException(
- "RPC response length mismatch on rpc success");
- }
- }
- } else { // Rpc Request failed
- // Verify that length was correct
- if (totalLen != headerLen) {
- throw new RpcClientException(
- "RPC response length mismatch on rpc error");
- }
-
+ }
+ // verify that packet length was correct
+ if (packet.remaining() > 0) {
+ throw new RpcClientException("RPC response length mismatch");
+ }
+ if (status != RpcStatusProto.SUCCESS) { // Rpc Request failed
final String exceptionClassName = header.hasExceptionClassName() ?
header.getExceptionClassName() :
"ServerDidNotSetExceptionClassName";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index eb30aa2..83e4b9e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -27,29 +27,22 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.io.DataOutputOutputStream;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ipc.Client.ConnectionId;
import org.apache.hadoop.ipc.RPC.RpcInvoker;
import org.apache.hadoop.ipc.RpcWritable;
import org.apache.hadoop.ipc.protobuf.ProtobufRpcEngineProtos.RequestHeaderProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
-import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcResponseHeaderProto;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.SecretManager;
import org.apache.hadoop.security.token.TokenIdentifier;
-import org.apache.hadoop.util.ProtoUtil;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.concurrent.AsyncGet;
import org.apache.htrace.core.TraceScope;
import org.apache.htrace.core.Tracer;
import javax.net.SocketFactory;
-import java.io.DataInput;
-import java.io.DataOutput;
import java.io.IOException;
-import java.io.OutputStream;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
import java.net.InetSocketAddress;
@@ -146,7 +139,7 @@ public class ProtobufRpcEngine implements RpcEngine {
private Invoker(Class<?> protocol, Client.ConnectionId connId,
Configuration conf, SocketFactory factory) {
this.remoteId = connId;
- this.client = CLIENTS.getClient(conf, factory, RpcResponseWrapper.class);
+ this.client = CLIENTS.getClient(conf, factory, RpcWritable.Buffer.class);
this.protocolName = RPC.getProtocolName(protocol);
this.clientProtocolVersion = RPC
.getProtocolVersion(protocol);
@@ -193,7 +186,7 @@ public class ProtobufRpcEngine implements RpcEngine {
* the server.
*/
@Override
- public Object invoke(Object proxy, final Method method, Object[] args)
+ public Message invoke(Object proxy, final Method method, Object[] args)
throws ServiceException {
long startTime = 0;
if (LOG.isDebugEnabled()) {
@@ -228,11 +221,11 @@ public class ProtobufRpcEngine implements RpcEngine {
}
- Message theRequest = (Message) args[1];
- final RpcResponseWrapper val;
+ final Message theRequest = (Message) args[1];
+ final RpcWritable.Buffer val;
try {
- val = (RpcResponseWrapper) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
- new RpcRequestWrapper(rpcRequestHeader, theRequest), remoteId,
+ val = (RpcWritable.Buffer) client.call(RPC.RpcKind.RPC_PROTOCOL_BUFFER,
+ new RpcProtobufRequest(rpcRequestHeader, theRequest), remoteId,
fallbackToSimpleAuth);
} catch (Throwable e) {
@@ -256,7 +249,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (Client.isAsynchronousMode()) {
- final AsyncGet<RpcResponseWrapper, IOException> arr
+ final AsyncGet<RpcWritable.Buffer, IOException> arr
= Client.getAsyncRpcResponse();
final AsyncGet<Message, Exception> asyncGet
= new AsyncGet<Message, Exception>() {
@@ -278,7 +271,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
private Message getReturnMessage(final Method method,
- final RpcResponseWrapper rrw) throws ServiceException {
+ final RpcWritable.Buffer buf) throws ServiceException {
Message prototype = null;
try {
prototype = getReturnProtoType(method);
@@ -287,8 +280,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
Message returnMessage;
try {
- returnMessage = prototype.newBuilderForType()
- .mergeFrom(rrw.theResponseRead).build();
+ returnMessage = buf.getValue(prototype.getDefaultInstanceForType());
if (LOG.isTraceEnabled()) {
LOG.trace(Thread.currentThread().getId() + ": Response <- " +
@@ -329,201 +321,12 @@ public class ProtobufRpcEngine implements RpcEngine {
}
}
- interface RpcWrapper extends Writable {
- int getLength();
- }
- /**
- * Wrapper for Protocol Buffer Requests
- *
- * Note while this wrapper is writable, the request on the wire is in
- * Protobuf. Several methods on {@link org.apache.hadoop.ipc.Server and RPC}
- * use type Writable as a wrapper to work across multiple RpcEngine kinds.
- */
- private static abstract class RpcMessageWithHeader<T extends GeneratedMessage>
- implements RpcWrapper {
- T requestHeader;
- Message theRequest; // for clientSide, the request is here
- byte[] theRequestRead; // for server side, the request is here
-
- public RpcMessageWithHeader() {
- }
-
- public RpcMessageWithHeader(T requestHeader, Message theRequest) {
- this.requestHeader = requestHeader;
- this.theRequest = theRequest;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- OutputStream os = DataOutputOutputStream.constructOutputStream(out);
-
- ((Message)requestHeader).writeDelimitedTo(os);
- theRequest.writeDelimitedTo(os);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- requestHeader = parseHeaderFrom(readVarintBytes(in));
- theRequestRead = readMessageRequest(in);
- }
-
- abstract T parseHeaderFrom(byte[] bytes) throws IOException;
-
- byte[] readMessageRequest(DataInput in) throws IOException {
- return readVarintBytes(in);
- }
-
- private static byte[] readVarintBytes(DataInput in) throws IOException {
- final int length = ProtoUtil.readRawVarint32(in);
- final byte[] bytes = new byte[length];
- in.readFully(bytes);
- return bytes;
- }
-
- public T getMessageHeader() {
- return requestHeader;
- }
-
- public byte[] getMessageBytes() {
- return theRequestRead;
- }
-
- @Override
- public int getLength() {
- int headerLen = requestHeader.getSerializedSize();
- int reqLen;
- if (theRequest != null) {
- reqLen = theRequest.getSerializedSize();
- } else if (theRequestRead != null ) {
- reqLen = theRequestRead.length;
- } else {
- throw new IllegalArgumentException(
- "getLength on uninitialized RpcWrapper");
- }
- return CodedOutputStream.computeRawVarint32Size(headerLen) + headerLen
- + CodedOutputStream.computeRawVarint32Size(reqLen) + reqLen;
- }
- }
-
- private static class RpcRequestWrapper
- extends RpcMessageWithHeader<RequestHeaderProto> {
- @SuppressWarnings("unused")
- public RpcRequestWrapper() {}
-
- public RpcRequestWrapper(
- RequestHeaderProto requestHeader, Message theRequest) {
- super(requestHeader, theRequest);
- }
-
- @Override
- RequestHeaderProto parseHeaderFrom(byte[] bytes) throws IOException {
- return RequestHeaderProto.parseFrom(bytes);
- }
-
- @Override
- public String toString() {
- return requestHeader.getDeclaringClassProtocolName() + "." +
- requestHeader.getMethodName();
- }
- }
-
- @InterfaceAudience.LimitedPrivate({"RPC"})
- public static class RpcRequestMessageWrapper
- extends RpcMessageWithHeader<RpcRequestHeaderProto> {
- public RpcRequestMessageWrapper() {}
-
- public RpcRequestMessageWrapper(
- RpcRequestHeaderProto requestHeader, Message theRequest) {
- super(requestHeader, theRequest);
- }
-
- @Override
- RpcRequestHeaderProto parseHeaderFrom(byte[] bytes) throws IOException {
- return RpcRequestHeaderProto.parseFrom(bytes);
- }
- }
-
- @InterfaceAudience.LimitedPrivate({"RPC"})
- public static class RpcResponseMessageWrapper
- extends RpcMessageWithHeader<RpcResponseHeaderProto> {
- public RpcResponseMessageWrapper() {}
-
- public RpcResponseMessageWrapper(
- RpcResponseHeaderProto responseHeader, Message theRequest) {
- super(responseHeader, theRequest);
- }
-
- @Override
- byte[] readMessageRequest(DataInput in) throws IOException {
- // error message contain no message body
- switch (requestHeader.getStatus()) {
- case ERROR:
- case FATAL:
- return null;
- default:
- return super.readMessageRequest(in);
- }
- }
-
- @Override
- RpcResponseHeaderProto parseHeaderFrom(byte[] bytes) throws IOException {
- return RpcResponseHeaderProto.parseFrom(bytes);
- }
- }
-
- /**
- * Wrapper for Protocol Buffer Responses
- *
- * Note while this wrapper is writable, the request on the wire is in
- * Protobuf. Several methods on {@link org.apache.hadoop.ipc.Server and RPC}
- * use type Writable as a wrapper to work across multiple RpcEngine kinds.
- */
- @InterfaceAudience.LimitedPrivate({"RPC"}) // temporarily exposed
- public static class RpcResponseWrapper implements RpcWrapper {
- Message theResponse; // for senderSide, the response is here
- byte[] theResponseRead; // for receiver side, the response is here
-
- public RpcResponseWrapper() {
- }
-
- public RpcResponseWrapper(Message message) {
- this.theResponse = message;
- }
-
- @Override
- public void write(DataOutput out) throws IOException {
- OutputStream os = DataOutputOutputStream.constructOutputStream(out);
- theResponse.writeDelimitedTo(os);
- }
-
- @Override
- public void readFields(DataInput in) throws IOException {
- int length = ProtoUtil.readRawVarint32(in);
- theResponseRead = new byte[length];
- in.readFully(theResponseRead);
- }
-
- @Override
- public int getLength() {
- int resLen;
- if (theResponse != null) {
- resLen = theResponse.getSerializedSize();
- } else if (theResponseRead != null ) {
- resLen = theResponseRead.length;
- } else {
- throw new IllegalArgumentException(
- "getLength on uninitialized RpcWrapper");
- }
- return CodedOutputStream.computeRawVarint32Size(resLen) + resLen;
- }
- }
-
@VisibleForTesting
@InterfaceAudience.Private
@InterfaceStability.Unstable
static Client getClient(Configuration conf) {
return CLIENTS.getClient(conf, SocketFactory.getDefault(),
- RpcResponseWrapper.class);
+ RpcWritable.Buffer.class);
}
@@ -691,16 +494,30 @@ public class ProtobufRpcEngine implements RpcEngine {
// which uses the rpc header. in the normal case we want to defer decoding
// the rpc header until needed by the rpc engine.
static class RpcProtobufRequest extends RpcWritable.Buffer {
- private RequestHeaderProto lazyHeader;
+ private volatile RequestHeaderProto requestHeader;
+ private Message payload;
public RpcProtobufRequest() {
}
- synchronized RequestHeaderProto getRequestHeader() throws IOException {
- if (lazyHeader == null) {
- lazyHeader = getValue(RequestHeaderProto.getDefaultInstance());
+ RpcProtobufRequest(RequestHeaderProto header, Message payload) {
+ this.requestHeader = header;
+ this.payload = payload;
+ }
+
+ RequestHeaderProto getRequestHeader() throws IOException {
+ if (getByteBuffer() != null && requestHeader == null) {
+ requestHeader = getValue(RequestHeaderProto.getDefaultInstance());
+ }
+ return requestHeader;
+ }
+
+ @Override
+ public void writeTo(ResponseBuffer out) throws IOException {
+ requestHeader.writeDelimitedTo(out);
+ if (payload != null) {
+ payload.writeDelimitedTo(out);
}
- return lazyHeader;
}
// this is used by htrace to name the span.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
index ac96a24..a789d83 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ResponseBuffer.java
@@ -27,8 +27,14 @@ import java.util.Arrays;
import org.apache.hadoop.classification.InterfaceAudience;
@InterfaceAudience.Private
-class ResponseBuffer extends DataOutputStream {
- ResponseBuffer(int capacity) {
+/** generates byte-length framed buffers. */
+public class ResponseBuffer extends DataOutputStream {
+
+ public ResponseBuffer() {
+ this(1024);
+ }
+
+ public ResponseBuffer(int capacity) {
super(new FramedBuffer(capacity));
}
@@ -39,7 +45,7 @@ class ResponseBuffer extends DataOutputStream {
return buf;
}
- void writeTo(OutputStream out) throws IOException {
+ public void writeTo(OutputStream out) throws IOException {
getFramedBuffer().writeTo(out);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
index 5125939..54fb98e 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/RpcWritable.java
@@ -24,7 +24,6 @@ import java.io.DataInputStream;
import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
-
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -34,6 +33,7 @@ import com.google.protobuf.CodedInputStream;
import com.google.protobuf.CodedOutputStream;
import com.google.protobuf.Message;
+// note anything marked public is solely for access by SaslRpcClient
@InterfaceAudience.Private
public abstract class RpcWritable implements Writable {
@@ -99,6 +99,10 @@ public abstract class RpcWritable implements Writable {
this.message = message;
}
+ Message getMessage() {
+ return message;
+ }
+
@Override
void writeTo(ResponseBuffer out) throws IOException {
int length = message.getSerializedSize();
@@ -128,11 +132,13 @@ public abstract class RpcWritable implements Writable {
}
}
- // adapter to allow decoding of writables and protobufs from a byte buffer.
- static class Buffer extends RpcWritable {
+ /**
+ * adapter to allow decoding of writables and protobufs from a byte buffer.
+ */
+ public static class Buffer extends RpcWritable {
private ByteBuffer bb;
- static Buffer wrap(ByteBuffer bb) {
+ public static Buffer wrap(ByteBuffer bb) {
return new Buffer(bb);
}
@@ -142,6 +148,10 @@ public abstract class RpcWritable implements Writable {
this.bb = bb;
}
+ ByteBuffer getByteBuffer() {
+ return bb;
+ }
+
@Override
void writeTo(ResponseBuffer out) throws IOException {
out.ensureCapacity(bb.remaining());
@@ -177,7 +187,7 @@ public abstract class RpcWritable implements Writable {
return RpcWritable.wrap(value).readFrom(bb);
}
- int remaining() {
+ public int remaining() {
return bb.remaining();
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/23abb09c/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
index c360937..60ae3b0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/SaslRpcClient.java
@@ -53,11 +53,11 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.GlobPattern;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcRequestMessageWrapper;
-import org.apache.hadoop.ipc.ProtobufRpcEngine.RpcResponseMessageWrapper;
import org.apache.hadoop.ipc.RPC.RpcKind;
import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.ResponseBuffer;
import org.apache.hadoop.ipc.RpcConstants;
+import org.apache.hadoop.ipc.RpcWritable;
import org.apache.hadoop.ipc.Server.AuthProtocol;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto;
import org.apache.hadoop.ipc.protobuf.RpcHeaderProtos.RpcRequestHeaderProto.OperationProto;
@@ -368,11 +368,13 @@ public class SaslRpcClient {
// loop until sasl is complete or a rpc error occurs
boolean done = false;
do {
- int totalLen = inStream.readInt();
- RpcResponseMessageWrapper responseWrapper =
- new RpcResponseMessageWrapper();
- responseWrapper.readFields(inStream);
- RpcResponseHeaderProto header = responseWrapper.getMessageHeader();
+ int rpcLen = inStream.readInt();
+ ByteBuffer bb = ByteBuffer.allocate(rpcLen);
+ inStream.readFully(bb.array());
+
+ RpcWritable.Buffer saslPacket = RpcWritable.Buffer.wrap(bb);
+ RpcResponseHeaderProto header =
+ saslPacket.getValue(RpcResponseHeaderProto.getDefaultInstance());
switch (header.getStatus()) {
case ERROR: // might get a RPC error during
case FATAL:
@@ -380,15 +382,14 @@ public class SaslRpcClient {
header.getErrorMsg());
default: break;
}
- if (totalLen != responseWrapper.getLength()) {
- throw new SaslException("Received malformed response length");
- }
-
if (header.getCallId() != AuthProtocol.SASL.callId) {
throw new SaslException("Non-SASL response during negotiation");
}
RpcSaslProto saslMessage =
- RpcSaslProto.parseFrom(responseWrapper.getMessageBytes());
+ saslPacket.getValue(RpcSaslProto.getDefaultInstance());
+ if (saslPacket.remaining() > 0) {
+ throw new SaslException("Received malformed response length");
+ }
// handle sasl negotiation process
RpcSaslProto.Builder response = null;
switch (saslMessage.getState()) {
@@ -452,16 +453,16 @@ public class SaslRpcClient {
return authMethod;
}
- private void sendSaslMessage(DataOutputStream out, RpcSaslProto message)
+ private void sendSaslMessage(OutputStream out, RpcSaslProto message)
throws IOException {
if (LOG.isDebugEnabled()) {
LOG.debug("Sending sasl message "+message);
}
- RpcRequestMessageWrapper request =
- new RpcRequestMessageWrapper(saslHeader, message);
- out.writeInt(request.getLength());
- request.write(out);
- out.flush();
+ ResponseBuffer buf = new ResponseBuffer();
+ saslHeader.writeDelimitedTo(buf);
+ message.writeDelimitedTo(buf);
+ buf.writeTo(out);
+ out.flush();
}
/**
@@ -634,12 +635,8 @@ public class SaslRpcClient {
.setState(SaslState.WRAP)
.setToken(ByteString.copyFrom(buf, 0, buf.length))
.build();
- RpcRequestMessageWrapper request =
- new RpcRequestMessageWrapper(saslHeader, saslMessage);
- DataOutputStream dob = new DataOutputStream(out);
- dob.writeInt(request.getLength());
- request.write(dob);
- }
+ sendSaslMessage(out, saslMessage);
+ }
}
/** Release resources used by wrapped saslClient */
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/48] hadoop git commit: HDFS-6962. ACL inheritance conflicts with
umaskmode. Contributed by Chris Nauroth.
Posted by aw...@apache.org.
HDFS-6962. ACL inheritance conflicts with umaskmode. Contributed by Chris Nauroth.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f0d5382f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f0d5382f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f0d5382f
Branch: refs/heads/HADOOP-13341
Commit: f0d5382ff3e31a47d13e4cb6c3a244cca82b17ce
Parents: d37dc5d
Author: Chris Nauroth <cn...@apache.org>
Authored: Tue Sep 6 11:02:39 2016 -0700
Committer: Chris Nauroth <cn...@apache.org>
Committed: Tue Sep 6 11:02:39 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/fs/FileContext.java | 8 +-
.../java/org/apache/hadoop/fs/FileSystem.java | 7 +-
.../hadoop/fs/permission/FsCreateModes.java | 101 ++
.../hadoop/fs/permission/FsPermission.java | 14 +
.../hadoop/fs/permission/package-info.java | 25 +
.../java/org/apache/hadoop/hdfs/DFSClient.java | 5 +-
.../ClientNamenodeProtocolTranslatorPB.java | 14 +-
.../hadoop/hdfs/web/WebHdfsFileSystem.java | 16 +-
.../hdfs/web/resources/PermissionParam.java | 9 +-
.../web/resources/UnmaskedPermissionParam.java | 51 +
.../src/main/proto/ClientNamenodeProtocol.proto | 2 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 4 +
...tNamenodeProtocolServerSideTranslatorPB.java | 16 +-
.../datanode/web/webhdfs/ParameterParser.java | 7 +
.../datanode/web/webhdfs/WebHdfsHandler.java | 6 +-
.../hadoop/hdfs/server/namenode/AclStorage.java | 7 +-
.../hdfs/server/namenode/FSDirMkdirOp.java | 3 +-
.../hdfs/server/namenode/FSDirSymlinkOp.java | 3 +-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 5 +-
.../hdfs/server/namenode/FSDirectory.java | 72 +-
.../web/resources/NamenodeWebHdfsMethods.java | 49 +-
.../src/main/resources/hdfs-default.xml | 14 +-
.../src/site/markdown/HdfsPermissionsGuide.md | 6 +
.../java/org/apache/hadoop/cli/TestAclCLI.java | 6 +-
.../cli/TestAclCLIWithPosixAclInheritance.java | 45 +
.../hdfs/server/namenode/FSAclBaseTest.java | 91 ++
.../server/namenode/TestGetBlockLocations.java | 2 +-
.../snapshot/TestRenameWithSnapshots.java | 4 +-
.../testAclCLIWithPosixAclInheritance.xml | 1075 ++++++++++++++++++
29 files changed, 1613 insertions(+), 54 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
index f235773..160a63d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileContext.java
@@ -46,6 +46,7 @@ import org.apache.hadoop.fs.Options.CreateOpts;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_DEFAULT;
@@ -671,7 +672,7 @@ public class FileContext {
CreateOpts.Perms permOpt = CreateOpts.getOpt(CreateOpts.Perms.class, opts);
FsPermission permission = (permOpt != null) ? permOpt.getValue() :
FILE_DEFAULT_PERM;
- permission = permission.applyUMask(getUMask());
+ permission = FsCreateModes.applyUMask(permission, getUMask());
final CreateOpts[] updatedOpts =
CreateOpts.setOpt(CreateOpts.perms(permission), opts);
@@ -717,8 +718,9 @@ public class FileContext {
ParentNotDirectoryException, UnsupportedFileSystemException,
IOException {
final Path absDir = fixRelativePart(dir);
- final FsPermission absFerms = (permission == null ?
- FsPermission.getDirDefault() : permission).applyUMask(getUMask());
+ final FsPermission absFerms = FsCreateModes.applyUMask(
+ permission == null ?
+ FsPermission.getDirDefault() : permission, getUMask());
new FSLinkResolver<Void>() {
@Override
public Void next(final AbstractFileSystem fs, final Path p)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c366729..9bde29d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -54,6 +54,7 @@ import org.apache.hadoop.fs.Options.Rename;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.io.MultipleIOException;
import org.apache.hadoop.io.Text;
@@ -925,9 +926,9 @@ public abstract class FileSystem extends Configured implements Closeable {
long blockSize,
Progressable progress
) throws IOException {
- return this.create(f, FsPermission.getFileDefault().applyUMask(
- FsPermission.getUMask(getConf())), overwrite, bufferSize,
- replication, blockSize, progress);
+ return this.create(f, FsCreateModes.applyUMask(
+ FsPermission.getFileDefault(), FsPermission.getUMask(getConf())),
+ overwrite, bufferSize, replication, blockSize, progress);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
new file mode 100644
index 0000000..a1ed0d7
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsCreateModes.java
@@ -0,0 +1,101 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.fs.permission;
+
+import java.text.MessageFormat;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * A class that stores both masked and unmasked create modes
+ * and is a drop-in replacement for masked permission.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public final class FsCreateModes extends FsPermission {
+ private final FsPermission unmasked;
+
+ /**
+ * Create from unmasked mode and umask.
+ *
+ * If the mode is already an FsCreateModes object, return it.
+ */
+ public static FsPermission applyUMask(FsPermission mode,
+ FsPermission umask) {
+ if (mode.getUnmasked() != null) {
+ return mode;
+ }
+ return create(mode.applyUMask(umask), mode);
+ }
+
+ /**
+ * Create from masked and unmasked modes.
+ */
+ public static FsCreateModes create(FsPermission masked,
+ FsPermission unmasked) {
+ assert masked.getUnmasked() == null;
+ assert unmasked.getUnmasked() == null;
+ return new FsCreateModes(masked, unmasked);
+ }
+
+ private FsCreateModes(FsPermission masked, FsPermission unmasked) {
+ super(masked);
+ this.unmasked = unmasked;
+ assert masked.getUnmasked() == null;
+ assert unmasked.getUnmasked() == null;
+ }
+
+ @Override
+ public FsPermission getMasked() {
+ return this;
+ }
+
+ @Override
+ public FsPermission getUnmasked() {
+ return unmasked;
+ }
+
+ @Override
+ public String toString() {
+ return MessageFormat.format("'{' masked: {0}, unmasked: {1} '}'",
+ super.toString(), getUnmasked());
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ if (!super.equals(o)) {
+ return false;
+ }
+ FsCreateModes that = (FsCreateModes) o;
+ return getUnmasked().equals(that.getUnmasked());
+ }
+
+ @Override
+ public int hashCode() {
+ int result = super.hashCode();
+ result = 31 * result + getUnmasked().hashCode();
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 78255bbd..48a5b1c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -138,6 +138,20 @@ public class FsPermission implements Writable {
}
/**
+ * Get masked permission if exists.
+ */
+ public FsPermission getMasked() {
+ return null;
+ }
+
+ /**
+ * Get unmasked permission if exists.
+ */
+ public FsPermission getUnmasked() {
+ return null;
+ }
+
+ /**
* Create and initialize a {@link FsPermission} from {@link DataInput}.
*/
public static FsPermission read(DataInput in) throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
new file mode 100644
index 0000000..6cee4ee
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/package-info.java
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/**
+ * This package provides support for HDFS permission and ACL.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+package org.apache.hadoop.fs.permission;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index ad86895..b0fcba2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -90,6 +90,7 @@ import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.NameNodeProxiesClient.ProxyAndInfo;
import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
@@ -1160,14 +1161,14 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
if (permission == null) {
permission = FsPermission.getFileDefault();
}
- return permission.applyUMask(dfsClientConf.getUMask());
+ return FsCreateModes.applyUMask(permission, dfsClientConf.getUMask());
}
private FsPermission applyUMaskDir(FsPermission permission) {
if (permission == null) {
permission = FsPermission.getDirDefault();
}
- return permission.applyUMask(dfsClientConf.getUMask());
+ return FsCreateModes.applyUMask(permission, dfsClientConf.getUMask());
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index 57f8fd6..f73abfd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -294,6 +294,10 @@ public class ClientNamenodeProtocolTranslatorPB implements
.setCreateParent(createParent)
.setReplication(replication)
.setBlockSize(blockSize);
+ FsPermission unmasked = masked.getUnmasked();
+ if (unmasked != null) {
+ builder.setUnmasked(PBHelperClient.convert(unmasked));
+ }
builder.addAllCryptoProtocolVersion(
PBHelperClient.convert(supportedVersions));
CreateRequestProto req = builder.build();
@@ -579,11 +583,15 @@ public class ClientNamenodeProtocolTranslatorPB implements
@Override
public boolean mkdirs(String src, FsPermission masked, boolean createParent)
throws IOException {
- MkdirsRequestProto req = MkdirsRequestProto.newBuilder()
+ MkdirsRequestProto.Builder builder = MkdirsRequestProto.newBuilder()
.setSrc(src)
.setMasked(PBHelperClient.convert(masked))
- .setCreateParent(createParent).build();
-
+ .setCreateParent(createParent);
+ FsPermission unmasked = masked.getUnmasked();
+ if (unmasked != null) {
+ builder.setUnmasked(PBHelperClient.convert(unmasked));
+ }
+ MkdirsRequestProto req = builder.build();
try {
return rpcProxy.mkdirs(null, req).getResult();
} catch (ServiceException e) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
index 968eea3..5389a02 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/WebHdfsFileSystem.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.fs.GlobalStorageStatistics;
import org.apache.hadoop.fs.GlobalStorageStatistics.StorageStatisticsProvider;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.StorageStatistics;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics;
import org.apache.hadoop.hdfs.DFSOpsCountStatistics.OpType;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
@@ -973,7 +974,8 @@ public class WebHdfsFileSystem extends FileSystem
if (permission == null) {
permission = FsPermission.getDefault();
}
- return permission.applyUMask(FsPermission.getUMask(getConf()));
+ return FsCreateModes.applyUMask(permission,
+ FsPermission.getUMask(getConf()));
}
private HdfsFileStatus getHdfsFileStatus(Path f) throws IOException {
@@ -1025,8 +1027,10 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.MKDIRS);
final HttpOpParam.Op op = PutOpParam.Op.MKDIRS;
+ final FsPermission modes = applyUMask(permission);
return new FsPathBooleanRunner(op, f,
- new PermissionParam(applyUMask(permission))
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked())
).run();
}
@@ -1313,9 +1317,11 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.CREATE);
+ final FsPermission modes = applyUMask(permission);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
return new FsPathOutputStreamRunner(op, f, bufferSize,
- new PermissionParam(applyUMask(permission)),
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked()),
new OverwriteParam(overwrite),
new BufferSizeParam(bufferSize),
new ReplicationParam(replication),
@@ -1331,9 +1337,11 @@ public class WebHdfsFileSystem extends FileSystem
statistics.incrementWriteOps(1);
storageStatistics.incrementOpCounter(OpType.CREATE_NON_RECURSIVE);
+ final FsPermission modes = applyUMask(permission);
final HttpOpParam.Op op = PutOpParam.Op.CREATE;
return new FsPathOutputStreamRunner(op, f, bufferSize,
- new PermissionParam(applyUMask(permission)),
+ new PermissionParam(modes.getMasked()),
+ new UnmaskedPermissionParam(modes.getUnmasked()),
new CreateFlagParam(flag),
new CreateParentParam(false),
new BufferSizeParam(bufferSize),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
index 42ff69d..1598cbc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/PermissionParam.java
@@ -54,7 +54,7 @@ public class PermissionParam extends ShortParam {
* @param value the parameter value.
*/
public PermissionParam(final FsPermission value) {
- super(DOMAIN, value == null? null: value.toShort(), null, null);
+ this(DOMAIN, value == null ? null : value.toShort(), null, null);
}
/**
@@ -62,7 +62,12 @@ public class PermissionParam extends ShortParam {
* @param str a string representation of the parameter value.
*/
public PermissionParam(final String str) {
- super(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ this(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ }
+
+ PermissionParam(final Domain domain, final Short value, final Short min,
+ final Short max) {
+ super(domain, value, min, max);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
new file mode 100644
index 0000000..bbd2a5c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/web/resources/UnmaskedPermissionParam.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.web.resources;
+
+import org.apache.hadoop.fs.permission.FsPermission;
+
+/**
+ * Unmasked permission parameter, use a Short to represent a FsPermission.
+ */
+public class UnmaskedPermissionParam extends PermissionParam {
+ /** Parameter name. */
+ public static final String NAME = "unmaskedpermission";
+
+ private static final Domain DOMAIN = new Domain(NAME, 8);
+
+ /**
+ * Constructor.
+ * @param value the parameter value.
+ */
+ public UnmaskedPermissionParam(final FsPermission value) {
+ super(DOMAIN, value == null ? null : value.toShort(), null, null);
+ }
+
+ /**
+ * Constructor.
+ * @param str a string representation of the parameter value.
+ */
+ public UnmaskedPermissionParam(final String str) {
+ super(DOMAIN, DOMAIN.parse(str), (short)0, (short)01777);
+ }
+
+ @Override
+ public String getName() {
+ return NAME;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 83f5828..54bed32 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -79,6 +79,7 @@ message CreateRequestProto {
required uint32 replication = 6; // Short: Only 16 bits used
required uint64 blockSize = 7;
repeated CryptoProtocolVersionProto cryptoProtocolVersion = 8;
+ optional FsPermissionProto unmasked = 9;
}
message CreateResponseProto {
@@ -264,6 +265,7 @@ message MkdirsRequestProto {
required string src = 1;
required FsPermissionProto masked = 2;
required bool createParent = 3;
+ optional FsPermissionProto unmasked = 4;
}
message MkdirsResponseProto {
required bool result = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f2ab321..7bdfd44 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -259,6 +259,10 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_PERMISSIONS_SUPERUSERGROUP_DEFAULT = "supergroup";
public static final String DFS_NAMENODE_ACLS_ENABLED_KEY = "dfs.namenode.acls.enabled";
public static final boolean DFS_NAMENODE_ACLS_ENABLED_DEFAULT = false;
+ public static final String DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY =
+ "dfs.namenode.posix.acl.inheritance.enabled";
+ public static final boolean
+ DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT = false;
public static final String DFS_NAMENODE_XATTRS_ENABLED_KEY = "dfs.namenode.xattrs.enabled";
public static final boolean DFS_NAMENODE_XATTRS_ENABLED_DEFAULT = true;
public static final String DFS_ADMIN = "dfs.cluster.administrators";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 641cd20..3974956 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -25,6 +25,8 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedEntries;
+import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.ContentSummary;
import org.apache.hadoop.fs.CreateFlag;
@@ -411,8 +413,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public CreateResponseProto create(RpcController controller,
CreateRequestProto req) throws ServiceException {
try {
+ FsPermission masked = req.hasUnmasked() ?
+ FsCreateModes.create(PBHelperClient.convert(req.getMasked()),
+ PBHelperClient.convert(req.getUnmasked())) :
+ PBHelperClient.convert(req.getMasked());
HdfsFileStatus result = server.create(req.getSrc(),
- PBHelperClient.convert(req.getMasked()), req.getClientName(),
+ masked, req.getClientName(),
PBHelperClient.convertCreateFlag(req.getCreateFlag()), req.getCreateParent(),
(short) req.getReplication(), req.getBlockSize(),
PBHelperClient.convertCryptoProtocolVersions(
@@ -651,8 +657,12 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
public MkdirsResponseProto mkdirs(RpcController controller,
MkdirsRequestProto req) throws ServiceException {
try {
- boolean result = server.mkdirs(req.getSrc(),
- PBHelperClient.convert(req.getMasked()), req.getCreateParent());
+ FsPermission masked = req.hasUnmasked() ?
+ FsCreateModes.create(PBHelperClient.convert(req.getMasked()),
+ PBHelperClient.convert(req.getUnmasked())) :
+ PBHelperClient.convert(req.getMasked());
+ boolean result = server.mkdirs(req.getSrc(), masked,
+ req.getCreateParent());
return MkdirsResponseProto.newBuilder().setResult(result).build();
} catch (IOException e) {
throw new ServiceException(e);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
index 3c7016e..bb28110 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/ParameterParser.java
@@ -37,6 +37,7 @@ import org.apache.hadoop.hdfs.web.resources.OffsetParam;
import org.apache.hadoop.hdfs.web.resources.OverwriteParam;
import org.apache.hadoop.hdfs.web.resources.PermissionParam;
import org.apache.hadoop.hdfs.web.resources.ReplicationParam;
+import org.apache.hadoop.hdfs.web.resources.UnmaskedPermissionParam;
import org.apache.hadoop.hdfs.web.resources.UserParam;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.token.Token;
@@ -108,6 +109,12 @@ class ParameterParser {
getFileFsPermission();
}
+ FsPermission unmaskedPermission() {
+ String value = param(UnmaskedPermissionParam.NAME);
+ return value == null ? null :
+ new UnmaskedPermissionParam(value).getFileFsPermission();
+ }
+
boolean overwrite() {
return new OverwriteParam(param(OverwriteParam.NAME)).getValue();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
index d354612..20e0a2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/web/webhdfs/WebHdfsHandler.java
@@ -66,6 +66,7 @@ import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.MD5MD5CRC32FileChecksum;
+import org.apache.hadoop.fs.permission.FsCreateModes;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.client.HdfsDataInputStream;
@@ -187,7 +188,10 @@ public class WebHdfsHandler extends SimpleChannelInboundHandler<HttpRequest> {
final int bufferSize = params.bufferSize();
final short replication = params.replication();
final long blockSize = params.blockSize();
- final FsPermission permission = params.permission();
+ final FsPermission unmaskedPermission = params.unmaskedPermission();
+ final FsPermission permission = unmaskedPermission == null ?
+ params.permission() :
+ FsCreateModes.create(params.permission(), unmaskedPermission);
final boolean createParent = params.createParent();
EnumSet<CreateFlag> flags = params.createFlag();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
index abd3755..58dfe12 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/AclStorage.java
@@ -73,11 +73,11 @@ public final class AclStorage {
*
* @param child INode newly created child
*/
- public static void copyINodeDefaultAcl(INode child) {
+ public static boolean copyINodeDefaultAcl(INode child) {
INodeDirectory parent = child.getParent();
AclFeature parentAclFeature = parent.getAclFeature();
if (parentAclFeature == null || !(child.isFile() || child.isDirectory())) {
- return;
+ return false;
}
// Split parent's entries into access vs. default.
@@ -88,7 +88,7 @@ public final class AclStorage {
// The parent may have an access ACL but no default ACL. If so, exit.
if (parentDefaultEntries.isEmpty()) {
- return;
+ return false;
}
// Pre-allocate list size for access entries to copy from parent.
@@ -145,6 +145,7 @@ public final class AclStorage {
}
child.setPermission(newPerm);
+ return true;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
index bf5ff00..2d1914f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirMkdirOp.java
@@ -219,7 +219,8 @@ class FSDirMkdirOp {
final INodeDirectory dir = new INodeDirectory(inodeId, name, permission,
timestamp);
- INodesInPath iip = fsd.addLastINode(parent, dir, true);
+ INodesInPath iip =
+ fsd.addLastINode(parent, dir, permission.getPermission(), true);
if (iip != null && aclEntries != null) {
AclStorage.updateINodeAcl(dir, aclEntries, Snapshot.CURRENT_STATE_ID);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
index ec93952..6938a84 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirSymlinkOp.java
@@ -88,7 +88,8 @@ class FSDirSymlinkOp {
final INodeSymlink symlink = new INodeSymlink(id, null, perm, mtime, atime,
target);
symlink.setLocalName(localName);
- return fsd.addINode(iip, symlink) != null ? symlink : null;
+ return fsd.addINode(iip, symlink, perm.getPermission()) != null ?
+ symlink : null;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 077f04f..aa2be92 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -496,7 +496,8 @@ class FSDirWriteFileOp {
replication, preferredBlockSize, storagePolicyId, ecPolicy != null);
}
newNode.setLocalName(localName);
- INodesInPath iip = fsd.addINode(existing, newNode);
+ INodesInPath iip = fsd.addINode(existing, newNode,
+ permissions.getPermission());
if (iip != null) {
if (aclEntries != null) {
AclStorage.updateINodeAcl(newNode, aclEntries, CURRENT_STATE_ID);
@@ -594,7 +595,7 @@ class FSDirWriteFileOp {
modTime, modTime, replication, preferredBlockSize, ecPolicy != null);
newNode.setLocalName(localName);
newNode.toUnderConstruction(clientName, clientMachine);
- newiip = fsd.addINode(existing, newNode);
+ newiip = fsd.addINode(existing, newNode, permissions.getPermission());
} finally {
fsd.writeUnlock();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 5dc5a03..340d491 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -168,6 +168,10 @@ public class FSDirectory implements Closeable {
* ACL-related operations.
*/
private final boolean aclsEnabled;
+ /**
+ * Support for POSIX ACL inheritance. Not final for testing purpose.
+ */
+ private boolean posixAclInheritanceEnabled;
private final boolean xattrsEnabled;
private final int xattrMaxSize;
@@ -251,6 +255,10 @@ public class FSDirectory implements Closeable {
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_DEFAULT);
LOG.info("ACLs enabled? " + aclsEnabled);
+ this.posixAclInheritanceEnabled = conf.getBoolean(
+ DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY,
+ DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_DEFAULT);
+ LOG.info("POSIX ACL inheritance enabled? " + posixAclInheritanceEnabled);
this.xattrsEnabled = conf.getBoolean(
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_KEY,
DFSConfigKeys.DFS_NAMENODE_XATTRS_ENABLED_DEFAULT);
@@ -450,6 +458,18 @@ public class FSDirectory implements Closeable {
boolean isAclsEnabled() {
return aclsEnabled;
}
+
+ @VisibleForTesting
+ public boolean isPosixAclInheritanceEnabled() {
+ return posixAclInheritanceEnabled;
+ }
+
+ @VisibleForTesting
+ public void setPosixAclInheritanceEnabled(
+ boolean posixAclInheritanceEnabled) {
+ this.posixAclInheritanceEnabled = posixAclInheritanceEnabled;
+ }
+
boolean isXattrsEnabled() {
return xattrsEnabled;
}
@@ -952,16 +972,18 @@ public class FSDirectory implements Closeable {
* Add the given child to the namespace.
* @param existing the INodesInPath containing all the ancestral INodes
* @param child the new INode to add
+ * @param modes create modes
* @return a new INodesInPath instance containing the new child INode. Null
* if the adding fails.
* @throws QuotaExceededException is thrown if it violates quota limit
*/
- INodesInPath addINode(INodesInPath existing, INode child)
+ INodesInPath addINode(INodesInPath existing, INode child,
+ FsPermission modes)
throws QuotaExceededException, UnresolvedLinkException {
cacheName(child);
writeLock();
try {
- return addLastINode(existing, child, true);
+ return addLastINode(existing, child, modes, true);
} finally {
writeUnlock();
}
@@ -1067,12 +1089,51 @@ public class FSDirectory implements Closeable {
}
/**
+ * Turn on HDFS-6962 POSIX ACL inheritance when the property
+ * {@link DFSConfigKeys#DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY} is
+ * true and a compatible client has sent both masked and unmasked create
+ * modes.
+ *
+ * @param child INode newly created child
+ * @param modes create modes
+ */
+ private void copyINodeDefaultAcl(INode child, FsPermission modes) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("child: {}, posixAclInheritanceEnabled: {}, modes: {}",
+ child, posixAclInheritanceEnabled, modes);
+ }
+
+ if (posixAclInheritanceEnabled && modes != null &&
+ modes.getUnmasked() != null) {
+ //
+ // HDFS-6962: POSIX ACL inheritance
+ //
+ child.setPermission(modes.getUnmasked());
+ if (!AclStorage.copyINodeDefaultAcl(child)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("{}: no parent default ACL to inherit", child);
+ }
+ child.setPermission(modes.getMasked());
+ }
+ } else {
+ //
+ // Old behavior before HDFS-6962
+ //
+ AclStorage.copyINodeDefaultAcl(child);
+ }
+ }
+
+ /**
* Add a child to the end of the path specified by INodesInPath.
+ * @param existing the INodesInPath containing all the ancestral INodes
+ * @param inode the new INode to add
+ * @param modes create modes
+ * @param checkQuota whether to check quota
* @return an INodesInPath instance containing the new INode
*/
@VisibleForTesting
public INodesInPath addLastINode(INodesInPath existing, INode inode,
- boolean checkQuota) throws QuotaExceededException {
+ FsPermission modes, boolean checkQuota) throws QuotaExceededException {
assert existing.getLastINode() != null &&
existing.getLastINode().isDirectory();
@@ -1119,7 +1180,7 @@ public class FSDirectory implements Closeable {
return null;
} else {
if (!isRename) {
- AclStorage.copyINodeDefaultAcl(inode);
+ copyINodeDefaultAcl(inode, modes);
}
addToInodeMap(inode);
}
@@ -1128,7 +1189,8 @@ public class FSDirectory implements Closeable {
INodesInPath addLastINodeNoQuotaCheck(INodesInPath existing, INode i) {
try {
- return addLastINode(existing, i, false);
+ // All callers do not have create modes to pass.
+ return addLastINode(existing, i, null, false);
} catch (QuotaExceededException e) {
NameNode.LOG.warn("FSDirectory.addChildNoQuotaCheck - unexpected", e);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
index e3a5bad..3ab0c67 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/web/resources/NamenodeWebHdfsMethods.java
@@ -58,6 +58,8 @@ import org.apache.hadoop.fs.Options;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.fs.permission.FsCreateModes;
+import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -324,6 +326,9 @@ public class NamenodeWebHdfsMethods {
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
+ @QueryParam(UnmaskedPermissionParam.NAME)
+ @DefaultValue(UnmaskedPermissionParam.DEFAULT)
+ final UnmaskedPermissionParam unmaskedPermission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
@@ -362,11 +367,11 @@ public class NamenodeWebHdfsMethods {
final NoRedirectParam noredirect
) throws IOException, InterruptedException {
return put(ugi, delegation, username, doAsUser, ROOT, op, destination,
- owner, group, permission, overwrite, bufferSize, replication,
- blockSize, modificationTime, accessTime, renameOptions, createParent,
- delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
- createFlagParam, noredirect);
+ owner, group, permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime, renameOptions,
+ createParent, delegationTokenArgument, aclPermission, xattrName,
+ xattrValue, xattrSetFlag, snapshotName, oldSnapshotName,
+ excludeDatanodes, createFlagParam, noredirect);
}
/** Handle HTTP PUT request. */
@@ -393,6 +398,9 @@ public class NamenodeWebHdfsMethods {
final GroupParam group,
@QueryParam(PermissionParam.NAME) @DefaultValue(PermissionParam.DEFAULT)
final PermissionParam permission,
+ @QueryParam(UnmaskedPermissionParam.NAME)
+ @DefaultValue(UnmaskedPermissionParam.DEFAULT)
+ final UnmaskedPermissionParam unmaskedPermission,
@QueryParam(OverwriteParam.NAME) @DefaultValue(OverwriteParam.DEFAULT)
final OverwriteParam overwrite,
@QueryParam(BufferSizeParam.NAME) @DefaultValue(BufferSizeParam.DEFAULT)
@@ -432,10 +440,11 @@ public class NamenodeWebHdfsMethods {
) throws IOException, InterruptedException {
init(ugi, delegation, username, doAsUser, path, op, destination, owner,
- group, permission, overwrite, bufferSize, replication, blockSize,
- modificationTime, accessTime, renameOptions, delegationTokenArgument,
- aclPermission, xattrName, xattrValue, xattrSetFlag, snapshotName,
- oldSnapshotName, excludeDatanodes, createFlagParam, noredirect);
+ group, permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime, renameOptions,
+ delegationTokenArgument, aclPermission, xattrName, xattrValue,
+ xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ createFlagParam, noredirect);
return ugi.doAs(new PrivilegedExceptionAction<Response>() {
@Override
@@ -443,10 +452,11 @@ public class NamenodeWebHdfsMethods {
try {
return put(ugi, delegation, username, doAsUser,
path.getAbsolutePath(), op, destination, owner, group,
- permission, overwrite, bufferSize, replication, blockSize,
- modificationTime, accessTime, renameOptions, createParent,
- delegationTokenArgument, aclPermission, xattrName, xattrValue,
- xattrSetFlag, snapshotName, oldSnapshotName, excludeDatanodes,
+ permission, unmaskedPermission, overwrite, bufferSize,
+ replication, blockSize, modificationTime, accessTime,
+ renameOptions, createParent, delegationTokenArgument,
+ aclPermission, xattrName, xattrValue, xattrSetFlag,
+ snapshotName, oldSnapshotName, excludeDatanodes,
createFlagParam, noredirect);
} finally {
reset();
@@ -466,6 +476,7 @@ public class NamenodeWebHdfsMethods {
final OwnerParam owner,
final GroupParam group,
final PermissionParam permission,
+ final UnmaskedPermissionParam unmaskedPermission,
final OverwriteParam overwrite,
final BufferSizeParam bufferSize,
final ReplicationParam replication,
@@ -495,8 +506,9 @@ public class NamenodeWebHdfsMethods {
{
final URI uri = redirectURI(namenode, ugi, delegation, username,
doAsUser, fullpath, op.getValue(), -1L, blockSize.getValue(conf),
- exclDatanodes.getValue(), permission, overwrite, bufferSize,
- replication, blockSize, createParent, createFlagParam);
+ exclDatanodes.getValue(), permission, unmaskedPermission,
+ overwrite, bufferSize, replication, blockSize, createParent,
+ createFlagParam);
if(!noredirectParam.getValue()) {
return Response.temporaryRedirect(uri)
.type(MediaType.APPLICATION_OCTET_STREAM).build();
@@ -507,8 +519,11 @@ public class NamenodeWebHdfsMethods {
}
case MKDIRS:
{
- final boolean b = np.mkdirs(fullpath,
- permission.getDirFsPermission(), true);
+ FsPermission masked = unmaskedPermission.getValue() == null ?
+ permission.getDirFsPermission() :
+ FsCreateModes.create(permission.getDirFsPermission(),
+ unmaskedPermission.getDirFsPermission());
+ final boolean b = np.mkdirs(fullpath, masked, true);
final String js = JsonUtil.toJsonString("boolean", b);
return Response.ok(js).type(MediaType.APPLICATION_JSON).build();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 4eca80c..d427f88 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -457,7 +457,19 @@
</description>
</property>
-<property>
+ <property>
+ <name>dfs.namenode.posix.acl.inheritance.enabled</name>
+ <value>false</value>
+ <description>
+ Set to true to enable POSIX style ACL inheritance. When it is enabled
+ and the create request comes from a compatible client, the NameNode
+ will apply default ACLs from the parent directory to the create mode
+ and ignore the client umask. If no default ACL found, it will apply the
+ client umask.
+ </description>
+ </property>
+
+ <property>
<name>dfs.namenode.lazypersist.file.scrub.interval.sec</name>
<value>300</value>
<description>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
index ba027ac..7fd9b00 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HdfsPermissionsGuide.md
@@ -333,4 +333,10 @@ Configuration Parameters
default, ACLs are disabled. When ACLs are disabled, the NameNode rejects
all attempts to set an ACL.
+* `dfs.namenode.posix.acl.inheritance.enabled`
+ Set to true to enable POSIX style ACL inheritance. Disabled by default.
+ When it is enabled and the create request comes from a compatible client,
+ the NameNode will apply default ACLs from the parent directory to
+ the create mode and ignore the client umask. If no default ACL is found,
+ it will apply the client umask.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
index 061d328..75111bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLI.java
@@ -32,11 +32,15 @@ public class TestAclCLI extends CLITestHelperDFS {
private String namenode = null;
private String username = null;
+ protected void initConf() {
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ }
+
@Before
@Override
public void setUp() throws Exception {
super.setUp();
- conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_ACLS_ENABLED_KEY, true);
+ initConf();
cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
fs = cluster.getFileSystem();
namenode = conf.get(DFSConfigKeys.FS_DEFAULT_NAME_KEY, "file:///");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
new file mode 100644
index 0000000..ec31766
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/cli/TestAclCLIWithPosixAclInheritance.java
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.cli;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY;
+
+import org.junit.Test;
+
+/**
+ * Test ACL CLI with POSIX ACL inheritance enabled.
+ */
+public class TestAclCLIWithPosixAclInheritance extends TestAclCLI {
+
+ @Override
+ protected void initConf() {
+ super.initConf();
+ conf.setBoolean(DFS_NAMENODE_POSIX_ACL_INHERITANCE_ENABLED_KEY, true);
+ }
+
+ @Override
+ protected String getTestFile() {
+ return "testAclCLIWithPosixAclInheritance.xml";
+ }
+
+ @Test
+ @Override
+ public void testAll() {
+ super.testAll();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
index 216147a..71614f6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hdfs.server.namenode;
+import static org.apache.hadoop.fs.CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY;
import static org.apache.hadoop.hdfs.server.namenode.AclTestHelpers.*;
import static org.apache.hadoop.fs.permission.AclEntryScope.*;
import static org.apache.hadoop.fs.permission.AclEntryType.*;
@@ -889,6 +890,46 @@ public abstract class FSAclBaseTest {
}
@Test
+ public void testUMaskDefaultAclNewFile() throws Exception {
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, READ_WRITE),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ String oldUMask = fs.getConf().get(FS_PERMISSIONS_UMASK_KEY);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, "027");
+
+ FSDirectory fsDirectory = cluster.getNamesystem().getFSDirectory();
+ boolean oldEnabled = fsDirectory.isPosixAclInheritanceEnabled();
+
+ try {
+ fsDirectory.setPosixAclInheritanceEnabled(false);
+ Path filePath = new Path(path, "file1");
+ fs.create(filePath).close();
+ AclStatus s = fs.getAclStatus(filePath);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_WRITE)}, returned);
+ assertPermission(filePath, (short) 010640);
+
+ fsDirectory.setPosixAclInheritanceEnabled(true);
+ Path file2Path = new Path(path, "file2");
+ fs.create(file2Path).close();
+ AclStatus s2 = fs.getAclStatus(file2Path);
+ AclEntry[] returned2 = s2.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, READ_WRITE)}, returned2);
+ assertPermission(file2Path, (short) 010660);
+ } finally {
+ fsDirectory.setPosixAclInheritanceEnabled(oldEnabled);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, oldUMask);
+ }
+ }
+
+ @Test
public void testOnlyAccessAclNewFile() throws Exception {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
List<AclEntry> aclSpec = Lists.newArrayList(
@@ -943,6 +984,56 @@ public abstract class FSAclBaseTest {
}
@Test
+ public void testUMaskDefaultAclNewDir() throws Exception {
+ FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
+ List<AclEntry> aclSpec = Lists.newArrayList(
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL));
+ fs.setAcl(path, aclSpec);
+
+ String oldUMask = fs.getConf().get(FS_PERMISSIONS_UMASK_KEY);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, "027");
+
+ FSDirectory fsDirectory = cluster.getNamesystem().getFSDirectory();
+ boolean oldEnabled = fsDirectory.isPosixAclInheritanceEnabled();
+
+ try {
+ fsDirectory.setPosixAclInheritanceEnabled(false);
+ Path dirPath = new Path(path, "dir1");
+ fs.mkdirs(dirPath);
+ AclStatus s = fs.getAclStatus(dirPath);
+ AclEntry[] returned = s.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned);
+ assertPermission(dirPath, (short) 010750);
+
+ fsDirectory.setPosixAclInheritanceEnabled(true);
+ Path dir2Path = new Path(path, "dir2");
+ fs.mkdirs(dir2Path);
+ AclStatus s2 = fs.getAclStatus(dir2Path);
+ AclEntry[] returned2 = s2.getEntries().toArray(new AclEntry[0]);
+ assertArrayEquals(new AclEntry[]{
+ aclEntry(ACCESS, USER, "foo", ALL),
+ aclEntry(ACCESS, GROUP, ALL),
+ aclEntry(DEFAULT, USER, ALL),
+ aclEntry(DEFAULT, USER, "foo", ALL),
+ aclEntry(DEFAULT, GROUP, ALL),
+ aclEntry(DEFAULT, MASK, ALL),
+ aclEntry(DEFAULT, OTHER, NONE)}, returned2);
+ assertPermission(dir2Path, (short) 010770);
+ } finally {
+ fsDirectory.setPosixAclInheritanceEnabled(oldEnabled);
+ fs.getConf().set(FS_PERMISSIONS_UMASK_KEY, oldUMask);
+ }
+ }
+
+ @Test
public void testOnlyAccessAclNewDir() throws Exception {
FileSystem.mkdirs(fs, path, FsPermission.createImmutable((short)0750));
List<AclEntry> aclSpec = Lists.newArrayList(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
index eec5c98..0eb7132 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestGetBlockLocations.java
@@ -127,7 +127,7 @@ public class TestGetBlockLocations {
MOCK_INODE_ID, FILE_NAME.getBytes(StandardCharsets.UTF_8),
perm, 1, 1, new BlockInfo[] {}, (short) 1,
DFS_BLOCK_SIZE_DEFAULT);
- fsn.getFSDirectory().addINode(iip, file);
+ fsn.getFSDirectory().addINode(iip, file, null);
return fsn;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f0d5382f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
index ad3a5a1..91eec78 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestRenameWithSnapshots.java
@@ -1570,7 +1570,9 @@ public class TestRenameWithSnapshots {
FSDirectory fsdir2 = Mockito.spy(fsdir);
Mockito.doThrow(new NSQuotaExceededException("fake exception")).when(fsdir2)
.addLastINode((INodesInPath) Mockito.anyObject(),
- (INode) Mockito.anyObject(), Mockito.anyBoolean());
+ (INode) Mockito.anyObject(),
+ (FsPermission) Mockito.anyObject(),
+ Mockito.anyBoolean());
Whitebox.setInternalState(fsn, "dir", fsdir2);
// rename /test/dir1/foo to /test/dir2/subdir2/foo.
// FSDirectory#verifyQuota4Rename will pass since the remaining quota is 2.
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/48] hadoop git commit: YARN-5616. Clean up WeightAdjuster. (Yufei
Gu via kasha)
Posted by aw...@apache.org.
YARN-5616. Clean up WeightAdjuster. (Yufei Gu via kasha)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c0e492e5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c0e492e5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c0e492e5
Branch: refs/heads/HADOOP-13341
Commit: c0e492e50fa98d423c8a61c94d2d1f9553558b6d
Parents: 680be58
Author: Karthik Kambatla <ka...@apache.org>
Authored: Tue Sep 6 22:40:20 2016 -0700
Committer: Karthik Kambatla <ka...@apache.org>
Committed: Tue Sep 6 22:40:20 2016 -0700
----------------------------------------------------------------------
.../scheduler/fair/FairScheduler.java | 5 --
.../scheduler/fair/NewAppWeightBooster.java | 60 --------------------
.../scheduler/fair/WeightAdjuster.java | 36 ------------
3 files changed, 101 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index ac8dbe1..15e2318 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -186,7 +186,6 @@ public class FairScheduler extends
// an app can be reserved on
protected boolean sizeBasedWeight; // Give larger weights to larger jobs
- protected WeightAdjuster weightAdjuster; // Can be null for no weight adjuster
protected boolean continuousSchedulingEnabled; // Continuous Scheduling enabled or not
protected int continuousSchedulingSleepMs; // Sleep time for each pass in continuous scheduling
private Comparator<FSSchedulerNode> nodeAvailableResourceComparator =
@@ -563,10 +562,6 @@ public class FairScheduler extends
weight = Math.log1p(app.getDemand().getMemorySize()) / Math.log(2);
}
weight *= app.getPriority().getPriority();
- if (weightAdjuster != null) {
- // Run weight through the user-supplied weightAdjuster
- weight = weightAdjuster.adjustWeight(app, weight);
- }
ResourceWeights resourceWeights = app.getResourceWeights();
resourceWeights.setWeight((float)weight);
return resourceWeights;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
deleted file mode 100644
index fb32e56..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/NewAppWeightBooster.java
+++ /dev/null
@@ -1,60 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-
-/**
- * A {@link WeightAdjuster} implementation that gives a weight boost to new jobs
- * for a certain amount of time -- by default, a 3x weight boost for 60 seconds.
- * This can be used to make shorter jobs finish faster, emulating Shortest Job
- * First scheduling while not starving long jobs.
- */
-@Private
-@Unstable
-public class NewAppWeightBooster extends Configured implements WeightAdjuster {
- private static final float DEFAULT_FACTOR = 3;
- private static final long DEFAULT_DURATION = 5 * 60 * 1000;
-
- private float factor;
- private long duration;
-
- public void setConf(Configuration conf) {
- if (conf != null) {
- factor = conf.getFloat("mapred.newjobweightbooster.factor",
- DEFAULT_FACTOR);
- duration = conf.getLong("mapred.newjobweightbooster.duration",
- DEFAULT_DURATION);
- }
- super.setConf(conf);
- }
-
- public double adjustWeight(FSAppAttempt app, double curWeight) {
- long start = app.getStartTime();
- long now = System.currentTimeMillis();
- if (now - start < duration) {
- return curWeight * factor;
- } else {
- return curWeight;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c0e492e5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
deleted file mode 100644
index 67364ed..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/WeightAdjuster.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair;
-
-import org.apache.hadoop.classification.InterfaceAudience.Private;
-import org.apache.hadoop.classification.InterfaceStability.Unstable;
-import org.apache.hadoop.conf.Configurable;
-
-/**
- * A pluggable object for altering the weights of apps in the fair scheduler,
- * which is used for example by {@link NewAppWeightBooster} to give higher
- * weight to new jobs so that short jobs finish faster.
- *
- * May implement {@link Configurable} to access configuration parameters.
- */
-@Private
-@Unstable
-public interface WeightAdjuster {
- public double adjustWeight(FSAppAttempt app, double curWeight);
-}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/48] hadoop git commit: YARN-5608. TestAMRMClient.setup() fails
with ArrayOutOfBoundsException. Contributed by Daniel Templeton.
Posted by aw...@apache.org.
YARN-5608. TestAMRMClient.setup() fails with ArrayOutOfBoundsException. Contributed by Daniel Templeton.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9557127
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9557127
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9557127
Branch: refs/heads/HADOOP-13341
Commit: f9557127b3660e138cab8d9535a589677809384a
Parents: 07650bc
Author: Rohith Sharma K S <ro...@apache.org>
Authored: Tue Sep 6 14:09:45 2016 +0530
Committer: Rohith Sharma K S <ro...@apache.org>
Committed: Tue Sep 6 14:09:45 2016 +0530
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TestAMRMClient.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9557127/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
index 38178a4..239189a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
@@ -145,8 +145,13 @@ public class TestAMRMClient {
yarnClient.start();
// get node info
+ assertTrue("All node managers did not connect to the RM within the "
+ + "allotted 5-second timeout",
+ yarnCluster.waitForNodeManagersToConnect(5000L));
nodeReports = yarnClient.getNodeReports(NodeState.RUNNING);
-
+ assertEquals("Not all node managers were reported running",
+ nodeCount, nodeReports.size());
+
priority = Priority.newInstance(1);
priority2 = Priority.newInstance(2);
capability = Resource.newInstance(1024, 1);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/48] hadoop git commit: HDFS-10833. Fix JSON errors in WebHDFS.md
examples.
Posted by aw...@apache.org.
HDFS-10833. Fix JSON errors in WebHDFS.md examples.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cbd909ce
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cbd909ce
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cbd909ce
Branch: refs/heads/HADOOP-13341
Commit: cbd909ce2a5ac1da258f756fa1f93e84dd20b926
Parents: 378f624
Author: Andrew Wang <wa...@apache.org>
Authored: Fri Sep 2 14:50:34 2016 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Fri Sep 2 14:50:34 2016 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/site/markdown/WebHDFS.md | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cbd909ce/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
index 30f583e..c62fb2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/WebHDFS.md
@@ -1294,7 +1294,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
{
"entries":
{
- "type": "array"
+ "type": "array",
"items":
{
"description": "ACL entry.",
@@ -1318,7 +1318,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
"description": "True if the sticky bit is on.",
"type" : "boolean",
"required" : true
- },
+ }
}
}
}
@@ -1337,7 +1337,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
"type" : "array",
"items":
{
- "type" " "object",
+ "type" : "object",
"properties":
{
"name":
@@ -1368,7 +1368,7 @@ All operations, except for [`OPEN`](#Open_and_Read_a_File), either return a zero
"XAttrNames":
{
"description": "XAttr names.",
- "type" : "string"
+ "type" : "string",
"required" : true
}
}
@@ -1737,7 +1737,7 @@ See also: [`GETHOMEDIRECTORY`](#Get_Home_Directory), [Path](../../api/org/apache
"javaClassName": //an optional property
{
"description": "Java class name of the exception",
- "type" : "string",
+ "type" : "string"
}
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org