You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sh...@apache.org on 2018/09/22 01:40:14 UTC
[16/18] hadoop git commit: Merge branch 'trunk' into HDFS-12943
Merge branch 'trunk' into HDFS-12943
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c04e0c0e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c04e0c0e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c04e0c0e
Branch: refs/heads/HDFS-12943
Commit: c04e0c0e9951aab88d7e5a4f47bf24a045f6171c
Parents: 6c37db9 0cd6346
Author: Konstantin V Shvachko <sh...@apache.org>
Authored: Fri Sep 21 18:24:51 2018 -0700
Committer: Konstantin V Shvachko <sh...@apache.org>
Committed: Fri Sep 21 18:28:31 2018 -0700
----------------------------------------------------------------------
dev-support/bin/create-release | 4 +-
dev-support/bin/ozone-dist-layout-stitching | 32 +-
dev-support/bin/yetus-wrapper | 2 +-
dev-support/docker/Dockerfile | 212 +-
.../assemblies/hadoop-src-with-hdds.xml | 56 +
.../assemblies/hadoop-src-with-hdsl.xml | 56 -
.../hadoop-client-minicluster/pom.xml | 17 +-
.../hadoop-client-runtime/pom.xml | 11 +
.../hadoop-annotations/pom.xml | 24 +
hadoop-common-project/hadoop-common/pom.xml | 7 +
.../src/main/conf/log4j.properties | 23 -
.../apache/hadoop/crypto/CryptoStreamUtils.java | 21 +-
.../main/java/org/apache/hadoop/fs/Globber.java | 13 +-
.../apache/hadoop/fs/TrashPolicyDefault.java | 10 +-
.../main/java/org/apache/hadoop/ha/HAAdmin.java | 7 +-
.../org/apache/hadoop/io/nativeio/NativeIO.java | 15 +-
.../org/apache/hadoop/ipc/CallQueueManager.java | 5 +-
.../apache/hadoop/log/LogThrottlingHelper.java | 358 ++
.../apache/hadoop/security/SecurityUtil.java | 34 +-
.../org/apache/hadoop/util/CleanerUtil.java | 199 +
.../org/apache/hadoop/util/StringUtils.java | 2 +-
.../hadoop/util/curator/ZKCuratorManager.java | 10 +-
.../src/main/resources/core-default.xml | 24 +-
.../markdown/release/0.1.0/CHANGELOG.0.1.0.md | 101 +
.../markdown/release/0.1.0/CHANGES.0.1.0.md | 101 -
.../markdown/release/0.1.1/CHANGELOG.0.1.1.md | 39 +
.../markdown/release/0.1.1/CHANGES.0.1.1.md | 39 -
.../markdown/release/0.10.0/CHANGELOG.0.10.0.md | 101 +
.../markdown/release/0.10.0/CHANGES.0.10.0.md | 101 -
.../markdown/release/0.10.1/CHANGELOG.0.10.1.md | 49 +
.../markdown/release/0.10.1/CHANGES.0.10.1.md | 49 -
.../markdown/release/0.11.0/CHANGELOG.0.11.0.md | 96 +
.../markdown/release/0.11.0/CHANGES.0.11.0.md | 96 -
.../markdown/release/0.11.1/CHANGELOG.0.11.1.md | 34 +
.../markdown/release/0.11.1/CHANGES.0.11.1.md | 34 -
.../markdown/release/0.11.2/CHANGELOG.0.11.2.md | 33 +
.../markdown/release/0.11.2/CHANGES.0.11.2.md | 33 -
.../markdown/release/0.12.0/CHANGELOG.0.12.0.md | 113 +
.../markdown/release/0.12.0/CHANGES.0.12.0.md | 113 -
.../markdown/release/0.12.1/CHANGELOG.0.12.1.md | 59 +
.../markdown/release/0.12.1/CHANGES.0.12.1.md | 59 -
.../markdown/release/0.12.2/CHANGELOG.0.12.2.md | 34 +
.../markdown/release/0.12.2/CHANGES.0.12.2.md | 34 -
.../markdown/release/0.12.3/CHANGELOG.0.12.3.md | 38 +
.../markdown/release/0.12.3/CHANGES.0.12.3.md | 38 -
.../markdown/release/0.13.0/CHANGELOG.0.13.0.md | 173 +
.../markdown/release/0.13.0/CHANGES.0.13.0.md | 173 -
.../markdown/release/0.14.0/CHANGELOG.0.14.0.md | 214 +
.../markdown/release/0.14.0/CHANGES.0.14.0.md | 214 -
.../markdown/release/0.14.1/CHANGELOG.0.14.1.md | 33 +
.../markdown/release/0.14.1/CHANGES.0.14.1.md | 33 -
.../markdown/release/0.14.2/CHANGELOG.0.14.2.md | 40 +
.../markdown/release/0.14.2/CHANGES.0.14.2.md | 40 -
.../markdown/release/0.14.3/CHANGELOG.0.14.3.md | 34 +
.../markdown/release/0.14.3/CHANGES.0.14.3.md | 34 -
.../markdown/release/0.14.4/CHANGELOG.0.14.4.md | 39 +
.../markdown/release/0.14.4/CHANGES.0.14.4.md | 39 -
.../markdown/release/0.15.0/CHANGELOG.0.15.0.md | 190 +
.../markdown/release/0.15.0/CHANGES.0.15.0.md | 190 -
.../markdown/release/0.15.1/CHANGELOG.0.15.1.md | 49 +
.../markdown/release/0.15.1/CHANGES.0.15.1.md | 49 -
.../markdown/release/0.15.2/CHANGELOG.0.15.2.md | 51 +
.../markdown/release/0.15.2/CHANGES.0.15.2.md | 51 -
.../markdown/release/0.15.3/CHANGELOG.0.15.3.md | 35 +
.../markdown/release/0.15.3/CHANGES.0.15.3.md | 35 -
.../markdown/release/0.15.4/CHANGELOG.0.15.4.md | 31 +
.../markdown/release/0.15.4/CHANGES.0.15.4.md | 31 -
.../markdown/release/0.16.0/CHANGELOG.0.16.0.md | 225 ++
.../markdown/release/0.16.0/CHANGES.0.16.0.md | 225 --
.../markdown/release/0.16.1/CHANGELOG.0.16.1.md | 94 +
.../markdown/release/0.16.1/CHANGES.0.16.1.md | 94 -
.../markdown/release/0.16.2/CHANGELOG.0.16.2.md | 59 +
.../markdown/release/0.16.2/CHANGES.0.16.2.md | 59 -
.../markdown/release/0.16.3/CHANGELOG.0.16.3.md | 37 +
.../markdown/release/0.16.3/CHANGES.0.16.3.md | 37 -
.../markdown/release/0.16.4/CHANGELOG.0.16.4.md | 34 +
.../markdown/release/0.16.4/CHANGES.0.16.4.md | 34 -
.../markdown/release/0.17.0/CHANGELOG.0.17.0.md | 259 ++
.../markdown/release/0.17.0/CHANGES.0.17.0.md | 259 --
.../markdown/release/0.17.1/CHANGELOG.0.17.1.md | 44 +
.../markdown/release/0.17.1/CHANGES.0.17.1.md | 44 -
.../markdown/release/0.17.2/CHANGELOG.0.17.2.md | 43 +
.../markdown/release/0.17.2/CHANGES.0.17.2.md | 43 -
.../markdown/release/0.17.3/CHANGELOG.0.17.3.md | 42 +
.../markdown/release/0.17.3/CHANGES.0.17.3.md | 42 -
.../markdown/release/0.18.0/CHANGELOG.0.18.0.md | 331 ++
.../markdown/release/0.18.0/CHANGES.0.18.0.md | 331 --
.../markdown/release/0.18.1/CHANGELOG.0.18.1.md | 46 +
.../markdown/release/0.18.1/CHANGES.0.18.1.md | 46 -
.../markdown/release/0.18.2/CHANGELOG.0.18.2.md | 70 +
.../markdown/release/0.18.2/CHANGES.0.18.2.md | 70 -
.../markdown/release/0.18.3/CHANGELOG.0.18.3.md | 103 +
.../markdown/release/0.18.3/CHANGES.0.18.3.md | 103 -
.../markdown/release/0.18.4/CHANGELOG.0.18.4.md | 45 +
.../markdown/release/0.18.4/CHANGES.0.18.4.md | 45 -
.../markdown/release/0.19.0/CHANGELOG.0.19.0.md | 422 ++
.../markdown/release/0.19.0/CHANGES.0.19.0.md | 422 --
.../markdown/release/0.19.1/CHANGELOG.0.19.1.md | 94 +
.../markdown/release/0.19.1/CHANGES.0.19.1.md | 94 -
.../markdown/release/0.19.2/CHANGELOG.0.19.2.md | 80 +
.../markdown/release/0.19.2/CHANGES.0.19.2.md | 80 -
.../markdown/release/0.2.0/CHANGELOG.0.2.0.md | 103 +
.../markdown/release/0.2.0/CHANGES.0.2.0.md | 103 -
.../markdown/release/0.2.1/CHANGELOG.0.2.1.md | 33 +
.../markdown/release/0.2.1/CHANGES.0.2.1.md | 33 -
.../markdown/release/0.20.0/CHANGELOG.0.20.0.md | 341 ++
.../markdown/release/0.20.0/CHANGES.0.20.0.md | 341 --
.../markdown/release/0.20.1/CHANGELOG.0.20.1.md | 127 +
.../markdown/release/0.20.1/CHANGES.0.20.1.md | 127 -
.../markdown/release/0.20.2/CHANGELOG.0.20.2.md | 91 +
.../markdown/release/0.20.2/CHANGES.0.20.2.md | 91 -
.../release/0.20.203.0/CHANGELOG.0.20.203.0.md | 73 +
.../release/0.20.203.0/CHANGES.0.20.203.0.md | 73 -
.../release/0.20.203.1/CHANGELOG.0.20.203.1.md | 32 +
.../release/0.20.203.1/CHANGES.0.20.203.1.md | 32 -
.../release/0.20.204.0/CHANGELOG.0.20.204.0.md | 121 +
.../release/0.20.204.0/CHANGES.0.20.204.0.md | 121 -
.../release/0.20.205.0/CHANGELOG.0.20.205.0.md | 191 +
.../release/0.20.205.0/CHANGES.0.20.205.0.md | 191 -
.../markdown/release/0.20.3/CHANGELOG.0.20.3.md | 91 +
.../markdown/release/0.20.3/CHANGES.0.20.3.md | 91 -
.../markdown/release/0.21.0/CHANGELOG.0.21.0.md | 1360 +++++++
.../markdown/release/0.21.0/CHANGES.0.21.0.md | 1360 -------
.../markdown/release/0.21.1/CHANGELOG.0.21.1.md | 123 +
.../markdown/release/0.21.1/CHANGES.0.21.1.md | 123 -
.../markdown/release/0.22.0/CHANGELOG.0.22.0.md | 762 ++++
.../markdown/release/0.22.0/CHANGES.0.22.0.md | 762 ----
.../markdown/release/0.22.1/CHANGELOG.0.22.1.md | 107 +
.../markdown/release/0.22.1/CHANGES.0.22.1.md | 107 -
.../markdown/release/0.23.0/CHANGELOG.0.23.0.md | 1182 ++++++
.../markdown/release/0.23.0/CHANGES.0.23.0.md | 1182 ------
.../markdown/release/0.23.1/CHANGELOG.0.23.1.md | 484 +++
.../markdown/release/0.23.1/CHANGES.0.23.1.md | 484 ---
.../release/0.23.10/CHANGELOG.0.23.10.md | 121 +
.../markdown/release/0.23.10/CHANGES.0.23.10.md | 120 -
.../release/0.23.11/CHANGELOG.0.23.11.md | 83 +
.../markdown/release/0.23.11/CHANGES.0.23.11.md | 83 -
.../markdown/release/0.23.2/CHANGELOG.0.23.2.md | 177 +
.../markdown/release/0.23.2/CHANGES.0.23.2.md | 177 -
.../markdown/release/0.23.3/CHANGELOG.0.23.3.md | 334 ++
.../markdown/release/0.23.3/CHANGES.0.23.3.md | 334 --
.../markdown/release/0.23.4/CHANGELOG.0.23.4.md | 74 +
.../markdown/release/0.23.4/CHANGES.0.23.4.md | 74 -
.../markdown/release/0.23.5/CHANGELOG.0.23.5.md | 152 +
.../markdown/release/0.23.5/CHANGES.0.23.5.md | 152 -
.../markdown/release/0.23.6/CHANGELOG.0.23.6.md | 127 +
.../markdown/release/0.23.6/CHANGES.0.23.6.md | 127 -
.../markdown/release/0.23.7/CHANGELOG.0.23.7.md | 189 +
.../markdown/release/0.23.7/CHANGES.0.23.7.md | 189 -
.../markdown/release/0.23.8/CHANGELOG.0.23.8.md | 67 +
.../markdown/release/0.23.8/CHANGES.0.23.8.md | 67 -
.../markdown/release/0.23.9/CHANGELOG.0.23.9.md | 66 +
.../markdown/release/0.23.9/CHANGES.0.23.9.md | 66 -
.../markdown/release/0.24.0/CHANGELOG.0.24.0.md | 89 +
.../markdown/release/0.24.0/CHANGES.0.24.0.md | 89 -
.../markdown/release/0.3.0/CHANGELOG.0.3.0.md | 86 +
.../markdown/release/0.3.0/CHANGES.0.3.0.md | 86 -
.../markdown/release/0.3.1/CHANGELOG.0.3.1.md | 35 +
.../markdown/release/0.3.1/CHANGES.0.3.1.md | 35 -
.../markdown/release/0.3.2/CHANGELOG.0.3.2.md | 55 +
.../markdown/release/0.3.2/CHANGES.0.3.2.md | 55 -
.../markdown/release/0.4.0/CHANGELOG.0.4.0.md | 66 +
.../markdown/release/0.4.0/CHANGES.0.4.0.md | 66 -
.../markdown/release/0.5.0/CHANGELOG.0.5.0.md | 112 +
.../markdown/release/0.5.0/CHANGES.0.5.0.md | 112 -
.../markdown/release/0.6.0/CHANGELOG.0.6.0.md | 89 +
.../markdown/release/0.6.0/CHANGES.0.6.0.md | 89 -
.../markdown/release/0.6.1/CHANGELOG.0.6.1.md | 35 +
.../markdown/release/0.6.1/CHANGES.0.6.1.md | 35 -
.../markdown/release/0.6.2/CHANGELOG.0.6.2.md | 39 +
.../markdown/release/0.6.2/CHANGES.0.6.2.md | 39 -
.../markdown/release/0.7.0/CHANGELOG.0.7.0.md | 87 +
.../markdown/release/0.7.0/CHANGES.0.7.0.md | 87 -
.../markdown/release/0.7.1/CHANGELOG.0.7.1.md | 41 +
.../markdown/release/0.7.1/CHANGES.0.7.1.md | 41 -
.../markdown/release/0.7.2/CHANGELOG.0.7.2.md | 39 +
.../markdown/release/0.7.2/CHANGES.0.7.2.md | 39 -
.../markdown/release/0.8.0/CHANGELOG.0.8.0.md | 82 +
.../markdown/release/0.8.0/CHANGES.0.8.0.md | 82 -
.../markdown/release/0.9.0/CHANGELOG.0.9.0.md | 99 +
.../markdown/release/0.9.0/CHANGES.0.9.0.md | 99 -
.../markdown/release/0.9.1/CHANGELOG.0.9.1.md | 32 +
.../markdown/release/0.9.1/CHANGES.0.9.1.md | 32 -
.../markdown/release/0.9.2/CHANGELOG.0.9.2.md | 33 +
.../markdown/release/0.9.2/CHANGES.0.9.2.md | 33 -
.../markdown/release/1.0.0/CHANGELOG.1.0.0.md | 117 +
.../markdown/release/1.0.0/CHANGES.1.0.0.md | 117 -
.../markdown/release/1.0.1/CHANGELOG.1.0.1.md | 54 +
.../markdown/release/1.0.1/CHANGES.1.0.1.md | 54 -
.../markdown/release/1.0.2/CHANGELOG.1.0.2.md | 59 +
.../markdown/release/1.0.2/CHANGES.1.0.2.md | 59 -
.../markdown/release/1.0.3/CHANGELOG.1.0.3.md | 68 +
.../markdown/release/1.0.3/CHANGES.1.0.3.md | 68 -
.../markdown/release/1.0.4/CHANGELOG.1.0.4.md | 39 +
.../markdown/release/1.0.4/CHANGES.1.0.4.md | 39 -
.../markdown/release/1.1.0/CHANGELOG.1.1.0.md | 199 +
.../markdown/release/1.1.0/CHANGES.1.1.0.md | 199 -
.../markdown/release/1.1.1/CHANGELOG.1.1.1.md | 68 +
.../markdown/release/1.1.1/CHANGES.1.1.1.md | 68 -
.../markdown/release/1.1.2/CHANGELOG.1.1.2.md | 73 +
.../markdown/release/1.1.2/CHANGES.1.1.2.md | 73 -
.../markdown/release/1.1.3/CHANGELOG.1.1.3.md | 31 +
.../markdown/release/1.1.3/CHANGES.1.1.3.md | 31 -
.../markdown/release/1.2.0/CHANGELOG.1.2.0.md | 274 ++
.../markdown/release/1.2.0/CHANGES.1.2.0.md | 274 --
.../markdown/release/1.2.1/CHANGELOG.1.2.1.md | 62 +
.../markdown/release/1.2.1/CHANGES.1.2.1.md | 62 -
.../markdown/release/1.2.2/CHANGELOG.1.2.2.md | 32 +
.../markdown/release/1.2.2/CHANGES.1.2.2.md | 32 -
.../markdown/release/1.3.0/CHANGELOG.1.3.0.md | 139 +
.../markdown/release/1.3.0/CHANGES.1.3.0.md | 139 -
.../2.0.0-alpha/CHANGELOG.2.0.0-alpha.md | 398 ++
.../release/2.0.0-alpha/CHANGES.2.0.0-alpha.md | 398 --
.../2.0.1-alpha/CHANGELOG.2.0.1-alpha.md | 36 +
.../release/2.0.1-alpha/CHANGES.2.0.1-alpha.md | 36 -
.../2.0.2-alpha/CHANGELOG.2.0.2-alpha.md | 714 ++++
.../release/2.0.2-alpha/CHANGES.2.0.2-alpha.md | 714 ----
.../2.0.3-alpha/CHANGELOG.2.0.3-alpha.md | 588 +++
.../release/2.0.3-alpha/CHANGES.2.0.3-alpha.md | 588 ---
.../2.0.4-alpha/CHANGELOG.2.0.4-alpha.md | 76 +
.../release/2.0.4-alpha/CHANGES.2.0.4-alpha.md | 76 -
.../2.0.5-alpha/CHANGELOG.2.0.5-alpha.md | 35 +
.../release/2.0.5-alpha/CHANGES.2.0.5-alpha.md | 35 -
.../2.0.6-alpha/CHANGELOG.2.0.6-alpha.md | 31 +
.../release/2.0.6-alpha/CHANGES.2.0.6-alpha.md | 31 -
.../release/2.1.0-beta/CHANGELOG.2.1.0-beta.md | 911 +++++
.../release/2.1.0-beta/CHANGES.2.1.0-beta.md | 911 -----
.../release/2.1.1-beta/CHANGELOG.2.1.1-beta.md | 233 ++
.../release/2.1.1-beta/CHANGES.2.1.1-beta.md | 233 --
.../markdown/release/2.10.0/CHANGELOG.2.10.0.md | 484 +++
.../release/2.10.0/RELEASENOTES.2.10.0.md | 115 +
.../markdown/release/2.2.0/CHANGELOG.2.2.0.md | 140 +
.../markdown/release/2.2.0/CHANGES.2.2.0.md | 140 -
.../markdown/release/2.2.1/CHANGELOG.2.2.1.md | 38 +
.../markdown/release/2.2.1/CHANGES.2.2.1.md | 38 -
.../markdown/release/2.3.0/CHANGELOG.2.3.0.md | 665 ++++
.../markdown/release/2.3.0/CHANGES.2.3.0.md | 665 ----
.../markdown/release/2.4.0/CHANGELOG.2.4.0.md | 487 +++
.../markdown/release/2.4.0/CHANGES.2.4.0.md | 487 ---
.../markdown/release/2.4.1/CHANGELOG.2.4.1.md | 136 +
.../markdown/release/2.4.1/CHANGES.2.4.1.md | 136 -
.../markdown/release/2.5.0/CHANGELOG.2.5.0.md | 551 +++
.../markdown/release/2.5.0/CHANGES.2.5.0.md | 551 ---
.../markdown/release/2.5.1/CHANGELOG.2.5.1.md | 35 +
.../markdown/release/2.5.1/CHANGES.2.5.1.md | 35 -
.../markdown/release/2.5.2/CHANGELOG.2.5.2.md | 35 +
.../markdown/release/2.5.2/CHANGES.2.5.2.md | 35 -
.../markdown/release/2.6.0/CHANGELOG.2.6.0.md | 964 +++++
.../markdown/release/2.6.0/CHANGES.2.6.0.md | 964 -----
.../markdown/release/2.6.1/CHANGELOG.2.6.1.md | 202 +
.../markdown/release/2.6.1/CHANGES.2.6.1.md | 202 -
.../markdown/release/2.6.2/CHANGELOG.2.6.2.md | 61 +
.../markdown/release/2.6.2/CHANGES.2.6.2.md | 61 -
.../markdown/release/2.6.3/CHANGELOG.2.6.3.md | 77 +
.../markdown/release/2.6.3/CHANGES.2.6.3.md | 77 -
.../markdown/release/2.6.4/CHANGELOG.2.6.4.md | 98 +
.../markdown/release/2.6.4/CHANGES.2.6.4.md | 98 -
.../markdown/release/2.6.5/CHANGELOG.2.6.5.md | 132 +
.../markdown/release/2.6.5/CHANGES.2.6.5.md | 132 -
.../markdown/release/2.6.6/CHANGELOG.2.6.6.md | 57 +
.../markdown/release/2.6.6/CHANGES.2.6.6.md | 47 -
.../markdown/release/2.7.0/CHANGELOG.2.7.0.md | 988 +++++
.../markdown/release/2.7.0/CHANGES.2.7.0.md | 988 -----
.../release/2.7.0/RELEASENOTES.2.7.0.md | 7 +
.../markdown/release/2.7.1/CHANGELOG.2.7.1.md | 175 +
.../markdown/release/2.7.1/CHANGES.2.7.1.md | 182 -
.../release/2.7.1/RELEASENOTES.2.7.1.md | 7 -
.../markdown/release/2.7.2/CHANGELOG.2.7.2.md | 208 +
.../markdown/release/2.7.2/CHANGES.2.7.2.md | 208 -
.../markdown/release/2.7.3/CHANGELOG.2.7.3.md | 284 ++
.../markdown/release/2.7.3/CHANGES.2.7.3.md | 284 --
.../markdown/release/2.7.4/CHANGELOG.2.7.4.md | 326 ++
.../markdown/release/2.7.4/CHANGES.2.7.4.md | 326 --
.../release/2.7.4/RELEASENOTES.2.7.4.md | 14 +
.../markdown/release/2.7.5/CHANGELOG.2.7.5.md | 93 +
.../markdown/release/2.7.5/CHANGES.2.7.5.md | 32 -
.../release/2.7.5/RELEASENOTES.2.7.5.md | 12 +
.../markdown/release/2.7.6/CHANGELOG.2.7.6.md | 92 +
.../release/2.7.6/RELEASENOTES.2.7.6.md | 42 +
.../markdown/release/2.7.7/CHANGELOG.2.7.7.md | 47 +
.../release/2.7.7/RELEASENOTES.2.7.7.md | 21 +
.../markdown/release/2.7.8/CHANGELOG.2.7.8.md | 31 +
.../release/2.7.8/RELEASENOTES.2.7.8.md | 21 +
.../markdown/release/2.8.0/CHANGELOG.2.8.0.md | 2999 ++++++++++++++
.../markdown/release/2.8.0/CHANGES.2.8.0.md | 2993 --------------
.../release/2.8.0/RELEASENOTES.2.8.0.md | 7 +
.../markdown/release/2.8.1/CHANGELOG.2.8.1.md | 31 +
.../markdown/release/2.8.1/CHANGES.2.8.1.md | 31 -
.../markdown/release/2.8.2/CHANGELOG.2.8.2.md | 374 ++
.../markdown/release/2.8.2/CHANGES.2.8.2.md | 385 --
.../release/2.8.2/RELEASENOTES.2.8.2.md | 9 +-
.../markdown/release/2.8.3/CHANGELOG.2.8.3.md | 129 +
.../markdown/release/2.8.3/CHANGES.2.8.3.md | 69 -
.../release/2.8.3/RELEASENOTES.2.8.3.md | 17 +
.../markdown/release/2.8.4/CHANGELOG.2.8.4.md | 123 +
.../release/2.8.4/RELEASENOTES.2.8.4.md | 21 +
.../markdown/release/2.8.5/CHANGELOG.2.8.5.md | 70 +
.../release/2.8.5/RELEASENOTES.2.8.5.md | 21 +
.../markdown/release/2.9.0/CHANGELOG.2.9.0.md | 2013 ++++++++++
.../markdown/release/2.9.0/CHANGES.2.9.0.md | 1453 -------
.../release/2.9.0/RELEASENOTES.2.9.0.md | 189 +-
.../markdown/release/2.9.1/CHANGELOG.2.9.1.md | 279 ++
.../markdown/release/2.9.1/CHANGES.2.9.1.md | 277 --
.../release/2.9.1/RELEASENOTES.2.9.1.md | 2 +-
.../markdown/release/2.9.2/CHANGELOG.2.9.2.md | 202 +
.../release/2.9.2/RELEASENOTES.2.9.2.md | 21 +
.../3.0.0-alpha1/CHANGELOG.3.0.0-alpha1.md | 3751 ++++++++++++++++++
.../3.0.0-alpha1/CHANGES.3.0.0-alpha1.md | 3750 -----------------
.../3.0.0-alpha1/RELEASENOTES.3.0.0-alpha1.md | 14 +
.../3.0.0-alpha2/CHANGELOG.3.0.0-alpha2.md | 922 +++++
.../3.0.0-alpha2/CHANGES.3.0.0-alpha2.md | 919 -----
.../3.0.0-alpha2/RELEASENOTES.3.0.0-alpha2.md | 17 +-
.../3.0.0-alpha3/CHANGELOG.3.0.0-alpha3.md | 46 +
.../3.0.0-alpha3/CHANGES.3.0.0-alpha3.md | 46 -
.../3.0.0-alpha4/CHANGELOG.3.0.0-alpha4.md | 878 ++++
.../3.0.0-alpha4/CHANGES.3.0.0-alpha4.md | 887 -----
.../3.0.0-alpha4/RELEASENOTES.3.0.0-alpha4.md | 12 +-
.../3.0.0-beta1/CHANGELOG.3.0.0-beta1.md | 649 +++
.../release/3.0.0-beta1/CHANGES.3.0.0-beta1.md | 646 ---
.../3.0.0-beta1/RELEASENOTES.3.0.0-beta1.md | 13 +
.../markdown/release/3.0.0/CHANGELOG.3.0.0.md | 363 ++
.../markdown/release/3.0.0/CHANGES.3.0.0.md | 360 --
.../release/3.0.0/RELEASENOTES.3.0.0.md | 7 -
.../markdown/release/3.0.1/CHANGELOG.3.0.1.md | 231 ++
.../markdown/release/3.0.1/CHANGES.3.0.1.md | 241 --
.../markdown/release/3.0.2/CHANGELOG.3.0.2.md | 31 +
.../markdown/release/3.0.2/CHANGES.3.0.2.md | 31 -
.../markdown/release/3.0.3/CHANGELOG.3.0.3.md | 312 ++
.../markdown/release/3.0.3/CHANGES.3.0.3.md | 309 --
.../release/3.0.3/RELEASENOTES.3.0.3.md | 7 +
.../markdown/release/3.0.4/CHANGELOG.3.0.4.md | 189 +
.../release/3.0.4/RELEASENOTES.3.0.4.md | 50 +
.../markdown/release/3.1.0/CHANGELOG.3.1.0.md | 1042 +++++
.../markdown/release/3.1.0/CHANGES.3.1.0.md | 1022 -----
.../release/3.1.0/RELEASENOTES.3.1.0.md | 7 +
.../markdown/release/3.1.1/CHANGELOG.3.1.1.md | 502 +++
.../markdown/release/3.1.1/CHANGES.3.1.1.md | 498 ---
.../release/3.1.1/RELEASENOTES.3.1.1.md | 539 +--
.../markdown/release/3.1.2/CHANGELOG.3.1.2.md | 158 +
.../release/3.1.2/RELEASENOTES.3.1.2.md | 28 +
.../markdown/release/3.2.0/CHANGELOG.3.2.0.md | 881 ++++
.../release/3.2.0/RELEASENOTES.3.2.0.md | 134 +
.../java/org/apache/hadoop/fs/TestTrash.java | 6 +
.../hadoop/log/TestLogThrottlingHelper.java | 172 +
.../org/apache/hadoop/util/TestStringUtils.java | 9 +
.../util/curator/TestZKCuratorManager.java | 23 +
hadoop-dist/src/main/compose/ozone-hdfs/.env | 17 +
.../main/compose/ozone-hdfs/docker-compose.yaml | 60 +
.../src/main/compose/ozone-hdfs/docker-config | 76 +
.../src/main/compose/ozone/docker-config | 5 +
.../main/compose/ozonefs/docker-compose.yaml | 59 +
.../src/main/compose/ozonefs/docker-config | 35 +
.../src/main/compose/ozoneperf/docker-config | 4 +
.../compose/ozonescripts/.ssh/authorized_keys | 16 +
.../src/main/compose/ozonescripts/.ssh/config | 18 +
.../main/compose/ozonescripts/.ssh/environment | 16 +
.../src/main/compose/ozonescripts/.ssh/id_rsa | 42 +
.../main/compose/ozonescripts/.ssh/id_rsa.pub | 16 +
.../src/main/compose/ozonescripts/Dockerfile | 33 +
.../src/main/compose/ozonescripts/README.md | 38 +
.../compose/ozonescripts/docker-compose.yaml | 42 +
.../src/main/compose/ozonescripts/docker-config | 38 +
hadoop-dist/src/main/compose/ozonescripts/ps.sh | 17 +
.../src/main/compose/ozonescripts/start.sh | 24 +
.../src/main/compose/ozonescripts/stop.sh | 17 +
hadoop-dist/src/main/ozone/README.txt | 51 +
hadoop-dist/src/main/smoketest/README.md | 30 +
.../src/main/smoketest/basic/basic.robot | 47 +
.../src/main/smoketest/basic/ozone-shell.robot | 82 +
hadoop-dist/src/main/smoketest/commonlib.robot | 24 +
.../src/main/smoketest/ozonefs/ozonefs.robot | 35 +
hadoop-dist/src/main/smoketest/test.sh | 101 +
hadoop-hdds/client/pom.xml | 5 +-
.../apache/hadoop/hdds/scm/XceiverClient.java | 209 -
.../hadoop/hdds/scm/XceiverClientHandler.java | 202 -
.../hdds/scm/XceiverClientInitializer.java | 74 -
.../hadoop/hdds/scm/XceiverClientRatis.java | 101 +-
.../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +-
.../hdds/scm/storage/ChunkInputStream.java | 30 +-
.../hdds/scm/storage/ChunkOutputStream.java | 13 +-
hadoop-hdds/common/pom.xml | 11 +-
.../common/src/main/conf/log4j.properties | 157 +
.../org/apache/hadoop/hdds/HddsConfigKeys.java | 19 +
.../java/org/apache/hadoop/hdds/HddsUtils.java | 8 +-
.../org/apache/hadoop/hdds/cli/GenericCli.java | 26 +-
.../hadoop/hdds/cli/GenericParentCommand.java | 25 +
.../hdds/cli/MissingSubcommandException.java | 35 +
.../apache/hadoop/hdds/scm/ScmConfigKeys.java | 19 +
.../hadoop/hdds/scm/container/ContainerID.java | 26 +-
.../container/common/helpers/ContainerInfo.java | 11 +-
.../scm/container/common/helpers/Pipeline.java | 46 +-
.../container/common/helpers/PipelineID.java | 13 +-
.../scm/storage/ContainerProtocolCalls.java | 62 +-
.../apache/hadoop/ozone/OzoneConfigKeys.java | 20 +
.../org/apache/hadoop/ozone/OzoneConsts.java | 2 +
.../container/common/helpers/BlockData.java | 255 ++
.../ozone/container/common/helpers/KeyData.java | 253 --
.../apache/hadoop/ozone/lock/ActiveLock.java | 101 +
.../apache/hadoop/ozone/lock/LockManager.java | 101 +
.../hadoop/ozone/lock/PooledLockFactory.java | 43 +
.../apache/hadoop/ozone/lock/package-info.java | 21 +
.../hadoop/utils/db/DBConfigFromFile.java | 34 +-
.../main/java/org/apache/ratis/RatisHelper.java | 75 +-
.../main/proto/DatanodeContainerProtocol.proto | 74 +-
.../main/proto/ScmBlockLocationProtocol.proto | 6 +-
hadoop-hdds/common/src/main/proto/hdds.proto | 14 +
.../common/src/main/resources/ozone-default.xml | 63 +
.../hadoop/ozone/lock/TestLockManager.java | 64 +
.../apache/hadoop/ozone/lock/package-info.java | 21 +
hadoop-hdds/container-service/pom.xml | 6 +-
.../apache/hadoop/hdds/scm/HddsServerUtil.java | 21 +
.../common/impl/OpenContainerBlockMap.java | 46 +-
.../common/report/PipelineReportPublisher.java | 73 +
.../common/report/ReportPublisherFactory.java | 4 +
.../statemachine/DatanodeStateMachine.java | 19 +-
.../statemachine/SCMConnectionManager.java | 7 +-
.../common/statemachine/StateContext.java | 44 +-
.../CloseContainerCommandHandler.java | 17 +-
.../DeleteBlocksCommandHandler.java | 4 +-
.../ReplicateContainerCommandHandler.java | 129 +-
.../states/datanode/InitDatanodeState.java | 22 +-
.../states/endpoint/RegisterEndpointTask.java | 8 +-
.../common/transport/server/XceiverServer.java | 140 -
.../transport/server/XceiverServerGrpc.java | 28 +-
.../transport/server/XceiverServerHandler.java | 82 -
.../server/XceiverServerInitializer.java | 64 -
.../transport/server/XceiverServerSpi.java | 9 +
.../server/ratis/ContainerStateMachine.java | 42 +-
.../server/ratis/XceiverServerRatis.java | 132 +-
.../keyvalue/KeyValueBlockIterator.java | 16 +-
.../container/keyvalue/KeyValueContainer.java | 5 +-
.../container/keyvalue/KeyValueHandler.java | 124 +-
.../container/keyvalue/helpers/BlockUtils.java | 199 +
.../container/keyvalue/helpers/KeyUtils.java | 199 -
.../keyvalue/helpers/KeyValueContainerUtil.java | 12 +-
.../keyvalue/helpers/SmallFileUtils.java | 2 +-
.../keyvalue/impl/BlockManagerImpl.java | 229 ++
.../container/keyvalue/impl/KeyManagerImpl.java | 227 --
.../container/keyvalue/impl/package-info.java | 5 +-
.../keyvalue/interfaces/BlockManager.java | 84 +
.../keyvalue/interfaces/KeyManager.java | 84 -
.../keyvalue/interfaces/package-info.java | 21 +
.../background/BlockDeletingService.java | 10 +-
.../container/ozoneimpl/OzoneContainer.java | 126 +-
.../replication/ContainerReplicator.java | 27 +
.../DownloadAndImportReplicator.java | 136 +
.../replication/GrpcReplicationClient.java | 2 +-
.../replication/ReplicationSupervisor.java | 142 +
.../container/replication/ReplicationTask.java | 102 +
.../StorageContainerDatanodeProtocol.java | 10 +-
.../protocol/StorageContainerNodeProtocol.java | 6 +-
.../commands/CloseContainerCommand.java | 23 +-
...rDatanodeProtocolClientSideTranslatorPB.java | 6 +-
...rDatanodeProtocolServerSideTranslatorPB.java | 5 +-
.../StorageContainerDatanodeProtocol.proto | 10 +
.../ozone/container/common/ScmTestMock.java | 8 +-
.../common/TestDatanodeStateMachine.java | 42 +
.../TestReplicateContainerCommandHandler.java | 146 -
.../keyvalue/TestBlockManagerImpl.java | 211 +
.../keyvalue/TestChunkManagerImpl.java | 2 +-
.../container/keyvalue/TestKeyManagerImpl.java | 191 -
.../keyvalue/TestKeyValueBlockIterator.java | 30 +-
.../keyvalue/TestKeyValueContainer.java | 26 +-
.../container/keyvalue/TestKeyValueHandler.java | 38 +-
.../replication/TestReplicationSupervisor.java | 143 +
.../container/replication/package-info.java | 22 +
hadoop-hdds/framework/pom.xml | 5 +-
.../hadoop/hdds/server/events/EventQueue.java | 18 +
hadoop-hdds/pom.xml | 12 +-
hadoop-hdds/server-scm/pom.xml | 8 +-
.../org/apache/hadoop/hdds/scm/ScmUtils.java | 45 +
.../hadoop/hdds/scm/block/BlockManagerImpl.java | 339 +-
.../container/CloseContainerEventHandler.java | 90 +-
.../hdds/scm/container/ContainerMapping.java | 54 +-
.../scm/container/ContainerReportHandler.java | 55 +-
.../scm/container/ContainerStateManager.java | 67 +-
.../hadoop/hdds/scm/container/Mapping.java | 15 +-
.../replication/ReplicationActivityStatus.java | 55 +-
.../replication/ReplicationManager.java | 18 +-
.../scm/container/states/ContainerQueryKey.java | 110 +
.../scm/container/states/ContainerStateMap.java | 235 +-
.../hadoop/hdds/scm/events/SCMEvents.java | 27 +-
.../hdds/scm/exceptions/SCMException.java | 3 +-
.../hadoop/hdds/scm/node/DeadNodeHandler.java | 17 +
.../hadoop/hdds/scm/node/SCMNodeManager.java | 5 +-
.../hadoop/hdds/scm/node/StaleNodeHandler.java | 21 +-
.../hdds/scm/node/states/Node2ContainerMap.java | 123 +-
.../hdds/scm/node/states/Node2ObjectsMap.java | 162 +
.../hdds/scm/node/states/ReportResult.java | 105 +-
.../hdds/scm/pipelines/Node2PipelineMap.java | 45 +-
.../pipelines/PipelineActionEventHandler.java | 2 +
.../scm/pipelines/PipelineCloseHandler.java | 24 +-
.../hdds/scm/pipelines/PipelineManager.java | 180 +-
.../scm/pipelines/PipelineReportHandler.java | 59 +
.../hdds/scm/pipelines/PipelineSelector.java | 340 +-
.../scm/pipelines/PipelineStateManager.java | 136 +
.../scm/pipelines/ratis/RatisManagerImpl.java | 57 +-
.../standalone/StandaloneManagerImpl.java | 52 +-
.../hdds/scm/server/ChillModePrecheck.java | 56 +
.../apache/hadoop/hdds/scm/server/Precheck.java | 29 +
.../hdds/scm/server/SCMChillModeManager.java | 245 ++
.../scm/server/SCMClientProtocolServer.java | 53 +-
.../server/SCMDatanodeHeartbeatDispatcher.java | 23 +
.../scm/server/SCMDatanodeProtocolServer.java | 33 +-
.../scm/server/StorageContainerManager.java | 137 +-
.../apache/hadoop/hdds/scm/HddsTestUtils.java | 85 +
.../org/apache/hadoop/hdds/scm/TestUtils.java | 55 +-
.../hadoop/hdds/scm/block/TestBlockManager.java | 89 +-
.../hdds/scm/container/MockNodeManager.java | 4 +-
.../scm/container/TestContainerMapping.java | 54 +-
.../container/TestContainerReportHandler.java | 23 +-
.../container/TestContainerStateManager.java | 98 +
.../TestReplicationActivityStatus.java | 63 +
.../replication/TestReplicationManager.java | 104 +-
.../hdds/scm/node/TestDeadNodeHandler.java | 99 +-
.../hadoop/hdds/scm/node/TestNodeManager.java | 6 +-
.../scm/node/states/TestNode2ContainerMap.java | 35 +-
.../scm/server/TestSCMChillModeManager.java | 125 +
.../scm/server/TestSCMClientProtocolServer.java | 60 +
.../ozone/container/common/TestEndPoint.java | 5 +-
.../testutils/ReplicationNodeManagerMock.java | 5 +-
hadoop-hdds/tools/pom.xml | 4 +-
.../java/org/apache/hadoop/hdfs/DFSClient.java | 7 +-
.../hdfs/protocol/NoECPolicySetException.java | 37 +
.../apache/hadoop/hdfs/web/JsonUtilClient.java | 4 +
.../hadoop/fs/http/client/HttpFSFileSystem.java | 2 +-
.../hadoop/fs/http/server/FSOperations.java | 3 +
.../fs/http/client/BaseTestHttpFSWith.java | 35 +-
.../federation/router/RouterAdminServer.java | 23 +
.../src/site/markdown/HDFSRouterFederation.md | 26 +
.../store/driver/TestStateStoreZK.java | 53 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 +
.../qjournal/server/JournaledEditsCache.java | 2 +-
.../blockmanagement/BlockManagerSafeMode.java | 20 +-
.../BlockPlacementPolicyDefault.java | 29 +-
.../apache/hadoop/hdfs/server/common/Util.java | 3 +-
.../hdfs/server/datanode/BlockSender.java | 7 +-
.../hdfs/server/namenode/CacheManager.java | 42 +-
.../server/namenode/FSDirErasureCodingOp.java | 4 +
.../hdfs/server/namenode/FSEditLogAsync.java | 61 +-
.../hdfs/server/namenode/FSNamesystemLock.java | 46 +-
.../hdfs/server/namenode/ha/EditLogTailer.java | 34 +-
.../snapshot/DirectorySnapshottableFeature.java | 2 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 7 +
.../PBImageDelimitedTextWriter.java | 18 +-
.../src/main/resources/hdfs-default.xml | 17 +-
.../src/main/webapps/datanode/datanode.html | 3 +
.../src/main/webapps/static/dfs-dust.js | 4 +-
.../site/markdown/CentralizedCacheManagement.md | 5 +
.../src/site/markdown/HDFSDiskbalancer.md | 5 +-
.../apache/hadoop/hdfs/MiniDFSNNTopology.java | 17 +
.../TestDFSStripedOutputStreamWithFailure.java | 41 +-
.../TestUnsetAndChangeDirectoryEcPolicy.java | 23 +-
.../TestBlockManagerSafeMode.java | 81 +-
.../blockmanagement/TestReplicationPolicy.java | 28 +
.../TestDataNodeMultipleRegistrations.java | 4 +-
.../server/namenode/TestCacheDirectives.java | 49 +
.../hdfs/server/namenode/TestEditLogRace.java | 158 +-
.../server/namenode/ha/TestEditLogTailer.java | 98 +-
.../snapshot/TestSnapshotDiffReport.java | 6 +
.../TestOfflineImageViewer.java | 17 +
.../org/apache/hadoop/hdfs/web/TestWebHDFS.java | 18 +
.../test/resources/testErasureCodingConf.xml | 24 +
.../v2/app/speculate/DefaultSpeculator.java | 4 +-
.../mapreduce/v2/app/webapp/AMWebServices.java | 9 +-
.../v2/app/webapp/JAXBContextResolver.java | 7 +-
.../mapreduce/v2/app/webapp/TaskPage.java | 3 +-
.../v2/app/webapp/dao/MapTaskAttemptInfo.java | 39 +
.../app/webapp/dao/ReduceTaskAttemptInfo.java | 11 +-
.../v2/app/webapp/dao/TaskAttemptInfo.java | 14 +-
.../v2/app/webapp/dao/TaskAttemptsInfo.java | 18 +-
.../hadoop/mapreduce/v2/app/MRAppBenchmark.java | 20 +-
.../impl/TestTaskAttemptContainerRequest.java | 11 +-
.../v2/app/metrics/TestMRAppMetrics.java | 7 +
.../app/webapp/TestAMWebServicesAttempts.java | 3 +
.../src/site/markdown/MapredAppMasterRest.md | 2 +-
.../mapreduce/v2/hs/webapp/HsTaskPage.java | 3 +-
.../mapreduce/v2/hs/webapp/HsTasksBlock.java | 5 +-
.../mapreduce/v2/hs/webapp/HsWebServices.java | 9 +-
.../v2/hs/webapp/JAXBContextResolver.java | 4 +-
.../hadoop/mapred/ResourceMgrDelegate.java | 22 +
.../hadoop/mapred/TestClientRedirect.java | 25 +
hadoop-ozone/acceptance-test/README.md | 48 -
.../dev-support/bin/robot-all.sh | 18 -
.../dev-support/bin/robot-dnd-all.sh | 57 -
.../acceptance-test/dev-support/bin/robot.sh | 38 -
.../dev-support/docker/Dockerfile | 21 -
.../dev-support/docker/docker-compose.yaml | 23 -
hadoop-ozone/acceptance-test/pom.xml | 59 -
.../src/test/acceptance/basic/.env | 17 -
.../src/test/acceptance/basic/basic.robot | 50 -
.../test/acceptance/basic/docker-compose.yaml | 50 -
.../src/test/acceptance/basic/docker-config | 33 -
.../src/test/acceptance/basic/ozone-shell.robot | 86 -
.../src/test/acceptance/commonlib.robot | 78 -
.../src/test/acceptance/ozonefs/.env | 17 -
.../test/acceptance/ozonefs/docker-compose.yaml | 59 -
.../src/test/acceptance/ozonefs/docker-config | 34 -
.../src/test/acceptance/ozonefs/ozonefs.robot | 39 -
.../acceptance/ozonefs/ozonesinglenode.robot | 49 -
hadoop-ozone/client/pom.xml | 4 +-
.../ozone/client/io/ChunkGroupInputStream.java | 41 +-
.../ozone/client/io/OzoneInputStream.java | 5 +
.../hadoop/ozone/client/rpc/RpcClient.java | 3 +-
hadoop-ozone/common/pom.xml | 4 +-
hadoop-ozone/common/src/main/bin/ozone | 21 +-
.../common/src/main/bin/ozone-config.sh | 51 +
hadoop-ozone/common/src/main/bin/start-ozone.sh | 90 +-
hadoop-ozone/common/src/main/bin/stop-ozone.sh | 22 +-
.../hadoop/ozone/om/helpers/OmBucketInfo.java | 2 +-
.../hadoop/ozone/om/helpers/OmVolumeArgs.java | 2 +-
.../src/main/proto/OzoneManagerProtocol.proto | 2 +-
hadoop-ozone/docs/README.md | 13 +-
hadoop-ozone/docs/archetypes/default.md | 13 +-
hadoop-ozone/docs/config.toml | 23 -
hadoop-ozone/docs/config.yaml | 41 +
hadoop-ozone/docs/content/BucketCommands.md | 122 +
hadoop-ozone/docs/content/BuildingSources.md | 54 +
hadoop-ozone/docs/content/CommandShell.md | 245 +-
hadoop-ozone/docs/content/Concepts.md | 108 +
hadoop-ozone/docs/content/Dozone.md | 110 +
hadoop-ozone/docs/content/Freon.md | 64 +
hadoop-ozone/docs/content/GettingStarted.md | 369 --
hadoop-ozone/docs/content/Hdds.md | 65 +
hadoop-ozone/docs/content/JavaApi.md | 172 +
hadoop-ozone/docs/content/KeyCommands.md | 127 +
hadoop-ozone/docs/content/Metrics.md | 170 -
hadoop-ozone/docs/content/OzoneFS.md | 80 +
hadoop-ozone/docs/content/OzoneManager.md | 77 +
hadoop-ozone/docs/content/RealCluster.md | 74 +
hadoop-ozone/docs/content/Rest.md | 45 +-
hadoop-ozone/docs/content/RunningViaDocker.md | 73 +
hadoop-ozone/docs/content/RunningWithHDFS.md | 77 +
hadoop-ozone/docs/content/SCMCLI.md | 29 +
hadoop-ozone/docs/content/Settings.md | 142 +
hadoop-ozone/docs/content/VolumeCommands.md | 116 +
hadoop-ozone/docs/content/_index.md | 99 +-
hadoop-ozone/docs/pom.xml | 13 +-
hadoop-ozone/docs/static/NOTES.md | 13 +-
.../ozonedoc/layouts/_default/single.html | 16 +-
.../docs/themes/ozonedoc/layouts/index.html | 16 +
.../ozonedoc/layouts/partials/footer.html | 13 +-
.../ozonedoc/layouts/partials/header.html | 17 +-
.../ozonedoc/layouts/partials/navbar.html | 13 +-
.../ozonedoc/layouts/partials/sidebar.html | 21 +-
.../themes/ozonedoc/static/css/ozonedoc.css | 14 +-
hadoop-ozone/integration-test/pom.xml | 9 +-
.../container/TestContainerStateManager.java | 415 --
.../TestContainerStateManagerIntegration.java | 417 ++
.../hdds/scm/pipeline/TestNode2PipelineMap.java | 22 +-
.../hdds/scm/pipeline/TestPipelineClose.java | 15 +-
.../hdds/scm/pipeline/TestSCMRestart.java | 119 +
.../apache/hadoop/ozone/MiniOzoneCluster.java | 28 +-
.../hadoop/ozone/MiniOzoneClusterImpl.java | 101 +-
.../apache/hadoop/ozone/RatisTestHelper.java | 22 +
.../hadoop/ozone/TestMiniOzoneCluster.java | 4 +-
.../ozone/TestStorageContainerManager.java | 224 +-
.../TestStorageContainerManagerHelper.java | 8 +-
.../ozone/client/rest/TestOzoneRestClient.java | 8 +-
.../rpc/TestCloseContainerHandlingByClient.java | 5 +-
.../ozone/client/rpc/TestOzoneRpcClient.java | 8 +-
.../ozone/container/ContainerTestHelper.java | 84 +-
.../container/TestContainerReplication.java | 24 +-
.../common/TestBlockDeletingService.java | 12 +-
.../container/common/helpers/TestBlockData.java | 127 +
.../container/common/helpers/TestKeyData.java | 119 -
.../common/impl/TestCloseContainerHandler.java | 51 +-
.../common/impl/TestContainerPersistence.java | 154 +-
.../commandhandler/TestBlockDeletion.java | 9 +-
.../TestCloseContainerByPipeline.java | 5 +-
.../transport/server/ratis/TestCSMMetrics.java | 16 +-
.../container/metrics/TestContainerMetrics.java | 21 +-
.../container/ozoneimpl/TestOzoneContainer.java | 100 +-
.../container/server/TestContainerServer.java | 67 +-
.../server/TestContainerStateMachine.java | 2 +-
.../hadoop/ozone/freon/TestDataValidate.java | 119 +-
.../apache/hadoop/ozone/freon/TestFreon.java | 129 -
.../ozone/freon/TestRandomKeyGenerator.java | 106 +
...TestGenerateOzoneRequiredConfigurations.java | 131 -
.../hadoop/ozone/om/TestOzoneManager.java | 4 +-
.../hadoop/ozone/om/TestScmChillMode.java | 171 +
.../hadoop/ozone/ozShell/TestOzoneShell.java | 453 ++-
.../hadoop/ozone/scm/TestContainerSQLCli.java | 15 +-
.../ozone/scm/TestContainerSmallFile.java | 4 +-
.../TestGetCommittedBlockLengthAndPutKey.java | 12 +-
.../hadoop/ozone/web/client/TestKeys.java | 45 +-
.../hadoop/ozone/web/client/TestKeysRatis.java | 2 -
.../src/test/resources/log4j.properties | 5 +-
hadoop-ozone/objectstore-service/pom.xml | 56 +-
.../org/apache/hadoop/ozone/TestErrorCode.java | 53 -
.../apache/hadoop/ozone/web/TestErrorCode.java | 53 +
hadoop-ozone/ozone-manager/pom.xml | 6 +-
.../hadoop/ozone/om/BucketManagerImpl.java | 45 +-
.../apache/hadoop/ozone/om/KeyManagerImpl.java | 134 +-
.../hadoop/ozone/om/OMMetadataManager.java | 14 +-
.../hadoop/ozone/om/OmMetadataManagerImpl.java | 35 +-
.../apache/hadoop/ozone/om/OzoneManager.java | 35 +-
.../hadoop/ozone/om/OzoneManagerLock.java | 181 +
.../hadoop/ozone/om/VolumeManagerImpl.java | 41 +-
.../hadoop/ozone/om/exceptions/OMException.java | 3 +-
.../hadoop/ozone/web/ozShell/Handler.java | 49 +-
.../apache/hadoop/ozone/web/ozShell/Shell.java | 408 +-
.../web/ozShell/bucket/BucketCommands.java | 60 +
.../web/ozShell/bucket/CreateBucketHandler.java | 46 +-
.../web/ozShell/bucket/DeleteBucketHandler.java | 45 +-
.../web/ozShell/bucket/InfoBucketHandler.java | 42 +-
.../web/ozShell/bucket/ListBucketHandler.java | 89 +-
.../web/ozShell/bucket/UpdateBucketHandler.java | 66 +-
.../web/ozShell/keys/DeleteKeyHandler.java | 48 +-
.../ozone/web/ozShell/keys/GetKeyHandler.java | 77 +-
.../ozone/web/ozShell/keys/InfoKeyHandler.java | 45 +-
.../ozone/web/ozShell/keys/KeyCommands.java | 60 +
.../ozone/web/ozShell/keys/ListKeyHandler.java | 92 +-
.../ozone/web/ozShell/keys/PutKeyHandler.java | 80 +-
.../web/ozShell/volume/CreateVolumeHandler.java | 69 +-
.../web/ozShell/volume/DeleteVolumeHandler.java | 35 +-
.../web/ozShell/volume/InfoVolumeHandler.java | 37 +-
.../web/ozShell/volume/ListVolumeHandler.java | 82 +-
.../web/ozShell/volume/UpdateVolumeHandler.java | 49 +-
.../web/ozShell/volume/VolumeCommands.java | 61 +
.../ozone/om/ScmBlockLocationTestIngClient.java | 2 +-
.../hadoop/ozone/om/TestChunkStreams.java | 14 +-
.../hadoop/ozone/om/TestKeyDeletingService.java | 47 +-
.../hadoop/ozone/om/TestKeyManagerImpl.java | 165 +
.../hadoop/ozone/om/TestOzoneManagerLock.java | 192 +
hadoop-ozone/ozonefs/pom.xml | 18 +-
.../ozonefs/src/test/resources/log4j.properties | 1 +
hadoop-ozone/pom.xml | 46 +-
hadoop-ozone/tools/pom.xml | 6 +-
.../org/apache/hadoop/ozone/freon/Freon.java | 1136 +-----
.../hadoop/ozone/freon/RandomKeyGenerator.java | 1039 +++++
.../GenerateOzoneRequiredConfigurations.java | 12 +-
.../genesis/BenchMarkContainerStateMap.java | 24 +-
.../genesis/BenchMarkDatanodeDispatcher.java | 84 +-
.../apache/hadoop/ozone/genesis/Genesis.java | 9 +-
...TestGenerateOzoneRequiredConfigurations.java | 152 +
.../hadoop/ozone/genconf/package-info.java | 22 +
hadoop-project-dist/pom.xml | 34 +-
hadoop-project/pom.xml | 14 +-
hadoop-project/src/site/site.xml | 1 +
hadoop-tools/hadoop-aws/pom.xml | 4 +
.../fs/s3a/BasicAWSCredentialsProvider.java | 62 -
.../org/apache/hadoop/fs/s3a/Constants.java | 47 +-
.../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 32 +-
.../hadoop/fs/s3a/S3AInstrumentation.java | 5 +-
.../apache/hadoop/fs/s3a/S3ARetryPolicy.java | 24 +-
.../java/org/apache/hadoop/fs/s3a/S3AUtils.java | 60 +-
.../fs/s3a/SimpleAWSCredentialsProvider.java | 27 +-
.../fs/s3a/TemporaryAWSCredentialsProvider.java | 24 +-
.../fs/s3a/s3guard/DynamoDBMetadataStore.java | 430 +-
.../fs/s3a/s3guard/LocalMetadataStore.java | 26 +-
.../s3guard/S3GuardDataAccessRetryPolicy.java | 47 +
.../hadoop/fs/s3a/s3guard/S3GuardTool.java | 35 +
.../hadoop/fs/s3native/S3xLoginHelper.java | 121 +-
.../src/site/markdown/tools/hadoop-aws/index.md | 32 +-
.../site/markdown/tools/hadoop-aws/s3guard.md | 165 +-
.../site/markdown/tools/hadoop-aws/testing.md | 95 +-
.../tools/hadoop-aws/troubleshooting_s3a.md | 44 +-
.../fs/s3a/ITestS3AAWSCredentialsProvider.java | 20 +-
.../hadoop/fs/s3a/ITestS3AConfiguration.java | 51 +-
.../hadoop/fs/s3a/ITestS3ACredentialsInURL.java | 164 -
.../fs/s3a/ITestS3AFileSystemContract.java | 5 +
.../fs/s3a/TestS3AAWSCredentialsProvider.java | 50 +-
.../hadoop/fs/s3a/TestSSEConfiguration.java | 31 +-
.../s3guard/AbstractS3GuardToolTestBase.java | 27 +-
.../s3a/s3guard/ITestDynamoDBMetadataStore.java | 54 +-
.../ITestDynamoDBMetadataStoreScale.java | 595 ++-
.../s3a/s3guard/ITestS3GuardToolDynamoDB.java | 103 +-
.../AbstractITestS3AMetadataStoreScale.java | 24 +-
.../hadoop/fs/s3native/TestS3xLoginHelper.java | 70 +-
.../hadoop-aws/src/test/resources/core-site.xml | 10 +
.../hadoop/fs/azure/PageBlobOutputStream.java | 21 +-
.../fs/azure/ITestOutputStreamSemantics.java | 43 +
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 7 +
.../yarn/sls/scheduler/RMNodeWrapper.java | 6 +
hadoop-yarn-project/hadoop-yarn/bin/yarn | 5 +
.../yarn/api/ApplicationClientProtocol.java | 55 +
.../GetAttributesToNodesRequest.java | 74 +
.../GetAttributesToNodesResponse.java | 65 +
.../GetClusterNodeAttributesRequest.java | 47 +
.../GetClusterNodeAttributesResponse.java | 73 +
.../GetNodesToAttributesRequest.java | 65 +
.../GetNodesToAttributesResponse.java | 63 +
.../hadoop/yarn/api/records/NodeAttribute.java | 92 +
.../yarn/api/records/NodeAttributeInfo.java | 62 +
.../yarn/api/records/NodeAttributeKey.java | 66 +
.../yarn/api/records/NodeAttributeOpCode.java | 43 +
.../yarn/api/records/NodeAttributeType.java | 35 +
.../hadoop/yarn/api/records/NodeReport.java | 13 +
.../yarn/api/records/NodeToAttributeValue.java | 57 +
.../hadoop/yarn/api/records/Resource.java | 2 +-
.../yarn/api/resource/PlacementConstraint.java | 40 +-
.../yarn/api/resource/PlacementConstraints.java | 19 +
.../hadoop/yarn/conf/YarnConfiguration.java | 68 +-
.../ResourceManagerAdministrationProtocol.java | 13 +-
.../AttributeMappingOperationType.java | 42 +
.../api/protocolrecords/NodeToAttributes.java | 59 +
.../NodesToAttributesMappingRequest.java | 69 +
.../NodesToAttributesMappingResponse.java | 31 +
.../constraint/PlacementConstraintParser.java | 156 +-
.../main/proto/applicationclient_protocol.proto | 3 +
...esourcemanager_administration_protocol.proto | 1 +
..._server_resourcemanager_service_protos.proto | 16 +
.../src/main/proto/yarn_protos.proto | 44 +
.../src/main/proto/yarn_service_protos.proto | 23 +
.../resource/TestPlacementConstraintParser.java | 83 +-
.../distributedshell/ApplicationMaster.java | 51 +-
.../applications/distributedshell/Client.java | 9 +-
.../distributedshell/PlacementSpec.java | 19 +-
.../distributedshell/TestDistributedShell.java | 12 +-
.../yarn/service/api/records/Artifact.java | 1 -
.../yarn/service/api/records/Component.java | 1 -
.../yarn/service/api/records/ConfigFile.java | 1 -
.../yarn/service/api/records/Configuration.java | 1 -
.../yarn/service/api/records/Container.java | 1 -
.../hadoop/yarn/service/api/records/Error.java | 1 -
.../service/api/records/KerberosPrincipal.java | 2 -
.../api/records/PlacementConstraint.java | 3 -
.../service/api/records/PlacementPolicy.java | 3 -
.../service/api/records/PlacementScope.java | 3 -
.../yarn/service/api/records/PlacementType.java | 3 -
.../service/api/records/ReadinessCheck.java | 1 -
.../yarn/service/api/records/Resource.java | 1 -
.../api/records/ResourceInformation.java | 2 -
.../yarn/service/api/records/Service.java | 1 -
.../yarn/service/api/records/ServiceState.java | 1 -
.../yarn/service/api/records/ServiceStatus.java | 1 -
.../yarn/submarine/client/cli/CliConstants.java | 7 +
.../yarn/submarine/client/cli/CliUtils.java | 10 +-
.../yarn/submarine/client/cli/RunJobCli.java | 51 +-
.../submarine/client/cli/param/Quicklink.java | 71 +
.../client/cli/param/RunJobParameters.java | 70 +-
.../fs/DefaultRemoteDirectoryManager.java | 21 +-
.../common/fs/RemoteDirectoryManager.java | 4 +-
.../common/FSBasedSubmarineStorageImpl.java | 4 +-
.../yarnservice/YarnServiceJobSubmitter.java | 208 +-
.../runtimes/yarnservice/YarnServiceUtils.java | 63 +-
.../yarnservice/TestYarnServiceRunJobCli.java | 362 +-
.../common/fs/MockRemoteDirectoryManager.java | 7 +-
.../hadoop/yarn/client/api/YarnClient.java | 60 +-
.../yarn/client/api/impl/YarnClientImpl.java | 33 +-
.../hadoop/yarn/client/cli/ClusterCLI.java | 17 +
.../yarn/client/cli/NodeAttributesCLI.java | 715 ++++
.../apache/hadoop/yarn/client/cli/NodeCLI.java | 13 +-
.../hadoop/yarn/client/cli/TestClusterCLI.java | 32 +-
.../yarn/client/cli/TestNodeAttributesCLI.java | 537 +++
.../hadoop/yarn/client/cli/TestYarnCLI.java | 31 +-
.../ApplicationClientProtocolPBClientImpl.java | 55 +
.../ApplicationClientProtocolPBServiceImpl.java | 65 +
.../PlacementConstraintFromProtoConverter.java | 10 +-
.../pb/PlacementConstraintToProtoConverter.java | 11 +
.../pb/GetAttributesToNodesRequestPBImpl.java | 176 +
.../pb/GetAttributesToNodesResponsePBImpl.java | 207 +
.../GetClusterNodeAttributesRequestPBImpl.java | 75 +
.../GetClusterNodeAttributesResponsePBImpl.java | 160 +
.../pb/GetNodesToAttributesRequestPBImpl.java | 132 +
.../pb/GetNodesToAttributesResponsePBImpl.java | 181 +
.../impl/pb/NodeAttributeInfoPBImpl.java | 147 +
.../records/impl/pb/NodeAttributeKeyPBImpl.java | 140 +
.../records/impl/pb/NodeAttributePBImpl.java | 170 +
.../api/records/impl/pb/NodeReportPBImpl.java | 44 +-
.../impl/pb/NodeToAttributeValuePBImpl.java | 137 +
.../hadoop/yarn/nodelabels/AbstractLabel.java | 71 +
.../AttributeExpressionOperation.java | 26 +
.../hadoop/yarn/nodelabels/AttributeValue.java | 53 +
.../nodelabels/CommonNodeLabelsManager.java | 32 +-
.../nodelabels/FileSystemNodeLabelsStore.java | 270 +-
.../yarn/nodelabels/NodeAttributeStore.java | 77 +
.../yarn/nodelabels/NodeAttributesManager.java | 137 +
.../hadoop/yarn/nodelabels/NodeLabelUtil.java | 155 +
.../hadoop/yarn/nodelabels/NodeLabelsStore.java | 32 +-
.../NonAppendableFSNodeLabelStore.java | 46 +-
.../hadoop/yarn/nodelabels/RMNodeAttribute.java | 98 +
.../hadoop/yarn/nodelabels/RMNodeLabel.java | 110 +-
.../yarn/nodelabels/StringAttributeValue.java | 61 +
.../nodelabels/store/AbstractFSNodeStore.java | 213 +
.../yarn/nodelabels/store/FSStoreOpHandler.java | 131 +
.../hadoop/yarn/nodelabels/store/StoreOp.java | 49 +
.../nodelabels/store/op/AddClusterLabelOp.java | 73 +
.../store/op/AddNodeToAttributeLogOp.java | 71 +
.../nodelabels/store/op/FSNodeStoreLogOp.java | 52 +
.../store/op/NodeAttributeMirrorOp.java | 64 +
.../nodelabels/store/op/NodeLabelMirrorOp.java | 85 +
.../yarn/nodelabels/store/op/NodeToLabelOp.java | 75 +
.../store/op/RemoveClusterLabelOp.java | 75 +
.../store/op/RemoveNodeToAttributeLogOp.java | 71 +
.../store/op/ReplaceNodeToAttributeLogOp.java | 73 +
.../yarn/nodelabels/store/op/package-info.java | 21 +
.../yarn/nodelabels/store/package-info.java | 21 +
...nagerAdministrationProtocolPBClientImpl.java | 26 +-
...agerAdministrationProtocolPBServiceImpl.java | 31 +-
.../impl/pb/NodeToAttributesPBImpl.java | 164 +
.../NodesToAttributesMappingRequestPBImpl.java | 197 +
.../NodesToAttributesMappingResponsePBImpl.java | 50 +
...emoveFromClusterNodeLabelsRequestPBImpl.java | 24 +-
.../src/main/resources/yarn-default.xml | 86 +
.../hadoop/yarn/api/TestPBImplRecords.java | 114 +-
.../DummyCommonNodeLabelsManager.java | 8 +-
.../TestFileSystemNodeLabelsStore.java | 16 +-
.../yarn/nodelabels/TestNodeLabelUtil.java | 51 +
.../hadoop/yarn/server/AMRMClientRelayer.java | 412 +-
.../protocolrecords/NodeHeartbeatRequest.java | 17 +
.../impl/pb/NodeHeartbeatRequestPBImpl.java | 52 +
.../metrics/AMRMClientRelayerMetrics.java | 368 ++
.../yarn/server/metrics/package-info.java | 18 +
.../yarn/server/uam/UnmanagedAMPoolManager.java | 30 +-
.../server/uam/UnmanagedApplicationManager.java | 16 +-
.../hadoop/yarn/server/utils/BuilderUtils.java | 6 +-
.../yarn_server_common_service_protos.proto | 5 +
.../yarn/server/MockResourceManagerFacade.java | 46 +-
.../yarn/server/TestAMRMClientRelayer.java | 2 +-
.../protocolrecords/TestProtocolRecords.java | 12 +
.../metrics/TestAMRMClientRelayerMetrics.java | 513 +++
.../uam/TestUnmanagedApplicationManager.java | 2 +-
.../server/nodemanager/ContainerExecutor.java | 3 +-
.../nodemanager/LinuxContainerExecutor.java | 15 +-
.../yarn/server/nodemanager/NodeManager.java | 70 +-
.../server/nodemanager/NodeStatusUpdater.java | 14 +
.../nodemanager/NodeStatusUpdaterImpl.java | 92 +-
.../amrmproxy/FederationInterceptor.java | 19 +-
.../containermanager/ContainerManagerImpl.java | 29 +-
.../launcher/ContainerLaunch.java | 5 +-
.../linux/resources/ResourceHandlerModule.java | 15 +
.../TrafficControlBandwidthHandlerImpl.java | 2 +-
.../runtime/DockerLinuxContainerRuntime.java | 95 +-
.../JavaSandboxLinuxContainerRuntime.java | 28 +-
.../runtime/docker/DockerCommandExecutor.java | 73 +-
.../runtime/docker/DockerInspectCommand.java | 16 +-
.../linux/runtime/docker/DockerRmCommand.java | 11 +-
.../localizer/ResourceLocalizationService.java | 87 +-
.../AbstractNodeDescriptorsProvider.java | 197 +
.../nodelabels/AbstractNodeLabelsProvider.java | 149 -
.../ConfigurationNodeAttributesProvider.java | 156 +
.../ConfigurationNodeLabelsProvider.java | 13 +-
.../nodelabels/NodeAttributesProvider.java | 32 +
.../nodelabels/NodeDescriptorsProvider.java | 45 +
.../nodelabels/NodeDescriptorsScriptRunner.java | 84 +
.../nodelabels/NodeLabelsProvider.java | 22 +-
.../ScriptBasedNodeAttributesProvider.java | 158 +
.../ScriptBasedNodeLabelsProvider.java | 126 +-
.../nodemanager/nodelabels/package-info.java | 28 +
.../recovery/NMLeveldbStateStoreService.java | 173 +-
.../recovery/NMStateStoreService.java | 29 +-
.../impl/container-executor.c | 153 +-
.../impl/container-executor.h | 8 +-
.../main/native/container-executor/impl/main.c | 12 +-
.../container-executor/impl/utils/docker-util.c | 5 +-
.../container-executor/impl/utils/docker-util.h | 30 +-
.../test/test-container-executor.c | 147 +
.../server/nodemanager/TestNodeManager.java | 2 +-
.../TestNodeStatusUpdaterForLabels.java | 52 +-
.../TestableFederationInterceptor.java | 5 +-
.../launcher/TestContainerLaunch.java | 81 +
.../runtime/TestDockerContainerRuntime.java | 47 +-
.../docker/TestDockerCommandExecutor.java | 23 +-
.../runtime/docker/TestDockerRmCommand.java | 35 +-
...TestConfigurationNodeAttributesProvider.java | 262 ++
.../TestConfigurationNodeLabelsProvider.java | 26 +-
.../TestScriptBasedNodeAttributesProvider.java | 251 ++
.../TestScriptBasedNodeLabelsProvider.java | 18 +-
.../recovery/NMMemoryStateStoreService.java | 18 +-
.../TestNMLeveldbStateStoreService.java | 269 +-
.../server/resourcemanager/AdminService.java | 106 +
.../server/resourcemanager/ClientRMService.java | 71 +-
.../resourcemanager/DefaultAMSProcessor.java | 3 +-
.../resourcemanager/RMActiveServiceContext.java | 14 +
.../server/resourcemanager/RMAppManager.java | 5 +-
.../yarn/server/resourcemanager/RMContext.java | 5 +
.../server/resourcemanager/RMContextImpl.java | 11 +
.../server/resourcemanager/ResourceManager.java | 14 +-
.../resourcemanager/ResourceTrackerService.java | 31 +-
.../ProportionalCapacityPreemptionPolicy.java | 3 +
.../FileSystemNodeAttributeStore.java | 102 +
.../nodelabels/NodeAttributesManagerImpl.java | 754 ++++
.../nodelabels/NodeAttributesStoreEvent.java | 52 +
.../NodeAttributesStoreEventType.java | 26 +
.../nodelabels/NodeLabelsUtils.java | 19 +
.../server/resourcemanager/rmnode/RMNode.java | 6 +
.../resourcemanager/rmnode/RMNodeImpl.java | 11 +
.../scheduler/SchedulerNode.java | 11 +
.../scheduler/capacity/CapacityScheduler.java | 42 +-
.../allocator/RegularContainerAllocator.java | 3 +-
.../constraint/PlacementConstraintsUtil.java | 148 +-
.../NodeAttributesUpdateSchedulerEvent.java | 41 +
.../scheduler/event/SchedulerEventType.java | 1 +
.../LocalityAppPlacementAllocator.java | 4 +
.../webapp/dao/NodeAttributeInfo.java | 65 +
.../webapp/dao/NodeAttributesInfo.java | 49 +
.../resourcemanager/webapp/dao/NodeInfo.java | 10 +
.../yarn/server/resourcemanager/MockNodes.java | 11 +
.../resourcemanager/NodeAttributeTestUtils.java | 54 +
.../server/resourcemanager/TestAppManager.java | 3 +
.../resourcemanager/TestClientRMService.java | 246 +-
.../resourcemanager/TestRMAdminService.java | 171 +-
.../TestResourceTrackerService.java | 88 +
...alCapacityPreemptionPolicyMockFramework.java | 5 +
...ionalCapacityPreemptionPolicyIntraQueue.java | 90 +-
.../nodelabels/NullRMNodeLabelsManager.java | 7 +
.../TestFileSystemNodeAttributeStore.java | 279 ++
.../nodelabels/TestNodeAttributesManager.java | 364 ++
.../capacity/TestContainerAllocation.java | 68 +
.../scheduler/capacity/TestUtils.java | 20 +-
...stSingleConstraintAppPlacementAllocator.java | 192 +-
.../webapp/TestRMWebServicesNodeLabels.java | 2 +-
.../webapp/TestRMWebServicesNodes.java | 2 +-
.../server/router/clientrm/ClientMethod.java | 71 +
.../DefaultClientRequestInterceptor.java | 25 +
.../clientrm/FederationClientInterceptor.java | 121 +-
.../router/clientrm/RouterClientRMService.java | 28 +
.../router/clientrm/RouterYarnClientUtils.java | 55 +
.../DefaultRMAdminRequestInterceptor.java | 9 +
.../router/rmadmin/RouterRMAdminService.java | 10 +
.../PassThroughClientRequestInterceptor.java | 25 +
.../TestFederationClientInterceptor.java | 37 +-
.../clientrm/TestRouterYarnClientUtils.java | 57 +
.../PassThroughRMAdminRequestInterceptor.java | 9 +
.../reader/TimelineReaderWebServices.java | 4 +-
.../TestTimelineReaderWebServicesBasicAcl.java | 11 +-
.../src/site/markdown/NodeAttributes.md | 156 +
.../site/markdown/PlacementConstraints.md.vm | 2 +
.../main/webapp/app/controllers/application.js | 5 +-
.../app/controllers/yarn-app/components.js | 2 +-
.../controllers/yarn-component-instance/info.js | 5 +-
.../yarn-component-instances/info.js | 3 +-
.../src/main/webapp/app/initializers/loader.js | 3 +
.../src/main/webapp/app/routes/application.js | 4 +-
.../app/routes/yarn-component-instance/info.js | 4 +-
.../app/serializers/yarn-component-instance.js | 1 -
.../webapp/app/serializers/yarn-container.js | 2 +-
.../app/serializers/yarn-service-component.js | 2 +-
.../app/serializers/yarn-timeline-container.js | 2 +-
.../webapp/app/templates/yarn-app/configs.hbs | 7 +-
.../templates/yarn-component-instance/info.hbs | 4 -
pom.xml | 5 +-
1034 files changed, 72254 insertions(+), 47687 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
index 2693301,0000000..387caa1
mode 100644,000000..100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournaledEditsCache.java
@@@ -1,412 -1,0 +1,412 @@@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.qjournal.server;
+
+import com.google.common.annotations.VisibleForTesting;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.server.namenode.EditLogFileOutputStream;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOp;
+import org.apache.hadoop.util.AutoCloseableLock;
+
+/**
+ * An in-memory cache of edits in their serialized form. This is used to serve
+ * the {@link Journal#getJournaledEdits(long, int)} call, used by the
+ * QJM when {@value DFSConfigKeys#DFS_HA_TAILEDITS_INPROGRESS_KEY} is
+ * enabled.
+ *
+ * <p>When a batch of edits is received by the JournalNode, it is put into this
+ * cache via {@link #storeEdits(byte[], long, long, int)}. Edits must be
+ * stored contiguously; if a batch of edits is stored that does not align with
+ * the previously stored edits, the cache will be cleared before storing new
+ * edits to avoid gaps. This decision is made because gaps are only handled
+ * when in recovery mode, which the cache is not intended to be used for.
+ *
+ * <p>Batches of edits are stored in a {@link TreeMap} mapping the starting
+ * transaction ID of the batch to the data buffer. Upon retrieval, the
+ * relevant data buffers are concatenated together and a header is added
+ * to construct a fully-formed edit data stream.
+ *
+ * <p>The cache is of a limited size capacity determined by
+ * {@value DFSConfigKeys#DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY}. If the capacity
+ * is exceeded after adding a new batch of edits, batches of edits are removed
+ * until the total size is less than the capacity, starting from the ones
+ * containing the oldest transactions. Transactions range in size, but a
+ * decent rule of thumb is that 200 bytes are needed per transaction. Monitoring
+ * the {@link JournalMetrics#rpcRequestCacheMissAmount} metric is recommended
+ * to determine if the cache is too small; it will indicate both how many
+ * cache misses occurred, and how many more transactions would have been
+ * needed in the cache to serve the request.
+ */
+class JournaledEditsCache {
+
+ private static final int INVALID_LAYOUT_VERSION = 0;
+ private static final long INVALID_TXN_ID = -1;
+
+ /** The capacity, in bytes, of this cache. */
+ private final int capacity;
+
+ /**
+ * Read/write lock pair wrapped in AutoCloseable; these refer to the same
+ * underlying lock.
+ */
+ private final AutoCloseableLock readLock;
+ private final AutoCloseableLock writeLock;
+
+ // ** Start lock-protected fields **
+
+ /**
+ * Stores the actual data as a mapping of the StartTxnId of a batch of edits
+ * to the serialized batch of edits. Stores only contiguous ranges; that is,
+ * the last transaction ID in one batch is always one less than the first
+ * transaction ID in the next batch. Though the map is protected by the lock,
+ * individual data buffers are immutable and can be accessed without locking.
+ */
+ private final NavigableMap<Long, byte[]> dataMap = new TreeMap<>();
+ /** Stores the layout version currently present in the cache. */
+ private int layoutVersion = INVALID_LAYOUT_VERSION;
+ /** Stores the serialized version of the header for the current version. */
+ private ByteBuffer layoutHeader;
+
+ /**
+ * The lowest/highest transaction IDs present in the cache.
+ * {@value INVALID_TXN_ID} if there are no transactions in the cache.
+ */
+ private long lowestTxnId;
+ private long highestTxnId;
+ /**
+ * The lowest transaction ID that was ever present in the cache since last
+ * being reset (i.e. since initialization or since reset due to being out of
+ * sync with the Journal). Until the cache size goes above capacity, this is
+ * equal to lowestTxnId.
+ */
+ private long initialTxnId;
+ /** The current total size of all buffers in this cache. */
+ private int totalSize;
+
+ // ** End lock-protected fields **
+
+ JournaledEditsCache(Configuration conf) {
+ capacity = conf.getInt(DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY,
+ DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_DEFAULT);
+ if (capacity > 0.9 * Runtime.getRuntime().maxMemory()) {
+ Journal.LOG.warn(String.format("Cache capacity is set at %d bytes but " +
+ "maximum JVM memory is only %d bytes. It is recommended that you " +
+ "decrease the cache size or increase the heap size.",
+ capacity, Runtime.getRuntime().maxMemory()));
+ }
+ Journal.LOG.info("Enabling the journaled edits cache with a capacity " +
+ "of bytes: " + capacity);
+ ReadWriteLock lock = new ReentrantReadWriteLock(true);
+ readLock = new AutoCloseableLock(lock.readLock());
+ writeLock = new AutoCloseableLock(lock.writeLock());
+ initialize(INVALID_TXN_ID);
+ }
+
+ /**
+ * Fetch the data for edits starting at the specific transaction ID, fetching
+ * up to {@code maxTxns} transactions. Populates a list of output buffers
+ * which contains a serialized version of the edits, and returns the count of
+ * edits contained within the serialized buffers. The serialized edits are
+ * prefixed with a standard edit log header containing information about the
+ * layout version. The transactions returned are guaranteed to have contiguous
+ * transaction IDs.
+ *
+ * If {@code requestedStartTxn} is higher than the highest transaction which
+ * has been added to this cache, a response with an empty buffer and a
+ * transaction count of 0 will be returned. If {@code requestedStartTxn} is
+ * lower than the lowest transaction currently contained in this cache, or no
+ * transactions have yet been added to the cache, an exception will be thrown.
+ *
+ * @param requestedStartTxn The ID of the first transaction to return. If any
+ * transactions are returned, it is guaranteed that
+ * the first one will have this ID.
+ * @param maxTxns The maximum number of transactions to return.
+ * @param outputBuffers A list to populate with output buffers. When
+ * concatenated, these form a full response.
+ * @return The number of transactions contained within the set of output
+ * buffers.
+ * @throws IOException If transactions are requested which cannot be served
+ * by this cache.
+ */
+ int retrieveEdits(long requestedStartTxn, int maxTxns,
+ List<ByteBuffer> outputBuffers) throws IOException {
+ int txnCount = 0;
+
+ try (AutoCloseableLock l = readLock.acquire()) {
+ if (lowestTxnId == INVALID_TXN_ID || requestedStartTxn < lowestTxnId) {
+ throw getCacheMissException(requestedStartTxn);
+ } else if (requestedStartTxn > highestTxnId) {
+ return 0;
+ }
+ outputBuffers.add(layoutHeader);
+ Iterator<Map.Entry<Long, byte[]>> incrBuffIter =
+ dataMap.tailMap(dataMap.floorKey(requestedStartTxn), true)
+ .entrySet().iterator();
+ long prevTxn = requestedStartTxn;
+ byte[] prevBuf = null;
+ // Stop when maximum transactions reached...
+ while ((txnCount < maxTxns) &&
+ // ... or there are no more entries ...
+ (incrBuffIter.hasNext() || prevBuf != null)) {
+ long currTxn;
+ byte[] currBuf;
+ if (incrBuffIter.hasNext()) {
+ Map.Entry<Long, byte[]> ent = incrBuffIter.next();
+ currTxn = ent.getKey();
+ currBuf = ent.getValue();
+ } else {
+ // This accounts for the trailing entry
+ currTxn = highestTxnId + 1;
+ currBuf = null;
+ }
+ if (prevBuf != null) { // True except for the first loop iteration
+ outputBuffers.add(ByteBuffer.wrap(prevBuf));
+ // if prevTxn < requestedStartTxn, the extra transactions will get
+ // removed after the loop, so don't include them in the txn count
+ txnCount += currTxn - Math.max(requestedStartTxn, prevTxn);
+ }
+ prevTxn = currTxn;
+ prevBuf = currBuf;
+ }
+ // Release the lock before doing operations on the buffers (deserializing
+ // to find transaction boundaries, and copying into an output buffer)
+ }
+ // Remove extra leading transactions in the first buffer
+ ByteBuffer firstBuf = outputBuffers.get(1); // 0th is the header
+ firstBuf.position(
+ findTransactionPosition(firstBuf.array(), requestedStartTxn));
+ // Remove trailing transactions in the last buffer if necessary
+ if (txnCount > maxTxns) {
+ ByteBuffer lastBuf = outputBuffers.get(outputBuffers.size() - 1);
+ int limit =
+ findTransactionPosition(lastBuf.array(), requestedStartTxn + maxTxns);
+ lastBuf.limit(limit);
+ txnCount = maxTxns;
+ }
+
+ return txnCount;
+ }
+
+ /**
+ * Store a batch of serialized edits into this cache. Removes old batches
+ * as necessary to keep the total size of the cache below the capacity.
+ * See the class Javadoc for more info.
+ *
+ * This attempts to always handle malformed inputs gracefully rather than
+ * throwing an exception, to allow the rest of the Journal's operations
+ * to proceed normally.
+ *
+ * @param inputData A buffer containing edits in serialized form
+ * @param newStartTxn The txn ID of the first edit in {@code inputData}
+ * @param newEndTxn The txn ID of the last edit in {@code inputData}
+ * @param newLayoutVersion The version of the layout used to serialize
+ * the edits
+ */
+ void storeEdits(byte[] inputData, long newStartTxn, long newEndTxn,
+ int newLayoutVersion) {
+ if (newStartTxn < 0 || newEndTxn < newStartTxn) {
+ Journal.LOG.error(String.format("Attempted to cache data of length %d " +
+ "with newStartTxn %d and newEndTxn %d",
+ inputData.length, newStartTxn, newEndTxn));
+ return;
+ }
+ try (AutoCloseableLock l = writeLock.acquire()) {
+ if (newLayoutVersion != layoutVersion) {
+ try {
+ updateLayoutVersion(newLayoutVersion, newStartTxn);
+ } catch (IOException ioe) {
+ Journal.LOG.error(String.format("Unable to save new edits [%d, %d] " +
+ "due to exception when updating to new layout version %d",
+ newStartTxn, newEndTxn, newLayoutVersion), ioe);
+ return;
+ }
+ } else if (lowestTxnId == INVALID_TXN_ID) {
+ Journal.LOG.info("Initializing edits cache starting from txn ID " +
+ newStartTxn);
+ initialize(newStartTxn);
+ } else if (highestTxnId + 1 != newStartTxn) {
+ // Cache is out of sync; clear to avoid storing noncontiguous regions
+ Journal.LOG.error(String.format("Edits cache is out of sync; " +
+ "looked for next txn id at %d but got start txn id for " +
+ "cache put request at %d. Reinitializing at new request.",
+ highestTxnId + 1, newStartTxn));
+ initialize(newStartTxn);
+ }
+
+ while ((totalSize + inputData.length) > capacity && !dataMap.isEmpty()) {
+ Map.Entry<Long, byte[]> lowest = dataMap.firstEntry();
+ dataMap.remove(lowest.getKey());
+ totalSize -= lowest.getValue().length;
+ }
+ if (inputData.length > capacity) {
+ initialize(INVALID_TXN_ID);
+ Journal.LOG.warn(String.format("A single batch of edits was too " +
+ "large to fit into the cache: startTxn = %d, endTxn = %d, " +
+ "input length = %d. The capacity of the cache (%s) must be " +
+ "increased for it to work properly (current capacity %d)." +
+ "Cache is now empty.",
+ newStartTxn, newEndTxn, inputData.length,
+ DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, capacity));
+ return;
+ }
+ if (dataMap.isEmpty()) {
+ lowestTxnId = newStartTxn;
+ } else {
+ lowestTxnId = dataMap.firstKey();
+ }
+
+ dataMap.put(newStartTxn, inputData);
+ highestTxnId = newEndTxn;
+ totalSize += inputData.length;
+ }
+ }
+
+ /**
+ * Skip through a given stream of edits until the given transaction ID is
+ * found. Return the number of bytes that appear prior to the given
+ * transaction.
+ *
+ * @param buf A buffer containing a stream of serialized edits
+ * @param txnId The transaction ID to search for
+ * @return The number of bytes appearing in {@code buf} <i>before</i>
+ * the start of the transaction with ID {@code txnId}.
+ */
+ private int findTransactionPosition(byte[] buf, long txnId)
+ throws IOException {
+ ByteArrayInputStream bais = new ByteArrayInputStream(buf);
+ FSEditLogLoader.PositionTrackingInputStream tracker =
+ new FSEditLogLoader.PositionTrackingInputStream(bais);
+ FSEditLogOp.Reader reader = FSEditLogOp.Reader.create(
+ new DataInputStream(tracker), tracker, layoutVersion);
+ long previousPos = 0;
+ while (reader.scanOp() < txnId) {
+ previousPos = tracker.getPos();
+ }
+ // tracker is backed by a byte[]; position cannot go above an integer
+ return (int) previousPos;
+ }
+
+ /**
+ * Update the layout version of the cache. This clears out all existing
+ * entries, and populates the new layout version and header for that version.
+ *
+ * @param newLayoutVersion The new layout version to be stored in the cache
+ * @param newStartTxn The new lowest transaction in the cache
+ */
+ private void updateLayoutVersion(int newLayoutVersion, long newStartTxn)
+ throws IOException {
+ StringBuilder logMsg = new StringBuilder()
+ .append("Updating edits cache to use layout version ")
+ .append(newLayoutVersion)
+ .append(" starting from txn ID ")
+ .append(newStartTxn);
+ if (layoutVersion != INVALID_LAYOUT_VERSION) {
+ logMsg.append("; previous version was ").append(layoutVersion)
+ .append("; old entries will be cleared.");
+ }
- Journal.LOG.info(logMsg);
++ Journal.LOG.info(logMsg.toString());
+ initialize(newStartTxn);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ EditLogFileOutputStream.writeHeader(newLayoutVersion,
+ new DataOutputStream(baos));
+ layoutVersion = newLayoutVersion;
+ layoutHeader = ByteBuffer.wrap(baos.toByteArray());
+ }
+
+ /**
+ * Initialize the cache back to a clear state.
+ *
+ * @param newInitialTxnId The new lowest transaction ID stored in the cache.
+ * This should be {@value INVALID_TXN_ID} if the cache
+ * is to remain empty at this time.
+ */
+ private void initialize(long newInitialTxnId) {
+ dataMap.clear();
+ totalSize = 0;
+ initialTxnId = newInitialTxnId;
+ lowestTxnId = initialTxnId;
+ highestTxnId = INVALID_TXN_ID; // this will be set later
+ }
+
+ /**
+ * Return the underlying data buffer used to store information about the
+ * given transaction ID.
+ *
+ * @param txnId Transaction ID whose containing buffer should be fetched.
+ * @return The data buffer for the transaction
+ */
+ @VisibleForTesting
+ byte[] getRawDataForTests(long txnId) {
+ try (AutoCloseableLock l = readLock.acquire()) {
+ return dataMap.floorEntry(txnId).getValue();
+ }
+ }
+
+ private CacheMissException getCacheMissException(long requestedTxnId) {
+ if (lowestTxnId == INVALID_TXN_ID) {
+ return new CacheMissException(0, "Cache is empty; either it was never " +
+ "written to or the last write overflowed the cache capacity.");
+ } else if (requestedTxnId < initialTxnId) {
+ return new CacheMissException(initialTxnId - requestedTxnId,
+ "Cache started at txn ID %d but requested txns starting at %d.",
+ initialTxnId, requestedTxnId);
+ } else {
+ return new CacheMissException(lowestTxnId - requestedTxnId,
+ "Oldest txn ID available in the cache is %d, but requested txns " +
+ "starting at %d. The cache size (%s) may need to be increased " +
+ "to hold more transactions (currently %d bytes containing %d " +
+ "transactions)", lowestTxnId, requestedTxnId,
+ DFSConfigKeys.DFS_JOURNALNODE_EDIT_CACHE_SIZE_KEY, capacity,
+ highestTxnId - lowestTxnId + 1);
+ }
+ }
+
+ static class CacheMissException extends IOException {
+
+ private static final long serialVersionUID = 0L;
+
+ private final long cacheMissAmount;
+
+ CacheMissException(long cacheMissAmount, String msgFormat,
+ Object... msgArgs) {
+ super(String.format(msgFormat, msgArgs));
+ this.cacheMissAmount = cacheMissAmount;
+ }
+
+ long getCacheMissAmount() {
+ return cacheMissAmount;
+ }
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ha/EditLogTailer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c04e0c0e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
----------------------------------------------------------------------
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org